refactor: Move decreaseOwnerCount onto AccountRootEntry

Replace the entry-taking decreaseOwnerCount free function with a
writable member of AccountRootEntry, and drop the temporary friend
declaration from the previous change. The member uses the entry's view
and journal, so call sites pass their journal to the entry constructor.
The AccountID-based overload stays as a shim.

.codecov.yml

@@ -58,19 +58,3 @@ ignore:
   - "src/tests/"
   - "include/xrpl/beast/test/"
   - "include/xrpl/beast/unit_test/"
-  # Telemetry modules. Telemetry compiles in by default (conanfile.py,
-  # CMakeLists.txt). SpanGuardScope.cpp sends spans to an in-memory exporter,
-  # but no unit test sends spans through the real OTLP export path, so these
-  # files are left out of the coverage report. This block belongs on the
-  # earliest branch that adds telemetry code — codecov config only flows
-  # child-ward, so an ignore added on a later branch can never cover the
-  # branches before it.
-  - "src/xrpld/telemetry/"
-  - "src/libxrpl/telemetry/"
-  - "include/xrpl/telemetry/"
-  - "src/libxrpl/beast/insight/OTelCollector.cpp"
-  - "include/xrpl/beast/insight/OTelCollector.h"
-  # Per-module span-name and span-label constant headers: compile-time
-  # constants only, colocated with their subsystem rather than under telemetry/.
-  - "**/*SpanNames.h"
-  - "**/*SpanLabels.h"

.cspell.config.yaml

@@ -19,7 +19,6 @@ dictionaries:
 ignoreRegExpList:
   - /\b[rs][1-9A-HJ-NP-Za-km-z]{25,34}/g # addresses and seeds
   - /\bC[A-Z0-9]{15}/g # CTIDs
-  - /\bXRPL_METRIC_[A-Z_]+/g # telemetry macro names; the splitter emits subwords like ISTOGRAM
   - /\b(XRPL|BEAST)_[A-Z_0-9]+_H_INCLUDED+/g # include guards
   - /\b(XRPL|BEAST)_[A-Z_0-9]+_H+/g # include guards
   - /::[a-z:_]+/g # things from other namespaces
@@ -70,7 +69,6 @@ words:
   - Btrfs
   - Buildx
   - canonicality
-  - CGNAT
   - canonicalised
   - cctools
   - changespq
@@ -105,8 +103,6 @@ words:
   - dearmor
   - decryptor
   - dedented
-  - Dedup
-  - dedup
   - deleteme
   - demultiplexer
   - deserializaton
@@ -117,17 +113,14 @@ words:
   - distro
   - doxyfile
   - dsymutil
-  - dthms
   - dxrpl
   - elgamal
   - enabled
   - enablerepo
   - endmacro
   - envrc
-  - EOCFG
   - exceptioned
   - EXPECT_STREQ
-  - exfiltration
   - Falco
   - fcontext
   - finalizers
@@ -136,8 +129,6 @@ words:
   - fsanitize
   - funclets
   - Gamal
-  - gantt
-  - Gantt
   - gcov
   - gcovr
   - ghead
@@ -145,16 +136,12 @@ words:
   - godexsoft
   - gpgcheck
   - gpgkey
-  - havetxset
   - Hinnant
   - hotwallet
-  - hicpp
-  - htpasswd
   - hwaddress
   - hwrap
   - ifndef
   - inequation
-  - initialiser
   - Injectivity
   - insuf
   - insuff
@@ -166,7 +153,6 @@ words:
   - itype
   - jemalloc
   - jlog
-  - jmeter
   - jtnofill
   - keylet
   - keylets
@@ -184,7 +170,6 @@ words:
   - libxrpl
   - llection
   - LOCALGOOD
-  - logql
   - logwstream
   - Lombrozo
   - lresolv
@@ -194,10 +179,11 @@ words:
   - mathbunnyru
   - mcmodel
   - MEMORYSTATUSEX
+  - MPTAMM
+  - MPTDEX
   - Merkle
   - misprediction
   - missingok
-  - MPTAMM
   - mptbalance
   - MPTDEX
   - mptflags
@@ -227,13 +213,11 @@ words:
   - nixfmt
   - nixos
   - nixpkgs
-  - NETOP
   - NOLINT
   - NOLINTNEXTLINE
   - nonxrp
   - noreplace
   - noripple
-  - nostd
   - nostdinc
   - notifempty
   - nudb
@@ -242,8 +226,6 @@ words:
   - Nyffenegger
   - onlatest
   - ostr
-  - otelc
-  - otelcol
   - otool
   - oxalica
   - pargs
@@ -254,12 +236,8 @@ words:
   - permdex
   - perminute
   - permissioned
-  - pgrep
-  - pkill
-  - pimpl
   - pointee
   - populator
-  - pratik
   - preauth
   - preauthorization
   - preauthorize
@@ -275,15 +253,12 @@ words:
   - qalloc
   - qbsprofile
   - queuable
-  - ransitioned
   - Raphson
   - rcflags
   - reencrypted
-  - reparent
   - replayer
   - repodata
   - repomd
-  - reqps
   - rerandomize
   - rerandomization
   - rerandomized
@@ -307,11 +282,6 @@ words:
   - rustfmt
   - rustup
   - sahyadri
-  - sanitisation
-  - sanitise
-  - sanitised
-  - sanitiser
-  - sanitising
   - Satoshi
   - scons
   - Schnorr
@@ -332,7 +302,6 @@ words:
   - soci
   - socidb
   - Sonatype
-  - speciality
   - sponsee
   - sponsees
   - SRPMS
@@ -340,7 +309,6 @@ words:
   - statsd
   - STATSDCOLLECTOR
   - stissue
-  - stledgerentry
   - stnum
   - stobj
   - stobject
@@ -354,14 +322,12 @@ words:
   - superpeer
   - superpeers
   - Swatinem
-  - synthesise
   - takergets
   - takerpays
   - ters
   - TMEndpointv2
   - toolchain
   - tparam
-  - traceql
   - trixie
   - tx
   - txid
@@ -369,9 +335,7 @@ words:
   - txjson
   - txn
   - txns
-  - txqueue
   - txs
-  - txset
   - ubsan
   - UBSAN
   - ufdio
@@ -385,8 +349,6 @@ words:
   - unfindable
   - unflatten
   - unfund
-  - ungated
-  - unheld
   - unimpair
   - unroutable
   - unscalable
@@ -397,20 +359,17 @@ words:
   - unsquelch
   - unsquelched
   - unsquelching
-  - unstored
   - unsuffixed
   - unvalidated
   - unveto
   - unvetoed
   - upvotes
   - USDB
-  - utilisation
   - variadics
   - venv
   - vfalco
   - vinnie
   - wasmi
-  - werror
   - wextra
   - wptr
   - writeme
@@ -428,8 +387,6 @@ words:
   - xrplf
   - xxhash
   - xxhasher
-  - xychart
-  - zpages
   - zstdio
-  - ripplex
-  - mseconds
+  - CGNAT
+  - ungated

.github/scripts/levelization/results/loops.txt

@@ -7,12 +7,6 @@ Loop: xrpld.app xrpld.rpc
 Loop: xrpld.app xrpld.shamap
   xrpld.shamap > xrpld.app
 
-Loop: xrpld.app xrpld.telemetry
-  xrpld.telemetry == xrpld.app
-
 Loop: xrpld.overlay xrpld.rpc
   xrpld.rpc ~= xrpld.overlay
 
-Loop: xrpld.overlay xrpld.telemetry
-  xrpld.telemetry ~= xrpld.overlay
-

.github/scripts/levelization/results/ordering.txt

@@ -53,11 +53,6 @@ libxrpl.shamap > xrpl.basics
 libxrpl.shamap > xrpl.nodestore
 libxrpl.shamap > xrpl.protocol
 libxrpl.shamap > xrpl.shamap
-libxrpl.telemetry > xrpl.basics
-libxrpl.telemetry > xrpl.config
-libxrpl.telemetry > xrpl.core
-libxrpl.telemetry > xrpl.protocol
-libxrpl.telemetry > xrpl.telemetry
 libxrpl.tx > xrpl.basics
 libxrpl.tx > xrpl.conditions
 libxrpl.tx > xrpl.core
@@ -65,7 +60,6 @@ libxrpl.tx > xrpl.json
 libxrpl.tx > xrpl.ledger
 libxrpl.tx > xrpl.protocol
 libxrpl.tx > xrpl.server
-libxrpl.tx > xrpl.telemetry
 libxrpl.tx > xrpl.tx
 test.app > test.jtx
 test.app > test.unit_test
@@ -136,12 +130,8 @@ test.ledger > xrpl.json
 test.ledger > xrpl.ledger
 test.ledger > xrpl.protocol
 test.nodestore > test.jtx
-test.nodestore > test.unit_test
 test.nodestore > xrpl.basics
-test.nodestore > xrpl.config
-test.nodestore > xrpld.app
 test.nodestore > xrpld.core
-test.nodestore > xrpld.telemetry
 test.nodestore > xrpl.nodestore
 test.nodestore > xrpl.rdb
 test.overlay > test.jtx
@@ -158,7 +148,6 @@ test.overlay > xrpl.protocol
 test.overlay > xrpl.resource
 test.overlay > xrpl.server
 test.overlay > xrpl.shamap
-test.overlay > xrpl.telemetry
 test.protocol > test.jtx
 test.protocol > test.unit_test
 test.protocol > xrpl.basics
@@ -201,26 +190,13 @@ tests.libxrpl > xrpl.nodestore
 tests.libxrpl > xrpl.peerfinder
 tests.libxrpl > xrpl.protocol
 tests.libxrpl > xrpl.protocol_autogen
-tests.libxrpl > xrpl.rdb
 tests.libxrpl > xrpl.resource
 tests.libxrpl > xrpl.server
 tests.libxrpl > xrpl.shamap
-tests.libxrpl > xrpl.telemetry
 tests.libxrpl > xrpl.tx
-tests.xrpld > xrpl.basics
-tests.xrpld > xrpl.config
-tests.xrpld > xrpl.consensus
-tests.xrpld > xrpl.core
-tests.xrpld > xrpld.app
-tests.xrpld > xrpld.overlay
 tests.xrpld > xrpld.rpc
-tests.xrpld > xrpld.telemetry
 tests.xrpld > xrpl.json
-tests.xrpld > xrpl.nodestore
-tests.xrpld > xrpl.peerfinder
 tests.xrpld > xrpl.protocol
-tests.xrpld > xrpl.server
-tests.xrpld > xrpl.telemetry
 xrpl.conditions > xrpl.basics
 xrpl.conditions > xrpl.protocol
 xrpl.config > xrpl.basics
@@ -228,7 +204,6 @@ xrpl.consensus > xrpl.basics
 xrpl.consensus > xrpl.json
 xrpl.consensus > xrpl.ledger
 xrpl.consensus > xrpl.protocol
-xrpl.consensus > xrpl.telemetry
 xrpl.core > xrpl.basics
 xrpl.core > xrpl.json
 xrpl.core > xrpl.protocol
@@ -264,15 +239,10 @@ xrpl.server > xrpl.resource
 xrpl.shamap > xrpl.basics
 xrpl.shamap > xrpl.nodestore
 xrpl.shamap > xrpl.protocol
-xrpl.telemetry > xrpl.basics
-xrpl.telemetry > xrpl.config
-xrpl.telemetry > xrpl.core
-xrpl.telemetry > xrpl.protocol
 xrpl.tx > xrpl.basics
 xrpl.tx > xrpl.core
 xrpl.tx > xrpl.ledger
 xrpl.tx > xrpl.protocol
-xrpl.tx > xrpl.telemetry
 xrpld.app > test.unit_test
 xrpld.app > xrpl.basics
 xrpld.app > xrpl.config
@@ -289,7 +259,6 @@ xrpld.app > xrpl.rdb
 xrpld.app > xrpl.resource
 xrpld.app > xrpl.server
 xrpld.app > xrpl.shamap
-xrpld.app > xrpl.telemetry
 xrpld.app > xrpl.tx
 xrpld.core > xrpl.basics
 xrpld.core > xrpl.config
@@ -310,7 +279,6 @@ xrpld.overlay > xrpl.protocol
 xrpld.overlay > xrpl.resource
 xrpld.overlay > xrpl.server
 xrpld.overlay > xrpl.shamap
-xrpld.overlay > xrpl.telemetry
 xrpld.overlay > xrpl.tx
 xrpld.peerfinder > xrpl.basics
 xrpld.peerfinder > xrpld.app
@@ -325,7 +293,6 @@ xrpld.perflog > xrpl.json
 xrpld.perflog > xrpl.nodestore
 xrpld.perflog > xrpl.protocol
 xrpld.perflog > xrpl.server
-xrpld.perflog > xrpl.telemetry
 xrpld.rpc > xrpl.basics
 xrpld.rpc > xrpl.config
 xrpld.rpc > xrpl.core
@@ -339,22 +306,9 @@ xrpld.rpc > xrpl.rdb
 xrpld.rpc > xrpl.resource
 xrpld.rpc > xrpl.server
 xrpld.rpc > xrpl.shamap
-xrpld.rpc > xrpl.telemetry
 xrpld.rpc > xrpl.tx
 xrpld.shamap > xrpl.basics
 xrpld.shamap > xrpld.core
 xrpld.shamap > xrpl.nodestore
 xrpld.shamap > xrpl.protocol
 xrpld.shamap > xrpl.shamap
-xrpld.telemetry > xrpl.basics
-xrpld.telemetry > xrpl.config
-xrpld.telemetry > xrpl.consensus
-xrpld.telemetry > xrpl.core
-xrpld.telemetry > xrpld.core
-xrpld.telemetry > xrpl.json
-xrpld.telemetry > xrpl.ledger
-xrpld.telemetry > xrpl.nodestore
-xrpld.telemetry > xrpl.protocol
-xrpld.telemetry > xrpl.rdb
-xrpld.telemetry > xrpl.server
-xrpld.telemetry > xrpl.telemetry

.github/scripts/otel-naming/README.md

@@ -1,83 +0,0 @@
-# OTel naming-consistency check
-
-`check_otel_naming.py` enforces the OpenTelemetry span-attribute naming
-convention documented in
-[CONTRIBUTING.md](../../../CONTRIBUTING.md#telemetry-span-attribute-naming)
-across every layer of the telemetry pipeline. The `*SpanNames.h` constants are
-the single source of truth (L1); every other layer must agree with them.
-
-## Running locally
-
-```
-python .github/scripts/otel-naming/check_otel_naming.py
-```
-
-It takes no arguments, can be run from any directory inside the repo, and uses
-only the Python standard library (no `pip install`, matching the levelization
-check). A non-zero exit code means a violation was found; the output lists each
-violation as `RULE | location | token | expected`.
-
-## What it checks
-
-The valid key set is **derived dynamically from the OTel code** — there is no
-hardcoded allowlist:
-
-- **L1 keys** come from the `namespace attr { ... }` blocks of every
-  `*SpanNames.h`, resolving the `makeStr("x")` / `join(seg::a, seg::b)` DSL
-  (cross-file, so `join(seg::rpc, ...)` resolves `seg::rpc` from the base
-  `SpanNames.h`). Each constant is resolved against **its own** header, so two
-  headers that define a same-named constant (e.g. a base `attr::ledgerHash` and
-  a domain `attr::ledgerHash`) each contribute their real wire key — a later
-  header cannot clobber an earlier one's value in a flat table.
-- **Legitimate dotted keys** = ONLY the keys the code actually sets as resource
-  attributes, i.e. the entries inside `Telemetry.cpp`'s `Resource::Create({...})`
-  call: the `semconv::service::*` keys (`service.*`) plus any `attr::<name>`
-  constants passed there (`xrpl.network.*`). A dotted key that is _declared_ in a
-  header but never set as a resource attr is a span attribute in resource
-  clothing — a Rule-A violation, even if it lives in the base `SpanNames.h`.
-- **L1-metrics** — instrument names, label keys and bounded label values come
-  from the `namespace metric` / `namespace label` / `namespace lval` blocks of
-  every `*MetricNames.h`, read as `inline constexpr char NAME[] = "wire";`.
-  These headers deliberately do **not** use the `makeStr`/`StaticStr` DSL the
-  span headers use: the OTel C++ API takes `nostd::string_view`, which
-  constructs from `char const*` but has no constructor from
-  `std::string_view`, so a `StaticStr` will not compile in an instrument-name
-  or label-key position.
-
-### Rules (each fails the build, when its inputs are present)
-
-| Rule | Check                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| A    | No stray dotted span-attribute key (only the derived resource keys may be dotted).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| G    | Attribute keys are `lower_snake_case` (`^[a-z][a-z0-9_]*$` per dot-segment) — no camelCase, UPPERCASE, or spaces.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| F    | No string literals as attribute keys or span-name arguments in `setAttribute`/`addEvent`/`span`/`rootSpan`/`childSpan` (`rootSpan` shares `span`'s `(cat, prefix, name)` signature). Attribute _values_ are exempt (runtime data); `*SpanNames.h` definitions and test files are exempt.                                                                                                                                                                                                                                                                                                                                                                     |
-| B    | Every collector `spanmetrics.dimensions` name exists in the L1 key set.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| C    | Every Tempo span-filter tag exists in the L1 key set.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| D    | Every dashboard label resolves to an L1 span attribute, a native-metric label (L6, emitted by MetricsRegistry), or a Prometheus/Grafana builtin. TraceQL scope prefixes (`span.`/`resource.`/…) are stripped before the L1 lookup.                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| E    | No dotted `xrpl.<domain>.<field>` attribute key in the runbook (only the L1 resource attrs `xrpl.network.*` may be dotted). Span names, filenames, OTel-standard keys, and metric labels are not flagged.                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
-| I    | No string literals as **metric** instrument names or label keys — the mirror of Rule F. Applies to the name passed to an `XRPL_METRIC_*` macro or a `meter->Create*` factory and to the label _keys_ in its label set. Label _values_, descriptions, `*MetricNames.h`, `MetricMacros.h` and test files are exempt. Scoped by metric **family** (first underscore segment): declaring a constant opts that family in, so the metric surface can be converted subsystem by subsystem. Unconverted families warn as Rule L.                                                                                                                                     |
-| J    | Metric instrument names follow the suffix conventions. A name is `lower_snake_case` with no `xrpld_`/`xrpl_` prefix: nothing in the export path adds a prefix, so queries use the bare name. A counter ends `_total`, a histogram ends `_us`/`_ms`/`_seconds`, and a gauge does not end `_total`. The instrument **kind** is read from the emit site, never guessed from words in the name. So a multi-series gauge carrying units in its label values (e.g. `nodestore_state` observing `write_mean_us`) is not a violation. A name created through two different factories is itself reported as a kind conflict, since no suffix can be correct for both. |
-| K    | Every metric named in `docker/telemetry/workload/expected_metrics.json` resolves to a declared constant, so a rename in code cannot leave the workload validator asserting a name nothing emits. PromQL selectors (`m{label="v"}`) and exporter-appended histogram suffixes (`_bucket`/`_count`/`_sum`) are normalized away first; groups fed by another emit path (`statsd_gauges`, `statsd_counters`, `spanmetrics`) are out of scope by design.                                                                                                                                                                                                           |
-
-Rule F runs **unconditionally** (it is a purely syntactic check on the
-call-sites and needs no `*SpanNames.h`), so a code path that calls
-`SpanGuard::span`/`setAttribute` directly without ever defining a header is
-still caught.
-
-### Warnings (printed, never fail the build)
-
-| Rule | Check                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| H    | A namespace-qualified constant (e.g. `foo::bar::myKey`) used at a telemetry call-site is not defined in any `*SpanNames.h`. The constant should live in the proper header; defining it in-place bypasses rules A/G/F. Warns rather than fails — the argument may be a legitimately dynamic value, and the header may live on a later branch. Bare locals and `std::` names are not warned.                                                                                                                                                                                                                                                                                                                   |
-| L    | A literal metric name in a family that has no `*MetricNames.h` constants yet. Rule I's ratchet defers these instead of failing the build on the whole pre-existing metric surface at once; the warning keeps the outstanding conversion work visible rather than silently accepted.                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| M    | A constant defined in a `*SpanNames.h` that no code in `src/**` or `include/**` references — the reverse of every failing rule above, which all start from a consumer and look for its L1 source. Deleting the last `setAttribute(attr::foo, …)` while leaving `attr::foo` in the header otherwise passes every rule and the compiler, and the telemetry silently stops being emitted. Whole files are searched rather than telemetry call sites only, since a constant is also passed to helpers and used as an attribute _value_. Constants only test code references are reported separately. Warns rather than fails: in a stacked chain a constant may legitimately land a commit before its call site. |
-
-## Presence-gated
-
-Every rule runs **only when the source files it needs are present** in the tree
-and is otherwise skipped (printed as `SKIP: <rule> — <reason>`), never failed.
-This keeps the check correct no matter how telemetry work is split across PRs —
-a stacked chain, one large PR, or independent per-stage PRs where (for example)
-the collector config lands before the dashboards. The collector/Tempo/dashboard/
-runbook layers are introduced in later phases; on a branch without them, only
-the L1-intrinsic rules (A, G, F) run.

.github/scripts/telemetry/check_bucket_parity.py

@@ -1,130 +0,0 @@
-#!/usr/bin/env python3
-"""Assert the C++ millisecond ladder agrees with the collector's spanmetrics ladder.
-
-The two are specified to match so a span-derived latency panel and a native
-histogram panel can be read on the same scale. Nothing else couples them, so
-extending one ladder alone -- sub-millisecond edges below 1ms, second-scale
-edges up to 30s -- silently leaves the other short. That failure is quiet:
-Prometheus returns the second-highest edge for a quantile landing in the
-`+Inf` bucket, so every quantile above a too-low ceiling reads back as a flat
-number that looks like a measurement rather than an error. This check is what
-makes the drift loud.
-
-The rule is containment, not equality:
-
-  * every representable collector edge MUST appear in the C++ ladder, so the
-    shared range reads identically;
-  * the C++ ladder MAY carry extra edges ABOVE the collector's highest edge,
-    because jobs outlive spans -- the updatepaths job type was measured
-    averaging ~60s, which no span approaches. Demanding equality would force a
-    ceiling that censors it, recreating the failure this guards against;
-  * collector edges below 1ms are expected to be ABSENT rather than missing:
-    beast::insight::Event rounds every duration up to a whole millisecond
-    before it reaches the histogram, so those edges could never collect a
-    sample.
-
-Exit 0 when the ladders agree, 1 with a diff when they do not.
-"""
-
-import re
-import sys
-from pathlib import Path
-
-HEADER = Path("include/xrpl/telemetry/HistogramBuckets.h")
-COLLECTOR = Path("docker/telemetry/otel-collector-config.yaml")
-
-# beast::insight::Event applies ceil<milliseconds>, so anything below 1ms
-# collapses onto the 1ms edge.
-REPRESENTABLE_FLOOR_MS = 1.0
-
-UNIT_TO_MS = {"ms": 1.0, "s": 1000.0}
-
-
-def collector_edges_ms():
-    """Parse the spanmetrics bucket list, normalising each edge to milliseconds."""
-    text = COLLECTOR.read_text()
-    match = re.search(r"buckets:\s*\[(.*?)\]", text, re.S)
-    if not match:
-        sys.exit(f"{COLLECTOR}: no 'buckets:' list found")
-
-    edges = []
-    for raw in match.group(1).split(","):
-        token = raw.strip()
-        if not token:
-            continue
-        parsed = re.fullmatch(r"([0-9.]+)(ms|s)", token)
-        if not parsed:
-            sys.exit(f"{COLLECTOR}: cannot parse bucket edge {token!r}")
-        edges.append(float(parsed.group(1)) * UNIT_TO_MS[parsed.group(2)])
-    return edges
-
-
-def cpp_edges_ms():
-    """Parse kMillisecondBuckets out of the header that owns every ladder."""
-    text = HEADER.read_text()
-    match = re.search(r"kMillisecondBuckets\{(.*?)\};", text, re.S)
-    if not match:
-        sys.exit(f"{HEADER}: kMillisecondBuckets not found")
-    return [
-        float(token.strip().replace("'", ""))
-        for token in match.group(1).split(",")
-        if token.strip()
-    ]
-
-
-def main():
-    collector = collector_edges_ms()
-    cpp = cpp_edges_ms()
-    required = [edge for edge in collector if edge >= REPRESENTABLE_FLOOR_MS]
-    if not required:
-        sys.exit(f"{COLLECTOR}: no edges at or above {REPRESENTABLE_FLOOR_MS} ms")
-    collector_top = max(required)
-
-    missing = [edge for edge in required if edge not in cpp]
-    # An extra C++ edge inside the collector's range means the two scales
-    # disagree where they overlap. Above the collector's top it is a deliberate
-    # extension.
-    inside_range = [e for e in cpp if e not in required and e < collector_top]
-
-    if not missing and not inside_range:
-        extensions = [e for e in cpp if e > collector_top]
-        summary = f"OK: all {len(required)} representable collector edges present"
-        if extensions:
-            pretty = ", ".join(f"{e:g}" for e in extensions)
-            summary += (
-                f"; {len(extensions)} extension edge(s) above "
-                f"{collector_top:g} ms: [{pretty}]"
-            )
-        print(summary)
-        return 0
-
-    print("Bucket ladder parity violated.", file=sys.stderr)
-    print(
-        f"  collector (>= {REPRESENTABLE_FLOOR_MS:g} ms): "
-        f"{[f'{e:g}' for e in required]}",
-        file=sys.stderr,
-    )
-    print(
-        f"  HistogramBuckets.h            : {[f'{e:g}' for e in cpp]}", file=sys.stderr
-    )
-    for edge in missing:
-        print(f"  MISSING from the C++ ladder: {edge:g} ms", file=sys.stderr)
-    for edge in inside_range:
-        print(
-            f"  C++ edge {edge:g} ms lies inside the collector's range but is not "
-            "a collector edge -- add it to the collector or drop it here",
-            file=sys.stderr,
-        )
-    print(
-        "\nThe two ladders must agree over their shared range. Extra C++ edges are\n"
-        "permitted only ABOVE the collector's highest edge. To re-price the shared\n"
-        f"range, edit the ladder in {HEADER} and the\n"
-        f"spanmetrics 'buckets:' list in {COLLECTOR}\n"
-        "in the same change, so both sides stay in step.",
-        file=sys.stderr,
-    )
-    return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/scripts/telemetry/check_regression_bounds.py

@@ -1,496 +0,0 @@
-#!/usr/bin/env python3
-"""Assert every workload-gate absolute bound is the one its own baseline implies.
-
-The regression gate in ``docker/telemetry/workload`` fails CI when a span or
-job-queue quantile grows. Whether it *can* fail is decided by
-``regression-thresholds.json``, and that file's numbers are derived from
-``baselines/baseline-timings.json`` plus the two histogram ladders. Nothing
-tied the three together, and the gate has now been broken three times by the
-same class of drift:
-
-  1. the microsecond ladder's floor moved 100us -> 1us, voiding every
-     job_queue baseline captured before it;
-  2. the spanmetrics ladder's floor moved 1ms -> 0.01ms, voiding every
-     sub-millisecond span baseline captured before it;
-  3. the absolute bounds stayed calibrated for a 5-25ms band the spans had
-     left, so a 100x regression on ``span.ledger.store.p95`` reported zero
-     regressions and exit 0.
-
-Each time the gate stayed green, which is indistinguishable from a passing
-build. Documentation did not prevent recurrence, so this is a check.
-
-The rule it enforces is the one recorded in ``regression-thresholds.json``
-under ``_absolute_bound_derivation``: for a baseline sitting in the half-open
-bucket ``(lo, hi]`` of its ladder, with ``hi_next`` the next edge above ``hi``,
-
-    max_abs_increase_* == hi_next - baseline
-
-so the gate trips only when the reading clears the bucket *above* the
-baseline's own. Six rules are checked:
-
-  A  the baseline's key set equals the surface ``regression-metrics.json``
-     declares (a stale key left behind reads as covered but never gates);
-  B  every gated key has a per-metric override, not a fallback default;
-  C  each absolute bound equals ``hi_next - baseline``;
-  D  each percentage bound stays below ``100 * bound / baseline``, so the
-     absolute bound remains the operative half of the ``AND`` -- the span
-     ladder's 2s/3s/4s edges are only 1.25x-1.5x apart, where this silently
-     stops being true;
-  E  no baseline carries the ladder-floor signature ``quantile x first_edge``,
-     which means every sample landed in the first bucket and the number is
-     interpolation arithmetic rather than a latency;
-  F  every entry in ``excluded_keys`` names a key the surface would otherwise
-     declare, carries a reason, and has neither a threshold override nor a
-     baseline value left behind.
-
-Rule A subtracts ``excluded_keys`` from BOTH sides of its comparison, so a
-quantile removed from the gated set neither reads as a missing baseline nor as
-an undeclared one -- an exclusion left in the baseline is one failure, rule F's,
-which names the file to edit. Rule F is what keeps that subtraction honest: an
-exclusion is the one edit here that makes the gate cover LESS, so a stale or
-misspelt entry must fail rather than silently widen itself.
-
-A PLACEHOLDER baseline -- ``"placeholder": true`` or an empty ``metrics``
-object -- exits 0, because that is the documented bootstrap state and CI has to
-stay green while a baseline is being recaptured. A missing, unreadable or
-malformed input is a different thing and exits 1: a check that reports success
-without having checked anything is the same green-build-that-is-not failure this
-script exists to prevent, so renaming or deleting one of its inputs must not
-silence it.
-
-A baseline ENTRY that is not an object, or whose value is not a positive finite
-number, is rejected before any rule runs -- see ``check_key`` and
-``_unusable_baseline``. Every rule does arithmetic on that value,
-and a degenerate one made the script crash with a traceback (rule D divides by
-it) or emit advice about the wrong file (a negative value inverts rule D's
-comparison). Reporting malformed input is what this script is for, so it must
-name the key rather than die on it.
-
-Exit 0 when every rule holds, 1 with per-key detail otherwise.
-"""
-
-import json
-import math
-import re
-import sys
-from pathlib import Path
-
-WORKLOAD = Path("docker/telemetry/workload")
-BASELINE = WORKLOAD / "baselines/baseline-timings.json"
-THRESHOLDS = WORKLOAD / "regression-thresholds.json"
-METRICS = WORKLOAD / "regression-metrics.json"
-COLLECTOR = Path("docker/telemetry/otel-collector-config.yaml")
-HEADER = Path("include/xrpl/telemetry/HistogramBuckets.h")
-
-UNIT_TO_MS = {"ms": 1.0, "s": 1000.0}
-# A bound may differ from the derived value only by double round-tripping.
-REL_TOLERANCE = 1e-12
-
-
-def is_number(value):
-    """True for a real JSON number. bool is an int subclass, so exclude it."""
-    return not isinstance(value, bool) and isinstance(value, (int, float))
-
-
-def read_text_or_exit(path):
-    """Read a required text input, or exit 1 naming the input that failed."""
-    try:
-        return path.read_text()
-    except OSError as exc:
-        sys.exit(f"{path}: required input could not be read -- {exc}")
-
-
-def read_json_or_exit(path):
-    """Read and parse a required JSON object, or exit 1 naming what failed.
-
-    All three JSON inputs are objects. A top-level null, list or number parses
-    fine and then dies on the first .get, so check the shape here rather than
-    report it as a traceback pointing into this script.
-    """
-    try:
-        parsed = json.loads(read_text_or_exit(path))
-    except json.JSONDecodeError as exc:
-        sys.exit(f"{path}: required input is not valid JSON -- {exc}")
-    if not isinstance(parsed, dict):
-        sys.exit(
-            f"{path}: required input is valid JSON but its top level is "
-            f"{type(parsed).__name__}, not an object -- nothing can be read from it"
-        )
-    return parsed
-
-
-def span_edges_ms():
-    """Parse the spanmetrics bucket list, normalising each edge to milliseconds."""
-    match = re.search(r"buckets:\s*\[(.*?)\]", read_text_or_exit(COLLECTOR), re.S)
-    if not match:
-        sys.exit(f"{COLLECTOR}: no 'buckets:' list found")
-    edges = []
-    for raw in match.group(1).split(","):
-        token = raw.strip()
-        if not token:
-            continue
-        parsed = re.fullmatch(r"([0-9.]+)(ms|s)", token)
-        if not parsed:
-            sys.exit(f"{COLLECTOR}: cannot parse bucket edge {token!r}")
-        edges.append(float(parsed.group(1)) * UNIT_TO_MS[parsed.group(2)])
-    return edges
-
-
-def microsecond_edges():
-    """Parse kMicrosecondBuckets out of the header that owns every ladder."""
-    match = re.search(r"kMicrosecondBuckets\{(.*?)\};", read_text_or_exit(HEADER), re.S)
-    if not match:
-        sys.exit(f"{HEADER}: kMicrosecondBuckets not found")
-    return [
-        float(token.strip().replace("'", ""))
-        for token in match.group(1).split(",")
-        if token.strip()
-    ]
-
-
-def declared_keys(metrics_cfg):
-    """Rebuild the flat key set regression-metrics.json declares.
-
-    Deliberately reimplemented rather than imported from ``prom_queries.py``,
-    which pulls in aiohttp; CI telemetry checks stay dependency-free. The key
-    format is fixed by that file's own ``_key_format`` field.
-
-    ``excluded_keys`` is NOT subtracted here: rule F needs the full product to
-    tell a real exclusion from a misspelt one. Callers that want the gated
-    surface subtract it themselves.
-    """
-    keys = set()
-    spans = metrics_cfg.get("spans", {})
-    for name in spans.get("names", []):
-        for quantile in spans.get("_quantiles", []):
-            keys.add(f"span.{name}.p{_quantile_label(quantile)}")
-    jobs = metrics_cfg.get("job_queue", {})
-    for name in jobs.get("names", []):
-        for phase in jobs.get("_phases", []):
-            for quantile in jobs.get("_quantiles", []):
-                keys.add(f"job.{name}.{phase}.p{_quantile_label(quantile)}")
-    return keys
-
-
-def _quantile_label(quantile):
-    """0.95 -> '95', 0.5 -> '50', matching capture_timings.py's key format."""
-    return f"{quantile * 100:g}".replace(".", "")
-
-
-def brackets(value, edges):
-    """Return ``(lo, hi, hi_next)`` for the bucket ``(lo, hi]`` holding value."""
-    padded = [0.0] + list(edges)
-    for i in range(1, len(padded)):
-        if value <= padded[i]:
-            hi_next = padded[i + 1] if i + 1 < len(padded) else None
-            return padded[i - 1], padded[i], hi_next
-    return None, None, None
-
-
-def resolve_override(key, thresholds):
-    """Return the override rule for a key, or None if it falls back to defaults."""
-    group, quantile = key.rsplit(".", 1)
-    return thresholds.get("overrides", {}).get(group, {}).get(quantile)
-
-
-def check_exclusions(metrics_cfg, thresholds, baseline_metrics, declared):
-    """Apply rule F to every entry in ``excluded_keys``.
-
-    An exclusion is the only edit to this config that makes the gate cover
-    LESS, so each entry has to prove it is deliberate and complete:
-
-      * it names a key the names x quantiles product would otherwise declare,
-        so a typo or a stale entry surviving a surface change is caught rather
-        than silently subtracting nothing;
-      * it carries a non-empty reason, because "why is this not gated" is the
-        question a future maintainer will ask and prose is the only answer;
-      * no threshold override and no baseline value are left behind, since
-        either would read as gated to anyone grepping for the key.
-
-    Args:
-        metrics_cfg:      Parsed regression-metrics.json.
-        thresholds:       Parsed regression-thresholds.json.
-        baseline_metrics: The baseline's ``metrics`` map.
-        declared:         Output of declared_keys(), before exclusions come off.
-
-    Returns:
-        A list of failure strings, empty when every entry is well formed.
-    """
-    failures = []
-    for key, reason in sorted(metrics_cfg.get("excluded_keys", {}).items()):
-        if key not in declared:
-            failures.append(
-                f"{key}: listed in excluded_keys but not produced by the "
-                f"names x quantiles product, so it subtracts nothing -- fix the "
-                f"spelling or drop the entry (rule F)"
-            )
-            continue
-        if not isinstance(reason, str) or not reason.strip():
-            failures.append(
-                f"{key}: excluded with no reason. Record why it is not gated, "
-                f"with the measurement behind it (rule F)"
-            )
-        if resolve_override(key, thresholds) is not None:
-            failures.append(
-                f"{key}: excluded but still has a threshold override, which "
-                f"reads as gated -- remove it from {THRESHOLDS} (rule F)"
-            )
-        if key in baseline_metrics:
-            failures.append(
-                f"{key}: excluded but still has a baseline value, so rule A "
-                f"would pass while nothing gates it -- remove it from "
-                f"{BASELINE} (rule F)"
-            )
-    return failures
-
-
-def _unusable_baseline(key, value, unit):
-    """Reject a baseline value no rule below could evaluate, or None if it is fine.
-
-    Every rule downstream does arithmetic on this number, and two of them break
-    on a degenerate one rather than reporting it:
-
-      * rule D computes ``100 * bound / value``, which raises
-        ZeroDivisionError on ``0.0``. The script then dies with a traceback
-        instead of naming the key -- a validator that crashes where it should
-        report is the same green-build-that-is-not failure in reverse;
-      * a NEGATIVE value makes that same ratio negative, so ``pct >= ratio`` is
-        true for any configured percentage and rule D fires with a message
-        telling the maintainer to lower ``max_pct_increase``. The advice is
-        wrong: the fault is the baseline, not the threshold;
-      * a non-numeric value raises TypeError inside rule E's subtraction.
-
-    Rule E does not cover the zero case, which is easy to assume it does: its
-    test ``abs(value - quantile * first_edge) <= 1e-9 * first_edge`` reduces to
-    ``quantile <= 1e-9`` when value is ``0.0``, and that is false for every
-    quantile this harness captures (0.5, 0.95, 0.99).
-
-    None of these arise from the normal pipeline -- ``histogram_quantile`` over
-    a first-bucket-only histogram returns ``quantile x first_edge``, never zero,
-    and rule E is the guard for exactly that. They arise from a hand-edited or
-    truncated baseline, which is precisely the input this script exists to
-    reject.
-
-    Args:
-        key:   Flat metric key, for the message.
-        value: The baseline value as read from the file.
-        unit:  The entry's unit, for the message.
-
-    Returns:
-        A failure string, or None when the value is usable.
-    """
-    # bool is a subclass of int; True would otherwise pass as the number 1.
-    if not is_number(value):
-        return (
-            f"{key}: baseline value {value!r} is not a number, so no bound can be "
-            f"derived from it. Recapture the baseline from a CI run rather than "
-            f"editing it by hand -- see baselines/README.md"
-        )
-    if not math.isfinite(value) or value <= 0:
-        return (
-            f"{key}: baseline is {value!r}{unit}, but a captured latency quantile "
-            f"is strictly positive and finite. A zero baseline leaves the "
-            f"percentage bound undefined, a negative one inverts it, and neither "
-            f"can bracket to a bucket -- so no rule below can be evaluated. "
-            f"Recapture the baseline from a CI run rather than editing it by hand "
-            f"-- see baselines/README.md"
-        )
-    return None
-
-
-def check_key(key, entry, thresholds, ladders):
-    """Apply rules B, C, D and E to one gated key. Returns a list of failures.
-
-    The rules assume the baseline entry holds a strictly positive, finite
-    number, which is what ``histogram_quantile`` yields. Anything else is
-    malformed input, and reporting malformed input is this script's whole job,
-    so it is rejected up front rather than arithmetic being attempted on it --
-    see ``_unusable_baseline``.
-
-    The entry's SHAPE is checked first, for the same reason. A hand edit that
-    writes the bare number instead of the ``{"value": .., "unit": ..}`` object
-    leaves no ``.get`` to call, and the script died with an AttributeError
-    traceback naming a line in itself rather than the key at fault.
-    """
-    if not isinstance(entry, dict):
-        return [
-            f"{key}: baseline entry {entry!r} is not an object carrying value and "
-            f"unit, so no bound can be derived from it. Recapture the baseline "
-            f"from a CI run rather than editing it by hand -- see "
-            f"baselines/README.md"
-        ]
-
-    value, unit = entry.get("value"), entry.get("unit", "")
-    edges = ladders.get(unit)
-    if value is None or edges is None:
-        return [f"{key}: baseline has no value, or unknown unit {unit!r}"]
-
-    unusable = _unusable_baseline(key, value, unit)
-    if unusable:
-        return [unusable]
-
-    failures = []
-    first_edge = edges[0]
-    quantile = int(key.rsplit(".p", 1)[1]) / 100.0
-    if abs(value - quantile * first_edge) <= 1e-9 * first_edge:
-        failures.append(
-            f"{key}: baseline {value!r} equals quantile {quantile:g} x the ladder "
-            f"floor {first_edge:g}{unit}, so every sample landed in the first "
-            f"bucket and this is bucket arithmetic, not a latency. No absolute "
-            f"bound can gate it -- add a finer ladder edge or drop the metric "
-            f"from {METRICS} (rule E)"
-        )
-        return failures
-
-    _, _, hi_next = brackets(value, edges)
-    if hi_next is None:
-        return [
-            f"{key}: baseline {value!r}{unit} sits in or above the ladder's top "
-            f"bucket, so there is no hi_next to derive a bound from -- extend the "
-            f"ladder (rule C)"
-        ]
-
-    rule = resolve_override(key, thresholds)
-    if rule is None:
-        failures.append(
-            f"{key}: no per-metric override, so it falls back to the defaults and "
-            f"gates on the percentage bound alone. Add an override in "
-            f"{THRESHOLDS} with "
-            f"max_abs_increase_{unit} = {hi_next - value!r} (rule B)"
-        )
-        return failures
-
-    if not isinstance(rule, dict):
-        return [
-            f"{key}: threshold override {rule!r} is not an object carrying "
-            f"max_abs_increase_{unit} and max_pct_increase -- fix it in {THRESHOLDS}"
-        ]
-
-    bound = rule.get("max_abs_increase_ms", rule.get("max_abs_increase_us"))
-    if bound is not None and not is_number(bound):
-        return [
-            f"{key}: max_abs_increase_{unit} is {bound!r}, not a number, so it "
-            f"cannot be compared with the derived bound -- fix it in {THRESHOLDS}"
-        ]
-    expected = hi_next - value
-    if bound is None or abs(bound - expected) > REL_TOLERANCE * expected:
-        failures.append(
-            f"{key}: absolute bound is {bound!r}, expected {expected!r} "
-            f"(hi_next {hi_next:g} - baseline {value!r}) (rule C)"
-        )
-
-    pct = rule.get("max_pct_increase")
-    if pct is None:
-        failures.append(f"{key}: no max_pct_increase, so the metric never gates")
-    elif not is_number(pct):
-        failures.append(
-            f"{key}: max_pct_increase is {pct!r}, not a number -- fix it in "
-            f"{THRESHOLDS}"
-        )
-    elif bound is not None and pct >= 100.0 * bound / value:
-        failures.append(
-            f"{key}: max_pct_increase {pct:g}% is at or above the absolute bound's "
-            f"{100.0 * bound / value:.1f}% of baseline, so the percentage bound "
-            f"becomes the operative one and the bucket guarantee is lost. Lower it "
-            f"or document the metric as percentage-gated (rule D)"
-        )
-    return failures
-
-
-def main():
-    missing = [
-        p for p in (BASELINE, THRESHOLDS, METRICS, COLLECTOR, HEADER) if not p.exists()
-    ]
-    if missing:
-        print("Cannot check workload regression bounds.", file=sys.stderr)
-        for path in missing:
-            print(f"  {path}: required input is absent", file=sys.stderr)
-        print(
-            "\nA missing input is not a reason to pass. Deleting or renaming one of\n"
-            "these would otherwise leave the gate reporting success without having\n"
-            "checked a single bound -- the failure this script exists to prevent. If\n"
-            "the workload harness has genuinely moved, update the paths here.",
-            file=sys.stderr,
-        )
-        return 1
-
-    baseline = read_json_or_exit(BASELINE)
-    thresholds = read_json_or_exit(THRESHOLDS)
-    metrics_cfg = read_json_or_exit(METRICS)
-
-    if baseline.get("placeholder") is True or not baseline.get("metrics"):
-        print("OK: baseline is a placeholder, bounds cannot be derived yet")
-        return 0
-
-    ladders = {"ms": span_edges_ms(), "us": microsecond_edges()}
-    gated = baseline["metrics"]
-    # A list or a string is truthy, so it survives the placeholder test above
-    # and then either crashes or reports its characters as gated keys.
-    if not isinstance(gated, dict):
-        sys.exit(
-            f"{BASELINE}: 'metrics' is {type(gated).__name__}, not an object of "
-            f"key -> {{value, unit}} -- recapture the baseline from a CI run"
-        )
-    failures = []
-
-    declared = declared_keys(metrics_cfg)
-    failures.extend(check_exclusions(metrics_cfg, thresholds, gated, declared))
-    # Rule A compares against the GATED surface, so a deliberately excluded
-    # quantile is not reported as a baseline that was never captured.
-    #
-    # The same keys come off rule A's over-coverage side too. An excluded key
-    # left in the baseline is rule F's finding, reported with the file to edit;
-    # rule A would add a second failure for the same single mistake, saying the
-    # key is not declared -- which is not even true, it is declared and then
-    # excluded. Only exclusions the surface really declares are subtracted, so a
-    # misspelt exclusion naming a stale baseline key still reaches rule A.
-    excluded_declared = set(metrics_cfg.get("excluded_keys", {})) & declared
-    declared -= excluded_declared
-    for key in sorted(set(gated) - declared - excluded_declared):
-        failures.append(
-            f"{key}: in the baseline but not declared by {METRICS}, so it is "
-            f"reported every run and can never gate -- remove it (rule A)"
-        )
-    for key in sorted(declared - set(gated)):
-        failures.append(
-            f"{key}: declared by {METRICS} but absent from the baseline, so it "
-            f"never gates -- capture a baseline for it (rule A)"
-        )
-
-    for key in sorted(gated):
-        if key in declared:
-            failures.extend(check_key(key, gated[key], thresholds, ladders))
-
-    if not failures:
-        excluded = metrics_cfg.get("excluded_keys", {})
-        print(
-            f"OK: {len(gated)} gated key(s); every absolute bound equals "
-            f"hi_next - baseline, every key has an override, and the absolute "
-            f"bound is the operative half of the AND for all of them"
-        )
-        # Printed, not silent: an exclusion narrows the gate, so the count
-        # belongs in the CI log where a reviewer sees it without opening a file.
-        if excluded:
-            print(
-                f"    {len(excluded)} declared key(s) deliberately not gated: "
-                f"{', '.join(sorted(excluded))} (see excluded_keys in {METRICS})"
-            )
-        return 0
-
-    print(
-        "Workload regression bounds are not derived from the baseline.", file=sys.stderr
-    )
-    for failure in failures:
-        print(f"  {failure}", file=sys.stderr)
-    print(
-        f"\nThe rule is recorded in {THRESHOLDS} under _absolute_bound_derivation:\n"
-        "a bound is hi_next - baseline, where hi_next is the edge above the top of\n"
-        "the bucket holding the baseline. Refreshing a baseline therefore obliges\n"
-        "you to re-derive its bound; see baselines/README.md.",
-        file=sys.stderr,
-    )
-    return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/scripts/telemetry/test_check_regression_bounds.py

@@ -1,462 +0,0 @@
-#!/usr/bin/env python3
-"""Tests for check_regression_bounds.py.
-
-The checker reads five files by path relative to the working directory, so each
-test assembles a scratch tree holding copies of the real inputs, mutates one
-thing, and runs the checker as a subprocess there. Testing the real entry point
-is deliberate: the contract under test is the exit code CI reads, and an
-in-process call would not exercise it.
-
-Two groups:
-
-* the input-handling contract -- a placeholder baseline must PASS because that
-  is the documented bootstrap state, while a missing, unreadable or malformed
-  input must FAIL. A checker that returns success without having checked
-  anything is the failure this whole gate exists to prevent;
-* one case per rule (A to F), so a rule that stops flagging is caught.
-
-stdlib unittest only; the repo installs no third-party runner for CI.
-"""
-
-import json
-import os
-import shutil
-import stat
-import subprocess
-import sys
-import tempfile
-import unittest
-from pathlib import Path
-
-SCRIPT_DIR = Path(__file__).resolve().parent
-CHECKER = SCRIPT_DIR / "check_regression_bounds.py"
-REPO = SCRIPT_DIR.parents[2]
-
-WORKLOAD = "docker/telemetry/workload"
-BASELINE = f"{WORKLOAD}/baselines/baseline-timings.json"
-THRESHOLDS = f"{WORKLOAD}/regression-thresholds.json"
-METRICS = f"{WORKLOAD}/regression-metrics.json"
-COLLECTOR = "docker/telemetry/otel-collector-config.yaml"
-HEADER = "include/xrpl/telemetry/HistogramBuckets.h"
-INPUTS = (BASELINE, THRESHOLDS, METRICS, COLLECTOR, HEADER)
-
-
-class CheckerCase(unittest.TestCase):
-    """Base class giving each test an isolated copy of the checker's inputs."""
-
-    def setUp(self):
-        self.tree = Path(tempfile.mkdtemp())
-        # Bound to THIS tree, not read off self.tree when the cleanup finally
-        # runs. A test that calls setUp again for a fresh tree (see the
-        # degenerate-baseline subTests) rebinds self.tree, and a late read would
-        # make every registered cleanup remove the LAST tree, leaving each
-        # earlier one behind in /tmp.
-        self.addCleanup(self._cleanup, self.tree)
-        for rel in INPUTS:
-            dest = self.tree / rel
-            dest.parent.mkdir(parents=True, exist_ok=True)
-            shutil.copy(REPO / rel, dest)
-        script = self.tree / ".github/scripts/telemetry/check_regression_bounds.py"
-        script.parent.mkdir(parents=True, exist_ok=True)
-        shutil.copy(CHECKER, script)
-
-    def _cleanup(self, tree):
-        """Remove one scratch tree, restoring permissions rmtree needs first."""
-        for path in tree.rglob("*"):
-            if path.is_file():
-                path.chmod(stat.S_IRUSR | stat.S_IWUSR)
-        shutil.rmtree(tree, ignore_errors=True)
-
-    def run_checker(self):
-        """Run the checker in the scratch tree, returning (code, stdout+stderr)."""
-        proc = subprocess.run(
-            [sys.executable, ".github/scripts/telemetry/check_regression_bounds.py"],
-            cwd=self.tree,
-            capture_output=True,
-            text=True,
-        )
-        return proc.returncode, proc.stdout + proc.stderr
-
-    def edit_json(self, rel, mutate):
-        """Load a scratch input, hand it to mutate(), write it back."""
-        path = self.tree / rel
-        data = json.loads(path.read_text())
-        mutate(data)
-        path.write_text(json.dumps(data, indent=2))
-
-    def read_json(self, rel):
-        """Read a scratch input without modifying it."""
-        return json.loads((self.tree / rel).read_text())
-
-    def gated(self, key):
-        """Return ``(baseline_value, configured_bound)`` for one gated key.
-
-        Read from the scratch copies of the real inputs rather than written as
-        literals, because a literal here is a copy of one particular baseline:
-        a hard-coded figure breaks the moment the baseline is refreshed, which is
-        the very drift check_regression_bounds.py exists to catch. Deriving the
-        figure keeps the assertion pinned to the rule instead of to a snapshot.
-        """
-        group, quantile = key.rsplit(".", 1)
-        rule = self.read_json(THRESHOLDS)["overrides"][group][quantile]
-        bound = rule.get("max_abs_increase_ms", rule.get("max_abs_increase_us"))
-        return self.read_json(BASELINE)["metrics"][key]["value"], bound
-
-
-class TestInputHandling(CheckerCase):
-    """A placeholder passes; a missing or broken input must not."""
-
-    def test_unmodified_tree_passes(self):
-        code, out = self.run_checker()
-        self.assertEqual(code, 0, out)
-        self.assertIn("gated key(s)", out)
-
-    def test_placeholder_flag_passes(self):
-        self.edit_json(BASELINE, lambda d: d.update(placeholder=True))
-        code, out = self.run_checker()
-        self.assertEqual(code, 0, out)
-        self.assertIn("placeholder", out)
-
-    def test_empty_metrics_baseline_passes(self):
-        self.edit_json(BASELINE, lambda d: d.update(metrics={}))
-        code, out = self.run_checker()
-        self.assertEqual(code, 0, out)
-        self.assertIn("placeholder", out)
-
-    def test_missing_baseline_fails_naming_the_input(self):
-        (self.tree / BASELINE).unlink()
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("baseline-timings.json", out)
-
-    def test_missing_collector_config_fails_naming_the_input(self):
-        (self.tree / COLLECTOR).unlink()
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("otel-collector-config.yaml", out)
-
-    @unittest.skipIf(os.geteuid() == 0, "root ignores the read permission bit")
-    def test_unreadable_baseline_fails(self):
-        (self.tree / BASELINE).chmod(0)
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("baseline-timings.json", out)
-        self.assertIn("could not be read", out)
-        self.assertNotIn("Traceback", out)
-
-    def test_malformed_baseline_json_fails(self):
-        (self.tree / BASELINE).write_text("{ not json")
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("valid JSON", out)
-
-    def test_malformed_thresholds_json_fails(self):
-        (self.tree / THRESHOLDS).write_text("]")
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("valid JSON", out)
-
-    def test_non_object_top_level_fails_naming_the_input(self):
-        """Valid JSON of the wrong shape must be named, not raise a traceback.
-
-        A top-level null, list or number parses, so it reaches the first .get
-        and dies pointing at a line in the checker rather than at the file the
-        operator has to fix.
-        """
-        for rel, text in (
-            (BASELINE, "null"),
-            (THRESHOLDS, "[]"),
-            (METRICS, "5"),
-        ):
-            with self.subTest(input=rel):
-                self.setUp()
-                (self.tree / rel).write_text(text)
-                code, out = self.run_checker()
-                self.assertEqual(code, 1, out)
-                self.assertIn(rel, out)
-                self.assertIn("not an object", out)
-                self.assertNotIn("Traceback", out)
-
-    def test_non_object_metrics_map_fails_naming_the_input(self):
-        """A string 'metrics' is truthy, so it slips past the placeholder test.
-
-        Left unchecked it reports the string's own characters as gated keys,
-        which is worse than a crash: the advice is wrong rather than absent.
-        An empty map still has to pass, because that is the bootstrap state.
-        """
-        self.edit_json(BASELINE, lambda d: d.update(metrics="span.tx.process.p99"))
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("'metrics' is str", out)
-        self.assertNotIn("Traceback", out)
-
-        self.setUp()
-        self.edit_json(BASELINE, lambda d: d.update(metrics={}))
-        code, out = self.run_checker()
-        self.assertEqual(code, 0, out)
-
-
-class TestRules(CheckerCase):
-    """One case per rule, so a rule that stops flagging is caught."""
-
-    def test_rule_a_flags_baseline_key_not_declared(self):
-        self.edit_json(
-            BASELINE,
-            lambda d: d["metrics"].update(
-                {"span.rpc.process.p99": {"unit": "ms", "value": 9.0}}
-            ),
-        )
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("(rule A)", out)
-
-    def test_rule_a_flags_declared_key_without_baseline(self):
-        self.edit_json(METRICS, lambda d: d["spans"]["names"].append("consensus.round"))
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("(rule A)", out)
-
-    def test_rule_b_flags_missing_override(self):
-        self.edit_json(THRESHOLDS, lambda d: d["overrides"].pop("span.ledger.build"))
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("(rule B)", out)
-
-    def test_rule_b_names_the_unit_suffixed_key(self):
-        """The key it tells the operator to add must be the key the code reads.
-
-        The bound is stored as max_abs_increase_ms or _us. A message naming a
-        bare max_abs_increase sends the operator to add a key nothing reads, so
-        the gate keeps failing with no explanation. Both suffixes are covered,
-        because a test on the ms side alone passes on a hard-coded "_ms".
-        """
-        for group, suffix in (
-            ("span.ledger.build", "max_abs_increase_ms"),
-            ("job.transaction.queued", "max_abs_increase_us"),
-        ):
-            with self.subTest(group=group):
-                self.setUp()
-                self.edit_json(THRESHOLDS, lambda d: d["overrides"].pop(group))
-                code, out = self.run_checker()
-                self.assertEqual(code, 1, out)
-                self.assertIn("(rule B)", out)
-                self.assertIn(suffix, out)
-                self.assertNotIn("max_abs_increase =", out)
-
-    def test_non_numeric_threshold_is_reported_not_crashed(self):
-        """A hand-edited bound that is a string must be named, not raise."""
-        self.edit_json(
-            THRESHOLDS,
-            lambda d: d["overrides"]["span.ledger.build"]["p95"].update(
-                max_abs_increase_ms="5.5"
-            ),
-        )
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("not a number", out)
-        self.assertNotIn("Traceback", out)
-
-    def test_rule_c_flags_rounded_bound(self):
-        """A bound rounded for readability is still not the derived bound."""
-        _, exact = self.gated("span.tx.process.p99")
-        rounded = round(exact, 4)
-        self.assertNotEqual(rounded, exact, "pick a key whose bound rounds visibly")
-        self.edit_json(
-            THRESHOLDS,
-            lambda d: d["overrides"]["span.tx.process"]["p99"].update(
-                max_abs_increase_ms=rounded
-            ),
-        )
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("(rule C)", out)
-
-    def test_rule_c_accepts_bound_within_relative_tolerance(self):
-        """The tolerance is 1e-12 relative, not exact equality."""
-        _, exact = self.gated("span.tx.process.p99")
-        self.edit_json(
-            THRESHOLDS,
-            lambda d: d["overrides"]["span.tx.process"]["p99"].update(
-                max_abs_increase_ms=exact * (1 + 5e-13)
-            ),
-        )
-        code, out = self.run_checker()
-        self.assertEqual(code, 0, out)
-
-    def test_rule_d_flags_percentage_bound_becoming_operative(self):
-        """Rule D trips at exactly 100 x bound / baseline, its ``>=`` boundary."""
-        baseline, bound = self.gated("span.tx.apply.p99")
-        boundary = 100.0 * bound / baseline
-        self.edit_json(
-            THRESHOLDS,
-            lambda d: d["overrides"]["span.tx.apply"]["p99"].update(
-                max_pct_increase=boundary
-            ),
-        )
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("(rule D)", out)
-
-    def test_rule_d_accepts_percentage_bound_just_below_the_boundary(self):
-        """The other side of rule D's boundary, so the test cannot pass vacuously."""
-        baseline, bound = self.gated("span.tx.apply.p99")
-        boundary = 100.0 * bound / baseline
-        self.edit_json(
-            THRESHOLDS,
-            lambda d: d["overrides"]["span.tx.apply"]["p99"].update(
-                max_pct_increase=boundary * (1 - 1e-9)
-            ),
-        )
-        code, out = self.run_checker()
-        self.assertEqual(code, 0, out)
-
-    def test_rule_a_ignores_an_excluded_key(self):
-        """An excluded key must not read as a baseline that was never captured.
-
-        The unmodified tree already exercises this — span.ledger.validate p95
-        and p99 are declared by the names x quantiles product, excluded, and
-        absent from the baseline — so this asserts the subtraction is what makes
-        it pass, by naming the keys in the reported exclusion line.
-        """
-        code, out = self.run_checker()
-        self.assertEqual(code, 0, out)
-        self.assertIn("span.ledger.validate.p95", out)
-        self.assertIn("deliberately not gated", out)
-
-    def test_rule_f_flags_exclusion_that_subtracts_nothing(self):
-        """A misspelt or stale exclusion silently narrows nothing — catch it."""
-        self.edit_json(
-            METRICS,
-            lambda d: d["excluded_keys"].update({"span.ledger.validate.p97": "typo"}),
-        )
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("(rule F)", out)
-        self.assertIn("subtracts nothing", out)
-
-    def test_rule_f_flags_exclusion_without_a_reason(self):
-        self.edit_json(
-            METRICS,
-            lambda d: d["excluded_keys"].update({"span.ledger.validate.p95": "   "}),
-        )
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("(rule F)", out)
-        self.assertIn("no reason", out)
-
-    def test_rule_f_flags_override_left_behind(self):
-        """An excluded key still carrying a bound reads as gated."""
-        self.edit_json(
-            THRESHOLDS,
-            lambda d: d["overrides"]["span.ledger.validate"].update(
-                {"p95": {"max_pct_increase": 50.0, "max_abs_increase_ms": 0.25}}
-            ),
-        )
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("(rule F)", out)
-        self.assertIn("still has a threshold override", out)
-
-    def test_rule_f_flags_baseline_value_left_behind(self):
-        """Excluded but still in the baseline: rule A passes, nothing gates."""
-        self.edit_json(
-            BASELINE,
-            lambda d: d["metrics"].update(
-                {"span.ledger.validate.p95": {"unit": "ms", "value": 0.24}}
-            ),
-        )
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("(rule F)", out)
-        self.assertIn("still has a baseline value", out)
-
-    # A key that is GATED, so seeding it exercises the numeric guard rather than
-    # rule F. It must not be one of the excluded keys: putting a value there
-    # trips "excluded but still has a baseline value" first and the test would
-    # pass for the wrong reason.
-    DEGENERATE_KEY = "span.tx.process.p50"
-
-    def _seed_degenerate_baseline(self, value):
-        """Replace one gated key's baseline value with an unusable one.
-
-        The threshold is deliberately left alone. The numeric guard runs before
-        every rule, so no bound arrangement is needed to reach it -- and on the
-        pre-fix checker this same seeding still reached the crash, because rule C
-        appends its failure and falls through to rule D's division.
-        """
-        key = self.DEGENERATE_KEY
-        self.assertNotIn(
-            key,
-            self.read_json(METRICS).get("excluded_keys", {}),
-            "DEGENERATE_KEY must be gated, not excluded",
-        )
-        self.edit_json(
-            BASELINE,
-            lambda d: d["metrics"].update({key: {"unit": "ms", "value": value}}),
-        )
-
-    def test_degenerate_baseline_is_reported_not_crashed(self):
-        """A zero, negative or non-numeric baseline must NAME the key, not raise.
-
-        Rule D computes ``100 * bound / value``, so a 0.0 baseline used to exit
-        via ZeroDivisionError and a traceback, and a negative one used to emit a
-        rule-D failure blaming ``max_pct_increase`` when the fault was the
-        baseline. Rule E does not cover the zero case: its test reduces to
-        ``quantile <= 1e-9`` there, false for every quantile captured.
-        """
-        # Asserting the guard's OWN wording, not merely "exit 1 with no
-        # traceback": the negative and bool cases already exited 1 without a
-        # traceback before the fix, by emitting a rule-D failure that blamed the
-        # wrong file. A looser assertion passes on that and proves nothing.
-        cases = (
-            (0.0, "strictly positive"),
-            (-1.0, "strictly positive"),
-            (float("nan"), "strictly positive"),
-            (float("inf"), "strictly positive"),
-            ("0.006", "is not a number"),
-            (True, "is not a number"),
-        )
-        for value, expected in cases:
-            with self.subTest(value=value):
-                self.setUp()  # a clean scratch tree per value
-                self._seed_degenerate_baseline(value)
-                code, out = self.run_checker()
-                self.assertEqual(code, 1, out)
-                self.assertNotIn("Traceback", out)
-                self.assertIn(self.DEGENERATE_KEY, out)
-                self.assertIn(expected, out)
-                # The old rule-D message told the maintainer to lower the
-                # percentage bound; the baseline is what is wrong.
-                self.assertNotIn("(rule D)", out)
-
-    def test_rule_e_flags_ladder_floor_signature(self):
-        """ledger.store's quantiles were the ladder floor times the quantile."""
-        store = {"p50": 0.005, "p95": 0.0095, "p99": 0.0099}
-        self.edit_json(METRICS, lambda d: d["spans"]["names"].append("ledger.store"))
-        self.edit_json(
-            BASELINE,
-            lambda d: d["metrics"].update(
-                {
-                    f"span.ledger.store.{q}": {"unit": "ms", "value": v}
-                    for q, v in store.items()
-                }
-            ),
-        )
-        self.edit_json(
-            THRESHOLDS,
-            lambda d: d["overrides"].update(
-                {
-                    "span.ledger.store": {
-                        q: {"max_pct_increase": 50.0, "max_abs_increase_ms": 0.05 - v}
-                        for q, v in store.items()
-                    }
-                }
-            ),
-        )
-        code, out = self.run_checker()
-        self.assertEqual(code, 1, out)
-        self.assertIn("(rule E)", out)
-
-
-if __name__ == "__main__":
-    unittest.main()

.github/workflows/on-pr.yml

@@ -70,24 +70,11 @@ jobs:
           files: |
             # These paths are unique to `on-pr.yml`.
             .github/scripts/levelization/**
-            .github/scripts/otel-naming/**
             .github/scripts/rename/**
-            .github/scripts/telemetry/**
             .github/workflows/reusable-check-levelization.yml
-            .github/workflows/reusable-check-otel-naming.yml
             .github/workflows/reusable-check-rename.yml
             .github/workflows/on-pr.yml
 
-            # The non-code layers the OTel naming check validates: the docs that
-            # publish attribute tables (Rule E) and the telemetry stack config —
-            # collector, Tempo, dashboards (Rules B, C, D). Without these paths a
-            # docs-only or dashboard-only pull request sets `go=false`, so the
-            # very layers those rules exist to police would never be checked.
-            # As with `README.md` below, matching one of these also switches on
-            # the rest of the workflow; there is a single `go` gate.
-            docs/**
-            docker/telemetry/**
-
             # Keep the paths below in sync with those in `on-trigger.yml`.
             .github/actions/build-deps/**
             .github/actions/release-info/**
@@ -127,61 +114,6 @@ jobs:
             package/**
             README.md
 
-      - name: Determine changed telemetry files
-        # A separate list from the one above, because the telemetry validation
-        # builds xrpld on the self-hosted runner and then runs a multi-node
-        # cluster for up to 30 minutes. The `go` gate above is true for almost
-        # any pull request, so gating this one on `go` alone would run that cost
-        # on every pull request in the repository.
-        #
-        # Keep these globs pointing at paths that exist -- two earlier entries
-        # matched zero tracked files, so C++ telemetry changes never triggered.
-        id: telemetry-changes
-        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
-        with:
-          files: |
-            # This workflow, the one it calls, and the harness that runs.
-            .github/workflows/on-pr.yml
-            .github/workflows/reusable-telemetry-validation.yml
-            docker/telemetry/**
-
-            # The telemetry modules themselves.
-            include/xrpl/telemetry/**
-            src/libxrpl/telemetry/**
-            src/xrpld/telemetry/**
-
-            # beast::insight, whose gauges and counters the validation asserts
-            # on. Both halves: the interface headers decide what the OTel
-            # collector can publish, so editing one moves the metric surface as
-            # surely as editing the implementation does.
-            include/xrpl/beast/insight/**
-            src/libxrpl/beast/insight/**
-
-            # The span-name and metric-name constants. These are the wire
-            # contract the harness asserts against by literal string, so a
-            # rename here compiles cleanly and silently empties the assertions.
-            # Matched by filename rather than directory because the convention
-            # colocates each header with the class it serves: they live under
-            # consensus/, overlay/, app/ledger/, app/main/, app/misc/, rpc/ and
-            # tx/, not under a telemetry/ directory.
-            **/*SpanNames.h
-            **/*MetricNames.h
-
-            # The GTests that pin those same constants, in both test binaries.
-            src/tests/libxrpl/telemetry/**
-            src/tests/xrpld/telemetry/**
-
-            # The checkers that gate this surface. otel-naming derives the legal
-            # attribute and metric names from the headers above; the telemetry
-            # scripts enforce the histogram-bucket parity and regression-bound
-            # rules the harness depends on.
-            .github/scripts/otel-naming/**
-            .github/scripts/telemetry/**
-
-            # The recipe, because it carries the telemetry option this decides
-            # the validation on.
-            conanfile.py
-
       - name: Check whether to run
         # This step determines whether the rest of the workflow should
         # run. The rest of the workflow will run if this job runs AND at
@@ -202,7 +134,6 @@ jobs:
           cat "${GITHUB_OUTPUT}"
     outputs:
       go: ${{ steps.go.outputs.go == 'true' }}
-      telemetry_changed: ${{ steps.telemetry-changes.outputs.any_changed == 'true' }}
 
   check-autogen:
     needs: should-run
@@ -214,11 +145,6 @@ jobs:
     if: ${{ needs.should-run.outputs.go == 'true' }}
     uses: ./.github/workflows/reusable-check-levelization.yml
 
-  check-otel-naming:
-    needs: should-run
-    if: ${{ needs.should-run.outputs.go == 'true' }}
-    uses: ./.github/workflows/reusable-check-otel-naming.yml
-
   check-rename:
     needs: should-run
     if: ${{ needs.should-run.outputs.go == 'true' }}
@@ -324,11 +250,6 @@ jobs:
       remote_username: ${{ secrets.NEXUS_REMOTE_USERNAME }}
       remote_password: ${{ secrets.NEXUS_REMOTE_PASSWORD }}
 
-  telemetry-validation:
-    needs: should-run
-    if: ${{ needs.should-run.outputs.go == 'true' && needs.should-run.outputs.telemetry_changed == 'true' }}
-    uses: ./.github/workflows/reusable-telemetry-validation.yml
-
   notify-clio:
     needs: upload-recipe
     runs-on: ubuntu-latest
@@ -350,7 +271,6 @@ jobs:
     needs:
       - check-autogen
       - check-levelization
-      - check-otel-naming
       - check-rename
       - check-release-commits
       - clang-tidy
@@ -359,7 +279,6 @@ jobs:
       - package
       - upload-recipe
       - notify-clio
-      - telemetry-validation
     runs-on: ubuntu-latest
     steps:
       - name: Fail

.github/workflows/reusable-check-otel-naming.yml

@@ -1,78 +0,0 @@
-# This workflow checks that OpenTelemetry span-attribute names stay consistent
-# across the code (*SpanNames.h), collector, Tempo, dashboards, and docs.
-# See .github/scripts/otel-naming/check_otel_naming.py and the
-# "Telemetry span attribute naming" section in CONTRIBUTING.md.
-name: Check OTel naming
-
-# This workflow can only be triggered by other workflows.
-on: workflow_call
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-otel-naming
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  otel-naming:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-      - name: Test the OTel naming checker
-        # The checker's own unit tests, run before the check itself so a broken
-        # rule is reported as a broken rule rather than as a naming violation
-        # (or, worse, as a rule that silently stops flagging anything).
-        # stdlib `unittest` only: the repo installs no third-party test runner
-        # for CI, and the checker itself is deliberately dependency-free.
-        run: python -m unittest discover -s .github/scripts/otel-naming -p 'test_*.py' --verbose
-      - name: Check OTel naming
-        # The script is stdlib-only and reads only files already in the tree;
-        # it enforces each rule only when the layer it needs is present, so it
-        # works whether telemetry changes land in one PR or several.
-        run: python .github/scripts/otel-naming/check_otel_naming.py
-      - name: Check histogram bucket parity
-        # The C++ millisecond ladder and the collector's spanmetrics ladder are
-        # specified to agree over their shared range. They were identical when
-        # first shipped, then the collector side alone was extended and nothing
-        # noticed for eleven phases: native histograms stayed capped at 5s while
-        # spans reached 30s, so every quantile above 5s reported a flat 5000.
-        # Nothing but a check keeps two lists in step.
-        run: python .github/scripts/telemetry/check_bucket_parity.py
-      - name: Test the workload regression-bounds checker
-        # Its own tests, run before the check itself so a broken rule is
-        # reported as a broken rule rather than as a threshold violation. They
-        # also pin the input-handling contract: a placeholder baseline passes
-        # (that is the documented bootstrap state) while a missing, unreadable
-        # or malformed input fails, so deleting an input cannot silence the
-        # check. stdlib unittest only, as with the naming checker.
-        run: python -m unittest discover -s .github/scripts/telemetry -p 'test_*.py' --verbose
-      - name: Check workload regression bounds
-        # The workload gate's absolute bounds are derived from the committed
-        # baseline plus the two ladders, and nothing tied the three together.
-        # That let the gate break three times the same way -- the microsecond
-        # floor moved, the span floor moved, then the bounds stayed calibrated
-        # for a band the spans had left, so a 100x regression reported zero
-        # regressions and exit 0. Every failure looked like a green build.
-        # This asserts each bound is still the one its own baseline implies.
-        run: python .github/scripts/telemetry/check_regression_bounds.py
-      - name: Install the workload harness dependencies
-        # Its own step so a PyPI outage is reported as an install failure rather
-        # than as a failing harness. Last in the job, because every check above
-        # is stdlib-only and stays reachable if this fails.
-        run: pip3 install -r docker/telemetry/workload/requirements.txt
-      - name: Test the telemetry workload harness
-        # Run here rather than in the telemetry validation so a broken harness is
-        # reported in seconds instead of after a full xrpld build, and on every
-        # PR rather than only when that workflow's paths match.
-        #
-        # Plain scripts, not `unittest discover`: these files hold bare test
-        # functions, not TestCase subclasses, so discover would collect nothing
-        # and exit 0. Each file fails when it collects no tests, which is what
-        # makes running them this way safe.
-        run: |
-          python3 docker/telemetry/workload/test_validate_telemetry.py
-          python3 docker/telemetry/workload/test_capture_timings.py
-          python3 docker/telemetry/workload/test_tx_submitter.py

.github/workflows/reusable-telemetry-validation.yml

@@ -1,446 +0,0 @@
-# Telemetry Validation CI Workflow
-#
-# Builds rippled, runs the multi-node workload harness, validates all telemetry
-# data, and runs performance benchmarks.
-#
-# The build runs whatever the caller's tree configures, so a telemetry-off tree
-# is compiled here too. Only the validation is gated: a build with the tracing
-# compiled out emits no spans and no metrics to assert on.
-#
-# The caller decides only whether to start this at all. on-pr.yml runs it when
-# telemetry-relevant paths changed.
-#
-# The workflow is intentionally heavyweight (builds rippled, starts Docker
-# services, runs a multi-node cluster) — it validates the full telemetry
-# stack end-to-end rather than individual unit tests.
-#
-# Architecture: three jobs to leverage cached dependencies:
-#   0. linux-image-tag — reads the CI image tag from the build matrix so this
-#      workflow cannot drift onto a different compiler than the main CI.
-#   1. build-xrpld — runs on a self-hosted runner inside the same container
-#      image the main CI uses. This ensures Conan packages are fetched from
-#      the XRPLF remote instead of built from source, and ccache hits the
-#      remote cache. It also reports whether the configured build has telemetry
-#      compiled in.
-#   2. validate-telemetry — runs on ubuntu-latest (which has Docker) to
-#      launch the telemetry stack (OTel collector, Prometheus, Tempo, etc.)
-#      and validate the full pipeline end-to-end, against the binary job 1
-#      uploaded. Skipped when job 1 reports telemetry off.
-
-name: Telemetry Validation
-
-on:
-  workflow_call:
-    inputs:
-      run_benchmark:
-        description: "Run performance benchmarks."
-        required: false
-        type: boolean
-        default: false
-
-concurrency:
-  group: telemetry-validation-${{ github.ref }}
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-env:
-  BUILD_DIR: build
-
-jobs:
-  # ── Job 0: Resolve the CI image tag ────────────────────────────────
-  # The tag is pinned once, alongside the build matrix, in linux.json. Reading
-  # it here rather than hardcoding a second copy means this workflow always
-  # builds in the same image (and therefore the same compiler) as the main CI.
-  # A hardcoded copy silently went stale and left this job on gcc 13 after the
-  # rest of CI moved to gcc 15, which broke the build on code the main CI
-  # compiled fine.
-  linux-image-tag:
-    runs-on: ubuntu-latest
-    outputs:
-      tag: ${{ steps.tag.outputs.tag }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Read nix image tag
-        id: tag
-        run: echo "tag=$(jq -r .image_tag .github/scripts/strategy-matrix/linux.json)" >>"${GITHUB_OUTPUT}"
-
-  # ── Job 1: Build xrpld in the same container the main CI uses ──────
-  # This ensures Conan binary packages are fetched from the XRPLF remote
-  # (matching package IDs) and ccache hits the remote compilation cache.
-  build-xrpld:
-    name: Build xrpld
-    needs: linux-image-tag
-    outputs:
-      telemetry: ${{ steps.telemetry.outputs.enabled }}
-    runs-on: [self-hosted, Linux, X64, heavy]
-    container: ghcr.io/xrplf/xrpld/nix-debian:${{ needs.linux-image-tag.outputs.tag }}
-    timeout-minutes: 60
-    env:
-      CCACHE_NAMESPACE: telemetry-validation
-      CCACHE_REMOTE_ONLY: true
-      CCACHE_REMOTE_STORAGE: http://cache.dev.ripplex.io:8080|layout=bazel
-      CCACHE_SLOPPINESS: include_file_ctime,include_file_mtime
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Prepare runner
-        uses: XRPLF/actions/prepare-runner@c00c22ada3bd6bcda48fcb0d62fbbab49fec8a0f
-        with:
-          enable_ccache: ${{ github.repository_owner == 'XRPLF' }}
-
-      - name: Print build environment
-        uses: XRPLF/actions/print-build-env@59dec886e4afb05a1724443af08baccbc045b574
-
-      - name: Get number of processors
-        uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf
-        id: nproc
-        with:
-          subtract: 2
-
-      # The nix image ships several toolchains, so CC/CXX must be set
-      # explicitly for Conan to detect the intended one. gcc matches the
-      # debian gcc-release config the main CI builds.
-      - name: Set compiler environment
-        uses: ./.github/actions/set-compiler-env
-        with:
-          compiler: gcc
-
-      - name: Setup Conan
-        uses: ./.github/actions/setup-conan
-
-      - name: Build dependencies
-        uses: ./.github/actions/build-deps
-        with:
-          build_nproc: ${{ steps.nproc.outputs.nproc }}
-          build_type: Release
-          log_verbosity: verbose
-
-      # The telemetry option is deliberately not repeated here. Conan owns it and
-      # the generated toolchain hands its value to CMake. A command-line define
-      # is written to the cache before the toolchain runs and wins over it, so it
-      # could turn telemetry on with the package never fetched.
-      - name: Configure CMake
-        working-directory: ${{ env.BUILD_DIR }}
-        run: |
-          cmake \
-              -G Ninja \
-              -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
-              -DCMAKE_BUILD_TYPE=Release \
-              ..
-
-      # A build with tracing compiled out emits nothing to assert on, so the
-      # validation job below gates on this. Read from the configured build, so
-      # this file holds no copy of the setting.
-      #
-      # Every value CMake does not accept as a boolean fails the job, including a
-      # missing entry. A gate that guessed would skip the validation silently,
-      # which is the one failure it must not have.
-      - name: Read whether telemetry is compiled in
-        id: telemetry
-        working-directory: ${{ env.BUILD_DIR }}
-        run: |
-          value=$(sed -n 's/^telemetry:[^=]*=//p' CMakeCache.txt)
-          case "${value}" in
-              True | TRUE | true | ON | on | YES | yes | Y | y | 1) enabled=true ;;
-              False | FALSE | false | OFF | off | NO | no | N | n | 0) enabled=false ;;
-              *)
-                  echo "telemetry reads '${value}' in CMakeCache.txt, which is not" >&2
-                  echo "a boolean. conan install sets it through the toolchain." >&2
-                  exit 1
-                  ;;
-          esac
-          echo "telemetry compiled in: ${enabled}"
-          echo "enabled=${enabled}" >>"${GITHUB_OUTPUT}"
-
-      - name: Build xrpld
-        working-directory: ${{ env.BUILD_DIR }}
-        env:
-          BUILD_NPROC: ${{ steps.nproc.outputs.nproc }}
-        run: |
-          cmake \
-              --build . \
-              --config Release \
-              --parallel "${BUILD_NPROC}" \
-              --target xrpld
-
-      - name: Show ccache statistics
-        if: ${{ github.repository_owner == 'XRPLF' }}
-        run: ccache --show-stats -vv
-
-      - name: Upload xrpld binary
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: xrpld-telemetry
-          path: ${{ env.BUILD_DIR }}/xrpld
-          retention-days: 1
-          if-no-files-found: error
-
-  # ── Job 2: Run telemetry validation on ubuntu-latest (has Docker) ──
-  validate-telemetry:
-    name: Telemetry Stack Validation
-    needs: build-xrpld
-    # Reuses the binary job 1 built and uploaded, so a skip here costs no build.
-    if: needs.build-xrpld.outputs.telemetry == 'true'
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Install Python dependencies
-        run: pip3 install -r docker/telemetry/workload/requirements.txt
-
-      - name: Download xrpld binary
-        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
-        with:
-          name: xrpld-telemetry
-          path: ${{ env.BUILD_DIR }}
-
-      - name: Make binaries and scripts executable
-        run: |
-          chmod +x ${{ env.BUILD_DIR }}/xrpld
-          chmod +x docker/telemetry/workload/*.sh
-
-      - name: Run full telemetry validation
-        id: validation
-        env:
-          RPC_RATE: "50"
-          RPC_DURATION: "120"
-          TX_TPS: "5"
-          TX_DURATION: "120"
-          RUN_BENCHMARK: ${{ inputs.run_benchmark }}
-        run: |
-          # The four rate/duration flags below are inert:
-          # run-full-validation.sh parses them and never reads them. Load shape
-          # comes from the default --profile full-validation. They are still
-          # passed so the flags stay exercised if they are ever wired up.
-          ARGS="--xrpld ${{ env.BUILD_DIR }}/xrpld"
-          ARGS="$ARGS --rpc-rate $RPC_RATE"
-          ARGS="$ARGS --rpc-duration $RPC_DURATION"
-          ARGS="$ARGS --tx-tps $TX_TPS"
-          ARGS="$ARGS --tx-duration $TX_DURATION"
-          if [ "$RUN_BENCHMARK" = "true" ]; then
-              ARGS="$ARGS --with-benchmark"
-          fi
-          docker/telemetry/workload/run-full-validation.sh $ARGS
-        # continue-on-error allows subsequent steps (artifact upload,
-        # summary printing) to run even if validation fails. The final
-        # "Check validation result" step re-checks steps.validation.outcome
-        # (the pre-continue-on-error result) and fails the job properly.
-        continue-on-error: true
-
-      - name: Upload validation reports
-        if: always()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: telemetry-validation-reports
-          path: /tmp/xrpld-validation/reports/
-          retention-days: 30
-
-      # Keyed on the validation step's own outcome, not job status. The step
-      # above sets continue-on-error, so the job is not failing at this point
-      # and `if: failure()` never fires -- which silently skipped these logs on
-      # every failed run, and they are the only record of why a node did not
-      # reach consensus.
-      #
-      # stdout.log matters as much as debug.log: a node that dies before its
-      # log sink opens writes no debug.log at all, so stdout is the only place
-      # its reason survives. A run that timed out at 4/5 nodes was left
-      # undiagnosable because that file was not collected.
-      - name: Upload node logs
-        if: always() && steps.validation.outcome != 'success'
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: xrpld-node-logs
-          path: |
-            /tmp/xrpld-validation/node*/debug.log
-            /tmp/xrpld-validation/node*/stdout.log
-            /tmp/xrpld-validation/*.log
-          retention-days: 7
-          if-no-files-found: warn
-
-      - name: Print validation summary
-        if: always()
-        run: |
-          REPORT="/tmp/xrpld-validation/reports/validation-report.json"
-          if [ -f "$REPORT" ]; then
-              echo "## Telemetry Validation Results" >>"$GITHUB_STEP_SUMMARY"
-              echo "" >>"$GITHUB_STEP_SUMMARY"
-              TOTAL=$(jq '.summary.total' "$REPORT")
-              PASSED=$(jq '.summary.passed' "$REPORT")
-              FAILED=$(jq '.summary.failed' "$REPORT")
-              echo "| Metric | Value |" >>"$GITHUB_STEP_SUMMARY"
-              echo "|--------|-------|" >>"$GITHUB_STEP_SUMMARY"
-              echo "| Total Checks | $TOTAL |" >>"$GITHUB_STEP_SUMMARY"
-              echo "| Passed | $PASSED |" >>"$GITHUB_STEP_SUMMARY"
-              echo "| Failed | $FAILED |" >>"$GITHUB_STEP_SUMMARY"
-              echo "" >>"$GITHUB_STEP_SUMMARY"
-              if [ "$FAILED" -gt 0 ]; then
-                  echo "### Failed Checks" >>"$GITHUB_STEP_SUMMARY"
-                  echo "" >>"$GITHUB_STEP_SUMMARY"
-                  jq -r '.checks[] | select(.passed == false) | "- **\(.name)**: \(.message)"' "$REPORT" >>"$GITHUB_STEP_SUMMARY"
-              fi
-          fi
-
-      # Publishes captured OTel timings + regression report to the Step Summary.
-      # When the committed baseline is a placeholder AND the capture is
-      # complete, emits a fenced JSON block that can be copy-pasted directly
-      # into baselines/baseline-timings.json. An incomplete capture is named as
-      # such and its JSON withheld — see the comment on that branch below.
-      # When the baseline is populated, summarises the top regressions so the
-      # PR author sees the failure reason without downloading artifacts.
-      - name: Print regression summary
-        if: always()
-        run: |
-          set -euo pipefail
-          TIMINGS="/tmp/xrpld-validation/reports/timings.json"
-          REGRESSION="/tmp/xrpld-validation/reports/regression-report.json"
-          BASELINE="docker/telemetry/workload/baselines/baseline-timings.json"
-
-          if [ ! -f "$TIMINGS" ]; then
-              echo "## Regression Gate: no timings captured" >>"$GITHUB_STEP_SUMMARY"
-              echo "::warning::capture_timings.py did not produce timings.json — regression gate was not evaluated."
-              exit 0
-          fi
-
-          if [ ! -f "$BASELINE" ]; then
-              echo "## Regression Gate: baseline file missing" >>"$GITHUB_STEP_SUMMARY"
-              echo "::error::baselines/baseline-timings.json not found in checkout"
-              exit 1
-          fi
-
-          # NOTE: do NOT use `jq -e` here. With -e, jq exits non-zero when the
-          # filter's result is boolean false — which is the normal case for a
-          # populated (non-placeholder) baseline — and that would be
-          # misreported as a parse failure. Plain `jq -r` exits 0 on any valid
-          # JSON, so a real non-zero exit genuinely means malformed JSON.
-          IS_PLACEHOLDER=$(jq -r '.placeholder == true or (.metrics | length == 0)' "$BASELINE") || {
-              echo "::error::Failed to parse baseline JSON"
-              exit 1
-          }
-
-          # Whether the capture is usable as baseline material is the capture's
-          # own verdict, carried in the artifact by capture_timings.py, which
-          # computes it against --min-capture-ratio. It is READ here, never
-          # re-derived: a second copy of the ratio rule in shell would be a
-          # second source of truth and would drift from the producer.
-          #
-          # `// false` covers both an artifact written before the block existed
-          # and a truncated one. Neither can prove it is complete, so neither is
-          # offered — the whole point is that a degraded capture must not look
-          # like a good one. Same `jq -r` reasoning as the baseline parse above.
-          CAPTURE_COMPLETE=$(jq -r '.capture.complete // false' "$TIMINGS") || {
-              echo "::error::Failed to parse timings JSON"
-              exit 1
-          }
-          if [ "$(jq -r 'has("capture")' "$TIMINGS")" = "true" ]; then
-              CAPTURE_COUNTS=$(jq -r '"\(.capture.captured)/\(.capture.declared)"' "$TIMINGS")
-              CAPTURE_SHORTFALL="only **$CAPTURE_COUNTS** declared metrics came back, below the capture's own minimum ratio"
-          else
-              CAPTURE_COUNTS="unknown"
-              CAPTURE_SHORTFALL="the artifact carries no \`capture\` block, so it predates completeness reporting and cannot state what it captured"
-          fi
-
-          echo "## OTel Timings Regression Gate" >>"$GITHUB_STEP_SUMMARY"
-          echo "" >>"$GITHUB_STEP_SUMMARY"
-
-          if [ "$IS_PLACEHOLDER" = "true" ] && [ "$CAPTURE_COMPLETE" != "true" ]; then
-              # The placeholder path is the ONLY route to a committed baseline,
-              # which makes it the one place an incomplete capture does lasting
-              # damage: pasted in, it silently narrows the gate to the keys that
-              # happened to come back. So the JSON is withheld rather than
-              # printed with a caveat — a warning above a copyable block is
-              # still a copyable block. The counts are shown so the reader knows
-              # how thin it was, and the artifact is still uploaded for anyone
-              # who needs to inspect it deliberately.
-              echo "### Baseline NOT refreshable from this run" >>"$GITHUB_STEP_SUMMARY"
-              echo "" >>"$GITHUB_STEP_SUMMARY"
-              echo "The committed baseline is a placeholder, so this run would" \
-                  "normally print a block to paste into" \
-                  "\`baselines/baseline-timings.json\`. It is withheld because" \
-                  "$CAPTURE_SHORTFALL, so the JSON may describe metrics that" \
-                  "were never measured. Pasting it would narrow the gate to" \
-                  "whichever keys were captured, with nothing reporting that" \
-                  "it had narrowed." >>"$GITHUB_STEP_SUMMARY"
-              echo "" >>"$GITHUB_STEP_SUMMARY"
-              echo "Fix the capture first — the usual cause is Prometheus not" \
-                  "being scraped for long enough, or nodes not reaching" \
-                  "consensus — then re-run. See the \`timings.json\` artifact's" \
-                  "\`capture\` block for the exact counts." >>"$GITHUB_STEP_SUMMARY"
-              echo "::error::Timing capture is incomplete ($CAPTURE_COUNTS metrics) — no baseline block printed. Do not refresh the baseline from this run."
-          elif [ "$IS_PLACEHOLDER" = "true" ]; then
-              echo "### Paste into \`baselines/baseline-timings.json\`" >>"$GITHUB_STEP_SUMMARY"
-              echo "" >>"$GITHUB_STEP_SUMMARY"
-              echo "The committed baseline is a placeholder. Open a PR replacing" \
-                  "its contents with the JSON block below to activate the" \
-                  "regression gate. The capture is complete" \
-                  "($CAPTURE_COUNTS declared metrics)." >>"$GITHUB_STEP_SUMMARY"
-              echo "" >>"$GITHUB_STEP_SUMMARY"
-              echo '```json' >>"$GITHUB_STEP_SUMMARY"
-              cat "$TIMINGS" >>"$GITHUB_STEP_SUMMARY"
-              echo '```' >>"$GITHUB_STEP_SUMMARY"
-          elif [ -f "$REGRESSION" ]; then
-              # Existence is not readability: a truncated report satisfies -f,
-              # and the `|| =0` fallbacks below would then render a clean table
-              # of zeros for a run that compared nothing. Same trap the note
-              # above the baseline parse warns about, so check the shape first.
-              SUMMARY_OK=$(jq -r 'if (.summary | type) == "object" then "yes" else "no" end' \
-                  "$REGRESSION" 2>/dev/null) || SUMMARY_OK=no
-              if [ "$SUMMARY_OK" != "yes" ]; then
-                  echo "## Regression Gate: report unreadable" >>"$GITHUB_STEP_SUMMARY"
-                  echo "" >>"$GITHUB_STEP_SUMMARY"
-                  echo "\`$REGRESSION\` exists but carries no \`summary\` object, so no" \
-                      "table can be rendered. The pass/fail above still comes from the" \
-                      "comparator's exit code." >>"$GITHUB_STEP_SUMMARY"
-                  echo "::error::Regression report is present but has no summary object"
-              else
-                  # No `jq -e`: it exits non-zero when a field is legitimately 0 or
-                  # false, which the fallbacks would silently turn into 0 as well.
-                  REGR_COUNT=$(jq -r '.summary.regressions // 0' "$REGRESSION")
-                  IMPR_COUNT=$(jq -r '.summary.improvements // 0' "$REGRESSION")
-                  TOTAL=$(jq -r '.summary.total // 0' "$REGRESSION")
-                  MISSING_COUNT=$(jq -r '.summary.missing_in_current // 0' "$REGRESSION")
-                  COMPARED=$(jq -r '.summary.compared // 0' "$REGRESSION")
-                  # `total` is every key in the report, i.e. the union of the
-                  # baseline and this run, so it is not what was gated. The
-                  # comparator reports `compared` for that; do not derive it from
-                  # total minus missing, because a key can also be skipped for
-                  # being new or for having no data on either side.
-                  echo "| Stat | Count |" >>"$GITHUB_STEP_SUMMARY"
-                  echo "|------|-------|" >>"$GITHUB_STEP_SUMMARY"
-                  echo "| Metrics in report | $TOTAL |" >>"$GITHUB_STEP_SUMMARY"
-                  echo "| Metrics compared | $COMPARED |" >>"$GITHUB_STEP_SUMMARY"
-                  echo "| Not captured this run | $MISSING_COUNT |" >>"$GITHUB_STEP_SUMMARY"
-                  echo "| Regressions | $REGR_COUNT |" >>"$GITHUB_STEP_SUMMARY"
-                  echo "| Improvements | $IMPR_COUNT |" >>"$GITHUB_STEP_SUMMARY"
-                  echo "" >>"$GITHUB_STEP_SUMMARY"
-                  if [ "$MISSING_COUNT" -gt 0 ]; then
-                      echo "::warning::$MISSING_COUNT baseline metric(s) were not captured this run, so they were not gated"
-                  fi
-                  if [ "$REGR_COUNT" -gt 0 ]; then
-                      echo "### Regressions" >>"$GITHUB_STEP_SUMMARY"
-                      echo "" >>"$GITHUB_STEP_SUMMARY"
-                      echo "| Metric | Baseline | Current | Δ | % | Unit |" >>"$GITHUB_STEP_SUMMARY"
-                      echo "|--------|---------:|--------:|--:|--:|------|" >>"$GITHUB_STEP_SUMMARY"
-                      jq -r '.metrics[] | select(.regressed) | "| \(.key) | \(.baseline) | \(.current) | \(.delta) | \(.pct_change)% | \(.unit) |"' \
-                          "$REGRESSION" >>"$GITHUB_STEP_SUMMARY"
-                  fi
-              fi
-          fi
-
-      - name: Cleanup
-        if: always()
-        run: |
-          docker/telemetry/workload/run-full-validation.sh --cleanup 2>/dev/null || true
-
-      - name: Check validation result
-        if: steps.validation.outcome == 'failure'
-        run: |
-          echo "Telemetry validation failed. Check the uploaded reports for details."
-          exit 1

.gitignore

@@ -100,6 +100,3 @@ target/
 
 # Rust build directory
 crates/target
-
-# Env. file carrying environmental setup data for local or cloud runs.
-.env.*

CMakeLists.txt

@@ -140,29 +140,6 @@ if(rocksdb)
     target_link_libraries(xrpl_libs INTERFACE RocksDB::rocksdb)
 endif()
 
-# OpenTelemetry distributed tracing (optional).
-# When on, links against opentelemetry-cpp and defines XRPL_ENABLE_TELEMETRY so
-# that SpanGuard factory methods produce real OTel spans.
-# When off, all tracing code compiles to no-ops with zero overhead and
-# opentelemetry-cpp is not needed at all.
-#
-# There is no CMake option. The one switch is `conan install -o telemetry=`,
-# which decides whether opentelemetry-cpp is fetched and sets the variable read
-# below through the generated toolchain.
-#
-# The define is set for every target in this project, and also PUBLIC on
-# xrpl.libxrpl.telemetry (cmake/XrplCore.cmake), so a parent project that
-# adds this one with add_subdirectory() and links that module gets it too.
-#
-# -DXRPL_ENABLE_TELEMETRY=OFF turns nothing off: that name is only a compile
-# definition added below, so CMake lists it as an unused variable at the end of
-# configuration.
-if(telemetry)
-    find_package(opentelemetry-cpp CONFIG REQUIRED)
-    add_compile_definitions(XRPL_ENABLE_TELEMETRY)
-    message(STATUS "OpenTelemetry tracing enabled")
-endif()
-
 # Work around changes to Conan recipe for now.
 if(TARGET nudb::core)
     set(nudb nudb::core)

CONTRIBUTING.md

@@ -377,150 +377,6 @@ run-clang-tidy -p build -quiet -fix -format -allow-no-checks src tests
 
 `-format` reformats the fixed code with [`.clang-format`](./.clang-format); without it the fixes are inserted in LLVM style and the `clang-format` hook rewrites them afterwards.
 
-## Telemetry span attribute naming
-
-OpenTelemetry span attribute keys follow these rules so they stay consistent
-across the code, the OTel collector, Tempo, Grafana dashboards, and docs. The
-constants in the `*SpanNames.h` headers are the single source of truth; every
-other layer must match them. A CI check enforces this end to end.
-
-1. Per-span unique attribute: bare field name — allowed when the field is
-   recorded by a single span/workflow, so the span name already supplies the
-   domain (e.g. `command`, `local`, `version` on `rpc.command` / `tx.process`).
-2. Shared attribute (same concept on more than one span): ONE key, reused
-   verbatim on every span that records it — the span name tells the occurrences
-   apart, so no per-emitter prefix is added. Pick the name by the field's
-   meaning: a property of a domain object keeps that object's bare field name
-   (`ledger_hash`, `ledger_seq`, `tx_hash`, `peer_id`, `full_validation`); a
-   field already qualified by a sub-kind keeps that qualifier on every emitter
-   (`proposal_trusted` on both `consensus.proposal.receive` and
-   `peer.proposal.receive`; `validation_trusted` likewise). Define it once in
-   the base `SpanNames.h` `namespace attr` block and re-export (`using`) it from
-   each domain header, so all emitters share the exact string.
-3. Collision qualifier: `<domain>_<field>` — only when a bare name would collide
-   with a DIFFERENT concept in the shared spanmetrics label space, or with the
-   OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`,
-   `consensus_phase`, `consensus_round`). This disambiguates distinct concepts
-   that share a word; it is NOT used to tag the same concept with the workflow
-   that emitted it — that is rule 2 (one shared name).
-4. Resource attribute: dotted `xrpl.<subsystem>.<field>` — reserved ONLY for
-   process/network identity set once at startup (`xrpl.network.id`,
-   `xrpl.network.type`). Never use the dotted `xrpl.` form for span attributes.
-5. Span names use `<subsystem>[.<component>]` (dotted). Only attribute _keys_
-   follow rules 1–4.
-
-All attribute keys are `lower_snake_case` (lowercase letters, digits, and
-underscores; each dot-separated segment of a resource key likewise). No
-camelCase, uppercase, or spaces.
-
-Standard OpenTelemetry semantic-convention keys keep their canonical dotted
-form (e.g. `service.*` resource attributes, `http.*` span attributes); the
-"no dotted form" rule above applies to xrpl-custom keys, not to OTel-standard
-conventions.
-
-Always reference the `*SpanNames.h` constants for attribute keys and span
-names — never pass a string literal as a key or as a `span`/`childSpan` name
-argument. (Attribute _values_ may be runtime data.)
-
-These rules are enforced by `.github/scripts/otel-naming/check_otel_naming.py`,
-run in CI on every pull request. The check derives the set of valid keys
-directly from the `*SpanNames.h` constants and the resource attributes the code
-registers, so there is no separate list to keep in sync. It cross-validates the
-collector, Tempo, dashboards, and docs against those keys, and each rule runs
-only when the file it needs is present — so it works whether telemetry changes
-land in one pull request or several. Run it locally with:
-
-```
-python .github/scripts/otel-naming/check_otel_naming.py
-```
-
-### Naming a wrong form in prose (`otel-naming:allow-dotted`)
-
-The doc rule (E) flags any dotted `` `xrpl.<domain>.<field>` `` key in the
-telemetry docs, because a reader copies those keys straight into a TraceQL or
-PromQL query. A doc that _teaches_ the convention, or records a rename, has to be
-able to name the wrong form as a counter-example. That mention is opted out with
-a marker naming exactly the keys the line is allowed to mention:
-
-```markdown
-Use `tx_hash`, not `xrpl.tx.hash`.
-<!-- otel-naming:allow-dotted: xrpl.tx.hash -->
-```
-
-- The marker applies to **its own line only**, and exempts **only the keys it
-  lists** (comma- and/or space-separated, backticks optional). A dotted key on a
-  marked line that the marker does not name still fails, so an exemption cannot
-  quietly widen when someone edits the line later.
-- A marker with no key list exempts nothing and reports a warning; so does a
-  marker naming a key the line no longer mentions (a stale exemption).
-- Never use it to keep a real attribute table dotted. If the doc publishes a key
-  an operator is meant to query, fix the key — the marker is for mentions, not
-  for published attributes.
-
-See [.github/scripts/otel-naming/README.md](.github/scripts/otel-naming/README.md)
-for the full rule list.
-
-## Telemetry metric naming
-
-The metric-side counterpart of the span rules above. Metric instrument names and
-metric label keys are duplicated across the emit site, the instrument
-registration, the unit test, `expected_metrics.json`, the dashboard PromQL and
-the runbook, so a rename touches six places and a typo in any one of them fails
-silently at runtime — a metric that never appears, or a label that never joins.
-The constants in the `*MetricNames.h` headers are the single source of truth for
-the C++ layers; a CI check validates the layers that cannot reference a constant.
-
-1. Instrument names are bare `lower_snake_case` with **no `xrpld_` prefix**.
-   Nothing in the export path adds a prefix, so queries use the bare name.
-2. A monotonic counter ends in `_total`, so `rate()` over it reads correctly and
-   a reader can tell it from a gauge at a glance.
-3. A duration carries its unit as the suffix — `_us`, `_ms` or `_seconds`. The
-   instruments these rules cover set no OTel `unit`, so the name is the only
-   place the unit shows.
-4. A gauge that snapshots current state takes no suffix (`jobq_saturation`,
-   `sync_state`), and never `_total`.
-5. Label keys are `lower_snake_case` and must have **bounded** cardinality. A
-   multi-series gauge discriminates its readings with the `metric` label rather
-   than minting one instrument per reading.
-6. Label **values** are declared as constants only when the code picks them from
-   a fixed set (`namespace lval`). A value derived from runtime data — a peer
-   address, a ledger hash — must never become a label on a metric.
-
-Always reference the `*MetricNames.h` constants for instrument names and label
-keys — never pass a string literal. (Label _values_ may be runtime data.) Note
-that these headers use `constexpr char[]`, not the `makeStr`/`StaticStr` DSL the
-`*SpanNames.h` headers use: the OTel C++ API takes `nostd::string_view`, which
-constructs from `char const*` but has no constructor from `std::string_view`, so
-`StaticStr` does not compile in an instrument-name or label-key position.
-
-Enforcement is by the same script as the span rules, whose metric rules are:
-
-- **I** — no string literal as an instrument name or label key at an emit site
-  (the mirror of Rule F). Scoped by metric _family_ (the first underscore
-  segment) so conversion can proceed subsystem by subsystem: declaring a
-  constant opts that family in. An unconverted family is reported as a
-  non-fatal **L** warning, keeping the remaining work visible.
-- **J** — the suffix conventions above. The instrument _kind_ is read from the
-  emit site, not guessed from the name, so a multi-series gauge whose units live
-  in its label values is not mistaken for a mis-suffixed duration. A name created
-  through two different factories is reported as a kind conflict rather than a
-  suffix complaint, because no suffix can be correct for both.
-- **K** — every metric named in `docker/telemetry/workload/expected_metrics.json`
-  resolves to a declared constant. This is the check that catches a metric
-  renamed in code while the workload validator still asserts the old name.
-  Groups whose names come from a different emit path (`statsd_gauges`,
-  `statsd_counters` from `beast::insight`, and collector-derived `spanmetrics`)
-  are out of scope by design.
-
-## Adding a new OTel metric
-
-See `include/xrpl/telemetry/MetricMacros.h` for the call-site macros covering every
-OTel instrument kind (Counter, UpDownCounter, Histogram, Gauge, and their
-Observable/async counterparts), `src/xrpld/telemetry/MetricNames.h` for the name
-and label constants to reference (and the rules above), and the "Adding a New
-Metric" section in [docs/telemetry-runbook.md](docs/telemetry-runbook.md) for the
-walkthrough and a need-to-macro lookup table.
-
 ## Contracts and instrumentation
 
 We are using [Antithesis](https://antithesis.com/) for continuous fuzzing,

OpenTelemetryPlan/00-tracing-fundamentals.md

@@ -1,570 +0,0 @@
-# Distributed Tracing Fundamentals
-
-> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md)
-> **Next**: [Architecture Analysis](./01-architecture-analysis.md)
-
----
-
-## What is Distributed Tracing?
-
-Distributed tracing is a method for tracking data objects as they flow through distributed systems. In a network like XRP Ledger, a single transaction touches multiple independent nodes—each with no shared memory or logging. Distributed tracing connects these dots.
-
-**Without tracing:** You see isolated logs on each node with no way to correlate them.
-
-**With tracing:** You see the complete journey of a transaction or an event across all nodes it touched.
-
----
-
-## Actors and Actions at a Glance
-
-### Actors
-
-| Who (Plain English)                            | Technical Term  |
-| ---------------------------------------------- | --------------- |
-| A single unit of work being tracked            | Span            |
-| The complete journey of a request              | Trace           |
-| Data that links spans across services          | Trace Context   |
-| Code that creates spans and propagates context | Instrumentation |
-| Service that receives and processes traces     | Collector       |
-| Storage and visualization system               | Backend (Tempo) |
-| Decision logic for which traces to keep        | Sampler         |
-
-### Actions
-
-| What Happens (Plain English)            | Technical Term          |
-| --------------------------------------- | ----------------------- |
-| Start tracking a new operation          | Create a Span           |
-| Connect a child operation to its parent | Set `parent_span_id`    |
-| Group all related operations together   | Share a `trace_id`      |
-| Pass tracking data between services     | Context Propagation     |
-| Decide whether to record a trace        | Sampling (Head or Tail) |
-| Send completed traces to storage        | Export (OTLP)           |
-
----
-
-## Core Concepts
-
-### 1. Trace
-
-A **trace** represents the entire journey of a request through the system. It has a unique `trace_id` that stays constant across all nodes.
-
-```
-Trace ID: abc123
-├── Node A: received transaction
-├── Node B: relayed transaction
-├── Node C: included in consensus
-└── Node D: applied to ledger
-```
-
-### 2. Span
-
-A **span** represents a single unit of work within a trace. Each span has:
-
-| Attribute        | Description                      | Example                    |
-| ---------------- | -------------------------------- | -------------------------- |
-| `trace_id`       | Identifies the trace             | `event123`                 |
-| `span_id`        | Unique identifier                | `span456`                  |
-| `parent_span_id` | Parent span (if any)             | `p_span123`                |
-| `name`           | Operation name                   | `rpc.submit`               |
-| `start_time`     | When work began (local time)     | `2024-01-15T10:30:00Z`     |
-| `end_time`       | When work completed (local time) | `2024-01-15T10:30:00.050Z` |
-| `attributes`     | Key-value metadata               | `tx_hash=ABC...`           |
-| `status`         | OK, ERROR MSG                    | `OK`                       |
-
-### 3. Trace Context
-
-**Trace context** is the data that propagates between services to link spans together. It contains:
-
-- `trace_id` - The trace this span belongs to
-- `span_id` - The current span (becomes parent for child spans)
-- `trace_flags` - Sampling decisions
-
----
-
-## How Spans Form a Trace
-
-Spans have parent-child relationships forming a tree structure:
-
-```mermaid
-flowchart TB
-    subgraph trace["Trace: abc123"]
-        A["tx.submit<br/>span_id: 001<br/>50ms"] --> B["tx.validate<br/>span_id: 002<br/>5ms"]
-        A --> C["tx.relay<br/>span_id: 003<br/>10ms"]
-        A --> D["tx.apply<br/>span_id: 004<br/>30ms"]
-        D --> E["ledger.update<br/>span_id: 005<br/>20ms"]
-    end
-
-    style A fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style B fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style C fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style D fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style E fill:#bf360c,stroke:#8c2809,color:#ffffff
-```
-
-**Reading the diagram:**
-
-- **tx.submit (blue, root)**: The top-level span representing the entire transaction submission; all other spans are its descendants.
-- **tx.validate, tx.relay, tx.apply (green)**: Direct children of tx.submit, representing the three main stages -- validation, relay to peers, and application to the ledger.
-- **ledger.update (red)**: A grandchild span nested under tx.apply, representing the actual ledger state mutation triggered by applying the transaction.
-- **Arrows (parent to child)**: Each arrow indicates a parent-child span relationship where the parent's completion depends on the child finishing.
-
-The same trace visualized as a **timeline (Gantt chart)**:
-
-```
-Time →   0ms    10ms    20ms    30ms    40ms    50ms
-         ├───────────────────────────────────────────┤
-tx.submit│▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓│
-         ├─────┤
-tx.valid │▓▓▓▓▓│
-         │     ├──────────┤
-tx.relay │     │▓▓▓▓▓▓▓▓▓▓│
-         │               ├────────────────────────────┤
-tx.apply │               │▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓│
-         │                         ├──────────────────┤
-ledger   │                         │▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓│
-```
-
----
-
-## Span Relationships
-
-Spans don't always form simple parent-child trees. Distributed tracing defines several relationship types to capture different causal patterns:
-
-### 1. Parent-Child (ChildOf)
-
-The default relationship. The parent span **depends on** or **contains** the child span. The child runs within the scope of the parent.
-
-```
-tx.submit (parent)
-├── tx.validate (child)     ← parent waits for this
-├── tx.relay (child)        ← parent waits for this
-└── tx.apply (child)        ← parent waits for this
-```
-
-**When to use:** Synchronous calls, nested operations, any case where the parent's completion depends on the child.
-
-### 2. Follows-From
-
-A causal relationship where the first span **triggers** the second, but does **not wait** for it. The originator fires and moves on.
-
-```
-Time →
-
-tx.receive [=======]
-                     ↓ triggers (follows-from)
-              tx.relay   [===========]   ← runs independently
-```
-
-**When to use:** Asynchronous jobs, queued work, fire-and-forget patterns. For example, a node receives a transaction and queues it for relay — the relay span _follows from_ the receive span but the receiver doesn't wait for relaying to complete.
-
-> **OpenTracing** defined `FollowsFrom` as a first-class reference type alongside `ChildOf`.
-> **OpenTelemetry** represents this using **Span Links** with descriptive attributes instead (see below).
-
-### 3. Span Links (Cross-Trace and Non-Hierarchical)
-
-Links connect spans that are **causally related but not in a parent-child hierarchy**. Unlike parent-child, links can cross trace boundaries.
-
-```
-Trace A                          Trace B
-──────                           ──────
-batch.schedule                   batch.execute
-├─ item.enqueue (span X)    ┌──► process.item
-├─ item.enqueue (span Y) ───┤    (links to X, Y, Z)
-├─ item.enqueue (span Z)    └──►
-```
-
-**Use cases:**
-
-| Pattern              | Description                                                                 |
-| -------------------- | --------------------------------------------------------------------------- |
-| **Batch processing** | A batch span links back to all individual spans that contributed to it      |
-| **Fan-in**           | An aggregation span links to the multiple producer spans it merges          |
-| **Fan-out**          | Multiple downstream spans link back to the single span that triggered them  |
-| **Async handoff**    | A deferred job links back to the request that queued it (follows-from)      |
-| **Cross-trace**      | Correlating spans across independent traces (e.g., retries, related events) |
-
-**Link structure:** Each link carries the target span's context plus optional attributes:
-
-```
-Link {
-    trace_id:   <target trace>
-    span_id:    <target span>
-    attributes: { "link.description": "triggered by batch scheduler" }
-}
-```
-
-### Relationship Summary
-
-```mermaid
-flowchart LR
-    subgraph parent_child["Parent-Child"]
-        direction TB
-        P["Parent"] --> C["Child"]
-    end
-
-    subgraph follows_from["Follows-From"]
-        direction TB
-        A["Span A"] -.->|triggers| B["Span B"]
-    end
-
-    subgraph links["Span Links"]
-        direction TB
-        X["`Span X
-(Trace 1)`"] -.-|link| Y["`Span Y
-(Trace 2)`"]
-    end
-
-    parent_child ~~~ follows_from ~~~ links
-
-    style P fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style C fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style A fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style B fill:#bf360c,stroke:#8c2809,color:#ffffff
-    style X fill:#4a148c,stroke:#38006b,color:#ffffff
-    style Y fill:#4a148c,stroke:#38006b,color:#ffffff
-```
-
-| Relationship     | Same Trace? | Dependency?                | OTel Mechanism    |
-| ---------------- | ----------- | -------------------------- | ----------------- |
-| **Parent-Child** | Yes         | Parent depends on child    | `parent_span_id`  |
-| **Follows-From** | Usually     | Causal but no dependency   | Link + attributes |
-| **Span Link**    | Either      | Correlation, no dependency | Link + attributes |
-
----
-
-## Trace ID Generation
-
-A `trace_id` is a 128-bit (16-byte) identifier that groups all spans belonging to one logical operation. How it's generated determines how easily you can find and correlate traces later.
-
-### General Approaches
-
-#### 1. Random (W3C Default)
-
-Generate a random 128-bit ID when a trace starts. Standard approach for most services.
-
-```
-trace_id = random_128_bits()
-```
-
-| Pros                        | Cons                                          |
-| --------------------------- | --------------------------------------------- |
-| Simple, standard            | No natural correlation to domain events       |
-| Guaranteed unique per trace | If propagation is lost, trace is broken       |
-| Works with all OTel tooling | "Find trace for TX abc" requires index lookup |
-
-#### 2. Deterministic (Derived from Domain Data)
-
-Compute the trace_id from a hash of a natural identifier. Every node independently derives the **same** trace_id for the same event.
-
-```
-trace_id = SHA-256(domain_identifier)[0:16]   // truncate to 128 bits
-```
-
-| Pros                                                | Cons                                                       |
-| --------------------------------------------------- | ---------------------------------------------------------- |
-| Propagation-resilient — same ID computed everywhere | Same event processed twice (retry) shares trace_id         |
-| Natural search — domain ID maps directly to trace   | Non-standard (tooling assumes random)                      |
-| No coordination needed between nodes                | 256→128 bit truncation (collision risk negligible at ~2⁶⁴) |
-
-#### 3. Hybrid (Deterministic Prefix + Random Suffix)
-
-First 8 bytes derived from domain data, last 8 bytes random.
-
-```
-trace_id = SHA-256(domain_identifier)[0:8] || random_64_bits()
-```
-
-| Pros                                        | Cons                                     |
-| ------------------------------------------- | ---------------------------------------- |
-| Prefix search: "find all traces for TX abc" | Must propagate to maintain full trace_id |
-| Unique per processing instance              | More complex generation logic            |
-| Retries get distinct trace_ids              | Partial correlation only (prefix match)  |
-
-### XRPL Workflow Analysis
-
-XRPL has a unique advantage: its core workflows produce **globally unique 256-bit hashes** that are known on every node. This makes deterministic trace_id generation practical in ways most systems can't achieve.
-
-#### Natural Identifiers by Workflow
-
-| Workflow            | Natural Identifier                | Size       | Known at Start?               | Same on All Nodes?               |
-| ------------------- | --------------------------------- | ---------- | ----------------------------- | -------------------------------- |
-| **Transaction**     | Transaction hash (`tid_`)         | 256-bit    | Yes — computed before signing | Yes — hash of canonical tx data  |
-| **Consensus round** | Previous ledger hash + ledger seq | 256+32 bit | Yes — known when round opens  | Yes — all validators agree       |
-| **Validation**      | Ledger hash being validated       | 256-bit    | Yes — from consensus result   | Yes — same closed ledger         |
-| **Ledger catch-up** | Target ledger hash                | 256-bit    | Yes — we know what to fetch   | Yes — identifies ledger globally |
-
-#### Where These Identifiers Live in Code
-
-```
-Transaction:     STTx::getTransactionID()     → uint256 tid_
-                 TMTransaction::rawTransaction → recompute hash from bytes
-
-Consensus:       ConsensusProposal::previousLedger_ → uint256 (previous ledger hash)
-                 ConsensusProposal::position_       → uint256 (TxSet hash)
-                 LedgerHeader::seq                  → uint32_t (ledger sequence)
-
-Validation:      STValidation::getLedgerHash()  → uint256
-                 STValidation::getNodeID()      → NodeID (160-bit)
-
-Ledger fetch:    InboundLedger constructor      → uint256 hash, uint32_t seq
-                 TMGetLedger::ledgerHash        → bytes (uint256)
-```
-
-### Recommended Strategy: Workflow-Scoped Deterministic
-
-Each workflow type derives its trace_id from its natural domain identifier:
-
-```
-Transaction trace:   trace_id = SHA-256("tx"    || tx_hash)[0:16]
-Consensus trace:     trace_id = SHA-256("cons"  || prev_ledger_hash || ledger_seq)[0:16]
-Ledger catch-up:     trace_id = SHA-256("fetch" || target_ledger_hash)[0:16]
-```
-
-The string prefix (`"tx"`, `"cons"`, `"fetch"`) prevents collisions between workflows that might share underlying hashes.
-
-**Why this works for XRPL:**
-
-1. **Propagation-resilient** — Even if a P2P message drops trace context, every node independently computes the same trace_id from the same tx_hash or ledger_hash. Spans still correlate.
-
-2. **Zero-cost search** — "Show me the trace for transaction ABC" becomes a direct lookup: compute `SHA-256("tx" || ABC)[0:16]` and query. No secondary index needed.
-
-3. **Cross-workflow linking via Span Links** — A consensus trace links to individual transaction traces. A validation span links to the consensus trace. This connects the full picture without forcing everything into one giant trace.
-
-### Cross-Workflow Correlation
-
-Each workflow gets its own trace. Span Links tie them together:
-
-```mermaid
-flowchart TB
-    subgraph tx_trace["Transaction Trace"]
-        direction LR
-        Tn["trace_id = f(tx_hash)"]:::note --> T1["tx.receive"] --> T2["tx.validate"] --> T3["tx.relay"]
-    end
-
-    subgraph cons_trace["Consensus Trace"]
-        direction LR
-        Cn["trace_id = f(prev_ledger, seq)"]:::note --> C1["cons.open"] --> C2["cons.propose"] --> C3["cons.accept"]
-    end
-
-    subgraph val_trace["Validation"]
-        direction LR
-        Vn["spans within consensus trace"]:::note --> V1["val.create"] --> V2["val.broadcast"]
-    end
-
-    subgraph fetch_trace["Catch-Up Trace"]
-        direction LR
-        Fn["trace_id = f(ledger_hash)"]:::note --> F1["fetch.request"] --> F2["fetch.receive"] --> F3["fetch.apply"]
-    end
-
-    C1 -.-|"`span link
-(tx traces)`"| T3
-    C3 --> V1
-    F1 -.-|"`span link
-(target ledger)`"| C3
-
-    classDef note fill:none,stroke:#888,stroke-dasharray:5 5,color:#333,font-style:italic
-    style T1 fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style T2 fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style T3 fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style C1 fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style C2 fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style C3 fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style V1 fill:#bf360c,stroke:#8c2809,color:#ffffff
-    style V2 fill:#bf360c,stroke:#8c2809,color:#ffffff
-    style F1 fill:#4a148c,stroke:#38006b,color:#ffffff
-    style F2 fill:#4a148c,stroke:#38006b,color:#ffffff
-    style F3 fill:#4a148c,stroke:#38006b,color:#ffffff
-```
-
-**Reading the diagram:**
-
-- **Transaction Trace (blue)**: An independent trace whose `trace_id` is deterministically derived from the transaction hash. Contains receive, validate, and relay spans.
-- **Consensus Trace (green)**: An independent trace whose `trace_id` is derived from the previous ledger hash and sequence number. Covers the open, propose, and accept phases.
-- **Validation (red)**: Validation spans live within the consensus trace (not a separate trace). They are created after the accept phase completes.
-- **Catch-Up Trace (purple)**: An independent trace for ledger acquisition, derived from the target ledger hash. Used when a node is behind and fetching missing ledgers.
-- **Dotted arrows (span links)**: Cross-trace correlations. Consensus links to transaction traces it included; catch-up links to the consensus trace that produced the target ledger.
-- **Solid arrow (C3 to V1)**: A parent-child relationship -- validation spans are direct children of the consensus accept span within the same trace.
-
-**How a query flows:**
-
-```
-"Why was TX abc slow?"
-  1. Compute trace_id = SHA-256("tx" || abc)[0:16]
-  2. Find transaction trace → see it was included in consensus round N
-  3. Follow span link → consensus trace for round N
-  4. See which phase was slow (propose? accept?)
-  5. If a node was catching up, follow link → catch-up trace
-```
-
-### Trade-offs to Consider
-
-| Concern                       | Mitigation                                                                                                                    |
-| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------- |
-| **Retries get same trace_id** | Add `attempt` attribute to root span; spans have unique span_ids and timestamps                                               |
-| **256→128 bit truncation**    | Birthday-bound collision at ~2⁶⁴ operations — negligible for XRPL's throughput                                                |
-| **Non-standard generation**   | OTel spec allows any 16-byte non-zero value; tooling works on the hex string                                                  |
-| **Hash computation cost**     | SHA-256 is ~0.3μs per call; XRPL already computes these hashes for other purposes                                             |
-| **Late-binding identifiers**  | Ledger hash isn't known until after consensus — validation spans use ledger_seq as fallback, then link to the consensus trace |
-
----
-
-## Distributed Traces Across Nodes
-
-In distributed systems like xrpld, traces span **multiple independent nodes**. The trace context must be propagated in network messages:
-
-```mermaid
-sequenceDiagram
-    participant Client
-    participant NodeA as Node A
-    participant NodeB as Node B
-    participant NodeC as Node C
-
-    Client->>NodeA: Submit TX<br/>(no trace context)
-
-    Note over NodeA: Creates new trace<br/>trace_id: abc123<br/>span: tx.receive
-
-    NodeA->>NodeB: Relay TX<br/>(trace_id: abc123, parent: 001)
-
-    Note over NodeB: Creates child span<br/>span: tx.relay<br/>parent_span_id: 001
-
-    NodeA->>NodeC: Relay TX<br/>(trace_id: abc123, parent: 001)
-
-    Note over NodeC: Creates child span<br/>span: tx.relay<br/>parent_span_id: 001
-
-    Note over NodeA,NodeC: All spans share trace_id: abc123<br/>enabling correlation across nodes
-```
-
-**Reading the diagram:**
-
-- **Client**: The external entity that submits a transaction. It does not carry trace context -- the trace originates at the first node.
-- **Node A**: The entry point that creates a new trace (trace_id: abc123) and the root span `tx.receive`. It relays the transaction to peers with trace context attached.
-- **Node B and Node C**: Peer nodes that receive the relayed transaction along with the propagated trace context. Each creates a child span under Node A's span, preserving the same `trace_id`.
-- **Arrows with trace context**: The relay messages carry `trace_id` and `parent_span_id`, allowing each downstream node to link its spans back to the originating span on Node A.
-
----
-
-## Context Propagation
-
-For traces to work across nodes, **trace context must be propagated** in messages.
-
-### What's in the Context (~26 bytes)
-
-| Field         | Size     | Description                                             |
-| ------------- | -------- | ------------------------------------------------------- |
-| `trace_id`    | 16 bytes | Identifies the entire trace (constant across all nodes) |
-| `span_id`     | 8 bytes  | The sender's current span (becomes parent on receiver)  |
-| `trace_flags` | 1 byte   | Sampling decision (bit 0 = sampled; bits 1-7 reserved)  |
-| `trace_state` | variable | Optional vendor-specific data (typically omitted)       |
-
-### How span_id Changes at Each Hop
-
-Only **one** `span_id` travels in the context - the sender's current span. Each node:
-
-1. Extracts the received `span_id` and uses it as the `parent_span_id`
-2. Creates a **new** `span_id` for its own span
-3. Sends its own `span_id` as the parent when forwarding
-
-```
-Node A                      Node B                      Node C
-──────                      ──────                      ──────
-
-Span AAA                    Span BBB                    Span CCC
-   │                           │                           │
-   ▼                           ▼                           ▼
-Context out:                Context out:                Context out:
-├─ trace_id: abc123         ├─ trace_id: abc123         ├─ trace_id: abc123
-├─ span_id: AAA ──────────► ├─ span_id: BBB ──────────► ├─ span_id: CCC ──────►
-└─ flags: 01                └─ flags: 01                └─ flags: 01
-                               │                           │
-                          parent = AAA               parent = BBB
-```
-
-The `trace_id` stays constant, but `span_id` **changes at every hop** to maintain the parent-child chain.
-
-### Propagation Formats
-
-There are two patterns:
-
-### HTTP/RPC Headers (W3C Trace Context)
-
-```
-traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01
-             │  │                                │                │
-             │  │                                │                └── Flags (sampled)
-             │  │                                └── Parent span ID (16 hex)
-             │  └── Trace ID (32 hex)
-             └── Version
-```
-
-### Protocol Buffers (xrpld P2P messages)
-
-xrpld P2P messages such as `TMTransaction` carry the trace context in two added byte fields alongside the existing payload: `trace_parent` holds the W3C traceparent (`trace_id`, `span_id`, and `trace_flags`), and `trace_state` holds the optional W3C tracestate. Together they propagate the trace across the P2P boundary so a receiving node can attach its spans to the sender's span.
-
----
-
-## Sampling
-
-Not every trace needs to be recorded. **Sampling** reduces overhead:
-
-### Head Sampling (at trace start)
-
-```
-Request arrives → Random N% chance → Record or skip entire trace
-```
-
-- ✅ Low overhead
-- ❌ May miss interesting traces
-
-> **xrpld note**: xrpld intentionally fixes head sampling at 100% (sample
-> everything) and does not expose a configurable ratio. A per-node ratio
-> would let different nodes make divergent keep/drop decisions for the same
-> distributed trace, producing broken/partial traces. A span with a remote
-> parent is decided by the same trace-id ratio sampler, not by the peer's
-> sampled flag, so nodes agree on every trace. Volume reduction is
-> delegated to collector-side tail sampling.
-
-### Tail Sampling (after trace completes)
-
-```
-Trace completes → Collector evaluates:
-                  - Error? → KEEP
-                  - Slow? → KEEP
-                  - Normal? → Sample 10%
-```
-
-- ✅ Never loses important traces
-- ❌ Higher memory usage at collector
-
----
-
-## Key Benefits for xrpld
-
-| Challenge                          | How Tracing Helps                        |
-| ---------------------------------- | ---------------------------------------- |
-| "Where is my transaction?"         | Follow trace across all nodes it touched |
-| "Why was consensus slow?"          | See timing breakdown of each phase       |
-| "Which node is the bottleneck?"    | Compare span durations across nodes      |
-| "What happened during the outage?" | Correlate errors across the network      |
-
----
-
-## Glossary
-
-| Term                 | Definition                                                          |
-| -------------------- | ------------------------------------------------------------------- |
-| **Trace**            | Complete journey of a request, identified by `trace_id`             |
-| **Span**             | Single operation within a trace                                     |
-| **Parent-Child**     | Span relationship where the parent depends on the child             |
-| **Follows-From**     | Causal relationship where originator doesn't wait for the result    |
-| **Span Link**        | Non-hierarchical connection between spans, possibly across traces   |
-| **Deterministic ID** | Trace ID derived from domain data (e.g., tx_hash) instead of random |
-| **Context**          | Data propagated between services (`trace_id`, `span_id`, flags)     |
-| **Instrumentation**  | Code that creates spans and propagates context                      |
-| **Collector**        | Service that receives, processes, and exports traces                |
-| **Backend**          | Storage/visualization system (Tempo)                                |
-| **Head Sampling**    | Sampling decision at trace start                                    |
-| **Tail Sampling**    | Sampling decision after trace completes                             |
-
----
-
-_Next: [Architecture Analysis](./01-architecture-analysis.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_

OpenTelemetryPlan/01-architecture-analysis.md

@@ -1,467 +0,0 @@
-# Architecture Analysis
-
-> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md)
-> **Related**: [Design Decisions](./02-design-decisions.md) | [Implementation Strategy](./03-implementation-strategy.md)
-
----
-
-## 1.1 Current xrpld Architecture Overview
-
-> **WS** = WebSocket | **UNL** = Unique Node List | **TxQ** = Transaction Queue | **StatsD** = Statistics Daemon
-
-The xrpld node software consists of several interconnected components that need instrumentation for distributed tracing:
-
-```mermaid
-flowchart TB
-    subgraph xrpld["xrpld Node"]
-        subgraph services["Core Services"]
-            RPC["RPC Server<br/>(HTTP/WS/gRPC)"]
-            Overlay["Overlay<br/>(P2P Network)"]
-            Consensus["Consensus<br/>(RCLConsensus)"]
-            ValidatorList["ValidatorList<br/>(UNL Mgmt)"]
-        end
-
-        JobQueue["JobQueue<br/>(Thread Pool)"]
-
-        subgraph processing["Processing Layer"]
-            NetworkOPs["NetworkOPs<br/>(Tx Processing)"]
-            LedgerMaster["LedgerMaster<br/>(Ledger Mgmt)"]
-            NodeStore["NodeStore<br/>(Database)"]
-            InboundLedgers["InboundLedgers<br/>(Ledger Sync)"]
-        end
-
-        subgraph appservices["Application Services"]
-            PathFind["PathFinding<br/>(Payment Paths)"]
-            TxQ["TxQ<br/>(Fee Escalation)"]
-            LoadMgr["LoadManager<br/>(Fee/Load)"]
-        end
-
-        subgraph observability["Existing Observability"]
-            PerfLog["PerfLog<br/>(JSON)"]
-            Insight["Insight<br/>(StatsD)"]
-            Logging["Logging<br/>(Journal)"]
-        end
-
-        services --> JobQueue
-        JobQueue --> processing
-        JobQueue --> appservices
-    end
-
-    style xrpld fill:#424242,stroke:#212121,color:#ffffff
-    style services fill:#1565c0,stroke:#0d47a1,color:#ffffff
-    style processing fill:#2e7d32,stroke:#1b5e20,color:#ffffff
-    style appservices fill:#6a1b9a,stroke:#4a148c,color:#ffffff
-    style observability fill:#e65100,stroke:#bf360c,color:#ffffff
-```
-
-**Reading the diagram:**
-
-- **Core Services (blue)**: The entry points into xrpld -- RPC Server handles client requests, Overlay manages peer-to-peer networking, Consensus drives agreement, and ValidatorList manages trusted validators.
-- **JobQueue (center)**: The asynchronous thread pool that decouples Core Services from the Processing and Application layers. All work flows through it.
-- **Processing Layer (green)**: Core business logic -- NetworkOPs processes transactions, LedgerMaster manages ledger state, NodeStore handles persistence, and InboundLedgers synchronizes missing data.
-- **Application Services (purple)**: Higher-level features -- PathFinding computes payment routes, TxQ manages fee-based queuing, and LoadManager tracks server load.
-- **Existing Observability (orange)**: The current monitoring stack (PerfLog, Insight, Journal logging) that OpenTelemetry will complement, not replace.
-- **Arrows (Services to JobQueue to layers)**: Work originates at Core Services, is enqueued onto the JobQueue, and dispatched to Processing or Application layers for execution.
-
----
-
-## 1.1.1 Actors and Actions
-
-### Actors
-
-| Who (Plain English)                       | Technical Term             |
-| ----------------------------------------- | -------------------------- |
-| Network node running XRPL software        | xrpld node                 |
-| External client submitting requests       | RPC Client                 |
-| Network neighbor sharing data             | Peer (PeerImp)             |
-| Request handler for client queries        | RPC Server (ServerHandler) |
-| Command executor for specific RPC methods | RPCHandler                 |
-| Agreement process between nodes           | Consensus (RCLConsensus)   |
-| Transaction processing coordinator        | NetworkOPs                 |
-| Background task scheduler                 | JobQueue                   |
-| Ledger state manager                      | LedgerMaster               |
-| Payment route calculator                  | PathFinding (Pathfinder)   |
-| Transaction waiting room                  | TxQ (Transaction Queue)    |
-| Fee adjustment system                     | LoadManager                |
-| Trusted validator list manager            | ValidatorList              |
-| Protocol upgrade tracker                  | AmendmentTable             |
-| Ledger state hash tree                    | SHAMap                     |
-| Persistent key-value storage              | NodeStore                  |
-
-### Actions
-
-| What Happens (Plain English)                   | Technical Term         |
-| ---------------------------------------------- | ---------------------- |
-| Client sends a request to a node               | `rpc.request`          |
-| Node executes a specific RPC command           | `rpc.command.*`        |
-| Node receives a transaction from a peer        | `tx.receive`           |
-| Node checks if a transaction is valid          | `tx.validate`          |
-| Node forwards a transaction to neighbors       | `tx.relay`             |
-| Nodes agree on which transactions to include   | `consensus.round`      |
-| Consensus progresses through phases            | `consensus.phase.*`    |
-| Node builds a new confirmed ledger             | `ledger.build`         |
-| Node fetches missing ledger data from peers    | `ledger.acquire`       |
-| Node computes payment routes                   | `pathfind.compute`     |
-| Node queues a transaction for later processing | `txq.enqueue`          |
-| Node increases fees due to high load           | `fee.escalate`         |
-| Node fetches the latest trusted validator list | `validator.list.fetch` |
-| Node votes on a protocol amendment             | `amendment.vote`       |
-| Node synchronizes state tree data              | `shamap.sync`          |
-
----
-
-## 1.2 Key Components for Instrumentation
-
-> **TxQ** = Transaction Queue | **UNL** = Unique Node List
-
-| Component          | Location                                   | Purpose                  | Trace Value                      |
-| ------------------ | ------------------------------------------ | ------------------------ | -------------------------------- |
-| **Overlay**        | `src/xrpld/overlay/`                       | P2P communication        | Message propagation timing       |
-| **PeerImp**        | `src/xrpld/overlay/detail/PeerImp.cpp`     | Individual peer handling | Per-peer latency                 |
-| **RCLConsensus**   | `src/xrpld/app/consensus/RCLConsensus.cpp` | Consensus algorithm      | Round timing, phase analysis     |
-| **NetworkOPs**     | `src/xrpld/app/misc/NetworkOPs.cpp`        | Transaction processing   | Tx lifecycle tracking            |
-| **ServerHandler**  | `src/xrpld/rpc/detail/ServerHandler.cpp`   | RPC entry point          | Request latency                  |
-| **RPCHandler**     | `src/xrpld/rpc/detail/RPCHandler.cpp`      | Command execution        | Per-command timing               |
-| **JobQueue**       | `src/xrpl/core/JobQueue.h`                 | Async task execution     | Queue wait times                 |
-| **PathFinding**    | `src/xrpld/app/paths/`                     | Payment path computation | Path latency, cache hits         |
-| **TxQ**            | `src/xrpld/app/misc/TxQ.cpp`               | Transaction queue/fees   | Queue depth, eviction rates      |
-| **LoadManager**    | `src/xrpld/app/main/LoadManager.cpp`       | Fee escalation/load      | Fee levels, load factors         |
-| **InboundLedgers** | `src/xrpld/app/ledger/InboundLedgers.cpp`  | Ledger acquisition       | Sync time, peer reliability      |
-| **ValidatorList**  | `src/xrpld/app/misc/ValidatorList.cpp`     | UNL management           | List freshness, fetch failures   |
-| **AmendmentTable** | `src/xrpld/app/misc/AmendmentTable.cpp`    | Protocol amendments      | Voting status, activation events |
-| **SHAMap**         | `src/xrpld/shamap/`                        | State hash tree          | Sync speed, missing nodes        |
-
----
-
-## 1.3 Transaction Flow Diagram
-
-Transaction flow spans multiple nodes in the network. Each node creates linked spans to form a distributed trace:
-
-```mermaid
-sequenceDiagram
-    participant Client
-    participant PeerA as Peer A (Receive)
-    participant PeerB as Peer B (Relay)
-    participant PeerC as Peer C (Validate)
-
-    Client->>PeerA: 1. Submit TX
-
-    rect rgb(230, 245, 255)
-        Note over PeerA: tx.receive SPAN START
-        PeerA->>PeerA: HashRouter Deduplication
-        PeerA->>PeerA: tx.validate (child span)
-    end
-
-    PeerA->>PeerB: 2. Relay TX (with trace ctx)
-
-    rect rgb(230, 245, 255)
-        Note over PeerB: tx.receive (linked span)
-    end
-
-    PeerB->>PeerC: 3. Relay TX
-
-    rect rgb(230, 245, 255)
-        Note over PeerC: tx.receive (linked span)
-        PeerC->>PeerC: tx.process
-    end
-
-    Note over Client,PeerC: DISTRIBUTED TRACE (same trace_id: abc123)
-```
-
-**Reading the diagram:**
-
-- **Client**: The external entity that submits a transaction to Peer A. It has no trace context -- the trace starts at the first node.
-- **Peer A (Receive)**: The entry node that creates the root span `tx.receive`, runs HashRouter deduplication to avoid processing duplicates, and creates a child `tx.validate` span.
-- **Peer A to Peer B arrow**: The relay message carries trace context (trace_id + parent span_id), enabling Peer B to create a linked span under the same trace.
-- **Peer B (Relay)**: Receives the transaction and trace context, creates a `tx.receive` span linked to Peer A's trace, then relays onward.
-- **Peer C (Validate)**: Final hop in this example. Creates a linked `tx.receive` span and runs `tx.process` to fully process the transaction.
-- **Blue rectangles**: Highlight the span boundaries on each node, showing where instrumentation creates and closes spans.
-
-### Trace Structure
-
-```
-trace_id: abc123
-├── span: tx.receive (Peer A)
-│   ├── span: tx.validate
-│   └── span: tx.relay
-├── span: tx.receive (Peer B) [parent: Peer A]
-│   └── span: tx.relay
-└── span: tx.receive (Peer C) [parent: Peer B]
-    └── span: tx.process
-```
-
----
-
-## 1.4 Consensus Round Flow
-
-Consensus rounds are multi-phase operations that benefit significantly from tracing:
-
-```mermaid
-flowchart TB
-    subgraph round["consensus.round (root span)"]
-        attrs["Attributes:<br/>ledger_seq = 12345678<br/>consensus_mode = proposing<br/>proposers = 35"]
-
-        subgraph open["consensus.phase.open"]
-            open_desc["Duration: ~3s<br/>Waiting for transactions"]
-        end
-
-        subgraph establish["consensus.phase.establish"]
-            est_attrs["proposals_received = 28<br/>disputes_resolved = 3"]
-            est_children["├── consensus.proposal.receive (×28)<br/>├── consensus.proposal.send (×1)<br/>└── consensus.dispute.resolve (×3)"]
-        end
-
-        subgraph accept["consensus.phase.accept"]
-            acc_attrs["transactions_applied = 150<br/>ledger_hash = DEF456..."]
-            acc_children["├── ledger.build<br/>└── ledger.validate"]
-        end
-
-        attrs --> open
-        open --> establish
-        establish --> accept
-    end
-
-    style round fill:#f57f17,stroke:#e65100,color:#ffffff
-    style open fill:#1565c0,stroke:#0d47a1,color:#ffffff
-    style establish fill:#2e7d32,stroke:#1b5e20,color:#ffffff
-    style accept fill:#c2185b,stroke:#880e4f,color:#ffffff
-```
-
-**Reading the diagram:**
-
-- **consensus.round (orange, root span)**: The top-level span encompassing the entire consensus round, with attributes like ledger sequence, mode, and proposer count.
-- **consensus.phase.open (blue)**: The first phase where the node waits (~3s) to collect incoming transactions before proposing.
-- **consensus.phase.establish (green)**: The negotiation phase where validators exchange proposals, resolve disputes, and converge on a transaction set. Child spans track each proposal received/sent and each dispute resolved.
-- **consensus.phase.accept (pink)**: The final phase where the agreed transaction set is applied, a new ledger is built, and the ledger is validated. Child spans cover `ledger.build` and `ledger.validate`.
-- **Arrows (open to establish to accept)**: The sequential flow through the three consensus phases. Each phase must complete before the next begins.
-
----
-
-## 1.5 RPC Request Flow
-
-> **WS** = WebSocket
-
-RPC requests support W3C Trace Context headers for distributed tracing across services:
-
-```mermaid
-flowchart TB
-    subgraph request["rpc.request (root span)"]
-        http["HTTP Request — POST /<br/>traceparent:<br/>00-abc123...-def456...-01"]
-
-        attrs["Attributes:<br/>http.method = POST<br/>net.peer.ip = 192.168.1.100<br/>command = submit"]
-
-        subgraph enqueue["jobqueue.enqueue"]
-            job_attr["job_type = jtCLIENT_RPC"]
-        end
-
-        subgraph command["rpc.command.submit"]
-            cmd_attrs["version = 2<br/>rpc_role = user"]
-            cmd_children["├── tx.deserialize<br/>├── tx.validate_local<br/>└── tx.submit_to_network"]
-        end
-
-        response["Response: 200 OK<br/>Duration: 45ms"]
-
-        http --> attrs
-        attrs --> enqueue
-        enqueue --> command
-        command --> response
-    end
-
-    style request fill:#2e7d32,stroke:#1b5e20,color:#ffffff
-    style enqueue fill:#1565c0,stroke:#0d47a1,color:#ffffff
-    style command fill:#e65100,stroke:#bf360c,color:#ffffff
-```
-
-**Reading the diagram:**
-
-- **rpc.request (green, root span)**: The outermost span representing the full RPC request lifecycle, from HTTP receipt to response. Carries the W3C `traceparent` header for distributed tracing.
-- **HTTP Request node**: Shows the incoming POST request with its `traceparent` header and extracted attributes (method, peer IP, command name).
-- **jobqueue.enqueue (blue)**: The span covering the asynchronous handoff from the RPC thread to the JobQueue worker thread. The trace context is preserved across this async boundary.
-- **rpc.command.submit (orange)**: The span for the actual command execution, with child spans for deserialization, local validation, and network submission.
-- **Response node**: The final output with HTTP status and total duration, marking the end of the root span.
-- **Arrows (top to bottom)**: The sequential processing pipeline -- receive request, extract attributes, enqueue job, execute command, return response.
-
----
-
-## 1.6 Key Trace Points
-
-> **TxQ** = Transaction Queue
-
-The following table identifies priority instrumentation points across the codebase:
-
-| Category        | Span Name              | File                   | Method                  | Priority |
-| --------------- | ---------------------- | ---------------------- | ----------------------- | -------- |
-| **Transaction** | `tx.receive`           | `PeerImp.cpp`          | `handleTransaction()`   | High     |
-| **Transaction** | `tx.validate`          | `NetworkOPs.cpp`       | `processTransaction()`  | High     |
-| **Transaction** | `tx.process`           | `NetworkOPs.cpp`       | `doTransactionSync()`   | High     |
-| **Transaction** | `tx.relay`             | `OverlayImpl.cpp`      | `relay()`               | Medium   |
-| **Consensus**   | `consensus.round`      | `RCLConsensus.cpp`     | `startRound()`          | High     |
-| **Consensus**   | `consensus.phase.*`    | `Consensus.h`          | `timerEntry()`          | High     |
-| **Consensus**   | `consensus.proposal.*` | `RCLConsensus.cpp`     | `peerProposal()`        | Medium   |
-| **RPC**         | `rpc.request`          | `ServerHandler.cpp`    | `onRequest()`           | High     |
-| **RPC**         | `rpc.command.*`        | `RPCHandler.cpp`       | `doCommand()`           | High     |
-| **Peer**        | `peer.connect`         | `OverlayImpl.cpp`      | `onHandoff()`           | Low      |
-| **Peer**        | `peer.message.*`       | `PeerImp.cpp`          | `onMessage()`           | Low      |
-| **Ledger**      | `ledger.acquire`       | `InboundLedgers.cpp`   | `acquire()`             | Medium   |
-| **Ledger**      | `ledger.build`         | `RCLConsensus.cpp`     | `buildLCL()`            | High     |
-| **PathFinding** | `pathfind.request`     | `PathRequest.cpp`      | `doUpdate()`            | High     |
-| **PathFinding** | `pathfind.compute`     | `Pathfinder.cpp`       | `findPaths()`           | High     |
-| **TxQ**         | `txq.enqueue`          | `TxQ.cpp`              | `apply()`               | High     |
-| **TxQ**         | `txq.apply`            | `TxQ.cpp`              | `processClosedLedger()` | High     |
-| **Fee**         | `fee.escalate`         | `LoadManager.cpp`      | `raiseLocalFee()`       | Medium   |
-| **Ledger**      | `ledger.replay`        | `LedgerReplayer.h`     | `replay()`              | Medium   |
-| **Ledger**      | `ledger.delta`         | `LedgerDeltaAcquire.h` | `processData()`         | Medium   |
-| **Validator**   | `validator.list.fetch` | `ValidatorList.cpp`    | `verify()`              | Medium   |
-| **Validator**   | `validator.manifest`   | `Manifest.cpp`         | `applyManifest()`       | Low      |
-| **Amendment**   | `amendment.vote`       | `AmendmentTable.cpp`   | `doVoting()`            | Low      |
-| **SHAMap**      | `shamap.sync`          | `SHAMap.cpp`           | `fetchRoot()`           | Medium   |
-
----
-
-## 1.7 Instrumentation Priority
-
-> **TxQ** = Transaction Queue
-
-```mermaid
-quadrantChart
-    title Instrumentation Priority Matrix
-    x-axis Low Complexity --> High Complexity
-    y-axis Low Value --> High Value
-    quadrant-1 Implement First
-    quadrant-2 Plan Carefully
-    quadrant-3 Quick Wins
-    quadrant-4 Consider Later
-
-    RPC Tracing: [0.2, 0.92]
-    Transaction Tracing: [0.55, 0.88]
-    Consensus Tracing: [0.78, 0.82]
-    PathFinding: [0.38, 0.75]
-    TxQ and Fees: [0.25, 0.65]
-    Ledger Sync: [0.62, 0.58]
-    Peer Message Tracing: [0.35, 0.25]
-    JobQueue Tracing: [0.2, 0.48]
-    Validator Mgmt: [0.48, 0.42]
-    Amendment Tracking: [0.15, 0.32]
-    SHAMap Operations: [0.72, 0.45]
-```
-
----
-
-## 1.8 Observable Outcomes
-
-> **TxQ** = Transaction Queue | **UNL** = Unique Node List
-
-After implementing OpenTelemetry, operators and developers will gain visibility into the following:
-
-### 1.8.1 What You Will See: Traces
-
-| Trace Type                 | Description                                                                                 | Example Query in Grafana/Tempo                  |
-| -------------------------- | ------------------------------------------------------------------------------------------- | ----------------------------------------------- |
-| **Transaction Lifecycle**  | Full journey from RPC submission through validation, relay, consensus, and ledger inclusion | `{service.name="xrpld" && tx_hash="ABC123..."}` |
-| **Cross-Node Propagation** | Transaction path across multiple xrpld nodes with timing                                    | `{relay_count > 0}`                             |
-| **Consensus Rounds**       | Complete round with all phases (open, establish, accept)                                    | `{span.name=~"consensus.round.*"}`              |
-| **RPC Request Processing** | Individual command execution with timing breakdown                                          | `{command="account_info"}`                      |
-| **Ledger Acquisition**     | Peer-to-peer ledger data requests and responses                                             | `{span.name="ledger.acquire"}`                  |
-| **PathFinding Latency**    | Path computation time and cache effectiveness for payment RPCs                              | `{span.name="pathfind.compute"}`                |
-| **TxQ Behavior**           | Queue depth, eviction patterns, fee escalation during congestion                            | `{span.name=~"txq.*"}`                          |
-| **Ledger Sync**            | Full acquisition timeline including delta and transaction fetches                           | `{span.name=~"ledger.acquire.*"}`               |
-| **Validator Health**       | UNL fetch success, manifest updates, stale list detection                                   | `{span.name=~"validator.*"}`                    |
-
-### 1.8.2 What You Will See: Metrics (Derived from Traces)
-
-| Metric                        | Description                             | Dashboard Panel             |
-| ----------------------------- | --------------------------------------- | --------------------------- |
-| **RPC Latency (p50/p95/p99)** | Response time distribution per command  | Heatmap by command          |
-| **Transaction Throughput**    | Transactions processed per second       | Time series graph           |
-| **Consensus Round Duration**  | Time to complete consensus phases       | Histogram                   |
-| **Cross-Node Latency**        | Time for transaction to reach N nodes   | Line chart with percentiles |
-| **Error Rate**                | Failed transactions/RPC calls by type   | Stacked bar chart           |
-| **PathFinding Latency**       | Path computation time per currency pair | Heatmap by currency         |
-| **TxQ Depth**                 | Queued transactions over time           | Time series with thresholds |
-| **Fee Escalation Level**      | Current fee multiplier                  | Gauge with alert thresholds |
-| **Ledger Sync Duration**      | Time to acquire missing ledgers         | Histogram                   |
-
-### 1.8.3 Concrete Dashboard Examples
-
-**Transaction Trace View (Tempo):**
-
-```
-┌────────────────────────────────────────────────────────────────────────────────┐
-│ Trace: abc123... (Transaction Submission)                    Duration: 847ms   │
-├────────────────────────────────────────────────────────────────────────────────┤
-│ ├── rpc.request [ServerHandler]                              ████░░░░░░  45ms  │
-│ │   └── rpc.command.submit [RPCHandler]                      ████░░░░░░  42ms  │
-│ │       └── tx.receive [NetworkOPs]                          ███░░░░░░░  35ms  │
-│ │           ├── tx.validate [TxQ]                            █░░░░░░░░░   8ms  │
-│ │           └── tx.relay [Overlay]                           ██░░░░░░░░  15ms  │
-│ │               ├── tx.receive [Node-B]                      █████░░░░░  52ms  │
-│ │               │   └── tx.relay [Node-B]                    ██░░░░░░░░  18ms  │
-│ │               └── tx.receive [Node-C]                      ██████░░░░  65ms  │
-│ └── consensus.round [RCLConsensus]                           ████████░░ 720ms  │
-│     ├── consensus.phase.open                                 ██░░░░░░░░ 180ms  │
-│     ├── consensus.phase.establish                            █████░░░░░ 480ms  │
-│     └── consensus.phase.accept                               █░░░░░░░░░  60ms  │
-└────────────────────────────────────────────────────────────────────────────────┘
-```
-
-**RPC Performance Dashboard Panel:**
-
-```
-┌─────────────────────────────────────────────────────────────┐
-│ RPC Command Latency (Last 1 Hour)                           │
-├─────────────────────────────────────────────────────────────┤
-│ Command          │ p50    │ p95    │ p99    │ Errors │ Rate │
-│──────────────────┼────────┼────────┼────────┼────────┼──────│
-│ account_info     │  12ms  │  45ms  │  89ms  │  0.1%  │ 150/s│
-│ submit           │  35ms  │ 120ms  │ 250ms  │  2.3%  │  45/s│
-│ ledger           │   8ms  │  25ms  │  55ms  │  0.0%  │  80/s│
-│ tx               │  15ms  │  50ms  │ 100ms  │  0.5%  │  60/s│
-│ server_info      │   5ms  │  12ms  │  20ms  │  0.0%  │ 200/s│
-└─────────────────────────────────────────────────────────────┘
-```
-
-**Consensus Health Dashboard Panel:**
-
-```mermaid
----
-config:
-    xyChart:
-        width: 1200
-        height: 400
-        plotReservedSpacePercent: 50
-        chartOrientation: vertical
-    themeVariables:
-        xyChart:
-            plotColorPalette: "#3498db"
----
-xychart-beta
-    title "Consensus Round Duration (Last 24 Hours)"
-    x-axis "Time of Day (Hours)" [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24]
-    y-axis "Duration (seconds)" 1 --> 5
-    line [2.1, 2.4, 2.8, 3.2, 3.8, 4.3, 4.5, 5.0, 4.7, 4.0, 3.2, 2.6, 2.0]
-```
-
-### 1.8.4 Operator Actionable Insights
-
-| Scenario                  | What You'll See                                                              | Action                                           |
-| ------------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------ |
-| **Slow RPC**              | Span showing which phase is slow (parsing, execution, serialization)         | Optimize specific code path                      |
-| **Transaction Stuck**     | Trace stops at validation; error attribute shows reason                      | Fix transaction parameters                       |
-| **Consensus Delay**       | Phase.establish taking too long; proposer attribute shows missing validators | Investigate network connectivity                 |
-| **Memory Spike**          | Large batch of spans correlating with memory increase                        | Tune batch_size or sampling                      |
-| **Network Partition**     | Traces missing cross-node links for specific peer                            | Check peer connectivity                          |
-| **Path Computation Slow** | pathfind.compute span shows high latency; cache miss rate in attributes      | Warm the RippleLineCache, check order book depth |
-| **TxQ Full**              | txq.enqueue spans show evictions; fee.escalate spans increasing              | Monitor fee levels, alert operators              |
-| **Ledger Sync Stalled**   | ledger.acquire spans timing out; peer reliability attributes show issues     | Check peer connectivity, add trusted peers       |
-| **UNL Stale**             | validator.list.fetch spans failing; last_update attribute aging              | Verify validator site URLs, check DNS            |
-
-### 1.8.5 Developer Debugging Workflow
-
-1. **Find Transaction**: Query by `tx_hash` to get full trace
-2. **Identify Bottleneck**: Look at span durations to find slowest component
-3. **Check Attributes**: Review `validity`, `rpc_status` for errors
-4. **Correlate Logs**: Use `trace_id` to find related PerfLog entries
-5. **Compare Nodes**: Filter by `service.instance.id` to compare behavior across nodes
-
----
-
-_Next: [Design Decisions](./02-design-decisions.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_

OpenTelemetryPlan/02-design-decisions.md

@@ -1,881 +0,0 @@
-# Design Decisions
-
-> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md)
-> **Related**: [Architecture Analysis](./01-architecture-analysis.md)
-
----
-
-## 2.1 OpenTelemetry Components
-
-> **OTLP** = OpenTelemetry Protocol
-
-### 2.1.1 SDK Selection
-
-**Primary Choice**: OpenTelemetry C++ SDK (`opentelemetry-cpp`)
-
-| Component                               | Purpose                | Required                  |
-| --------------------------------------- | ---------------------- | ------------------------- |
-| `opentelemetry-cpp::api`                | Tracing API headers    | Yes                       |
-| `opentelemetry-cpp::sdk`                | SDK implementation     | Yes                       |
-| `opentelemetry-cpp::ext`                | Extensions (exporters) | Yes                       |
-| `opentelemetry-cpp::otlp_http_exporter` | OTLP/HTTP export       | Yes (shipped in Phase 1b) |
-| `opentelemetry-cpp::otlp_grpc_exporter` | OTLP/gRPC export       | Future (not yet wired up) |
-
-### 2.1.2 Instrumentation Strategy
-
-**Manual Instrumentation** (recommended):
-
-| Approach   | Pros                                                            | Cons                                                    |
-| ---------- | --------------------------------------------------------------- | ------------------------------------------------------- |
-| **Manual** | Precise control, optimized placement, xrpld-specific attributes | More development effort                                 |
-| **Auto**   | Less code, automatic coverage                                   | Less control, potential overhead, limited customization |
-
----
-
-## 2.2 Exporter Configuration
-
-> **OTLP** = OpenTelemetry Protocol
-
-```mermaid
-flowchart TB
-    subgraph nodes["xrpld Nodes"]
-        node1["xrpld<br/>Node 1"]
-        node2["xrpld<br/>Node 2"]
-        node3["xrpld<br/>Node 3"]
-    end
-
-    collector["OpenTelemetry<br/>Collector<br/>(sidecar or standalone)"]
-
-    subgraph backends["Observability Backends"]
-        tempo["Tempo"]
-        elastic["Elastic<br/>APM"]
-    end
-
-    node1 -->|"OTLP/HTTP<br/>:4318"| collector
-    node2 -->|"OTLP/HTTP<br/>:4318"| collector
-    node3 -->|"OTLP/HTTP<br/>:4318"| collector
-
-    collector --> tempo
-    collector --> elastic
-
-    style nodes fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style backends fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style collector fill:#bf360c,stroke:#8c2809,color:#ffffff
-```
-
-**Reading the diagram:**
-
-- **xrpld Nodes (blue)**: The source of telemetry data. Each xrpld node exports spans via OTLP/HTTP on port 4318 (the only exporter shipped in Phase 1b).
-- **OpenTelemetry Collector (red)**: The central aggregation point that receives spans from all nodes. Can run as a sidecar (per-node) or standalone (shared). Handles batching, filtering, and routing.
-- **Observability Backends (green)**: The storage and visualization destinations. Tempo is the recommended backend for both development and production, and Elastic APM is an alternative. The Collector routes to one or more backends.
-- **Arrows (nodes to collector to backends)**: The data pipeline -- spans flow from nodes to the Collector over HTTP, then the Collector fans out to the configured backends.
-
-### 2.2.1 OTLP/HTTP (Shipped in Phase 1b)
-
-OTLP/HTTP is the only exporter wired up in Phase 1b. It is configured via
-`OtlpHttpExporterOptions` with the collector traces endpoint
-(`http://localhost:4318/v1/traces` by default) and a JSON content type
-(binary protobuf is also available).
-
-### 2.2.2 OTLP/gRPC (Future Work — Planned Upgrade)
-
-OTLP/gRPC is planned as a future upgrade from the HTTP exporter. The gRPC
-transport offers lower per-span overhead and tighter back-pressure semantics
-than HTTP/JSON, making it attractive for production deployments once the HTTP
-path is validated in earlier phases.
-
-Required to land this upgrade:
-
-1. Add `opentelemetry-cpp::otlp_grpc_exporter` to the Conan recipe (the
-   dependency already exists but is not linked in Phase 1b builds).
-2. Extend `TelemetryConfig.cpp` to parse an `exporter` key (`otlp_http`
-   default, `otlp_grpc` opt-in) and a gRPC endpoint override.
-3. In `Telemetry::start()` branch on the parsed exporter type and construct
-   either `OtlpHttpExporterFactory::Create(httpOpts)` or
-   `OtlpGrpcExporterFactory::Create(grpcOpts)` accordingly.
-4. Update the runbook and dashboards to document the alternate port and TLS
-   settings.
-
-When wired up, the gRPC path will use `OtlpGrpcExporterOptions` configured with
-the collector endpoint (host on port 4317), TLS credentials enabled, and a CA
-certificate path.
-
-Until that work lands, `OtlpGrpcExporterOptions` is **not** used by any code
-path in Phase 1b through Phase 5.
-
----
-
-## 2.3 Span Naming Conventions
-
-> **TxQ** = Transaction Queue | **UNL** = Unique Node List | **WS** = WebSocket
-
-### 2.3.1 Naming Schema
-
-```
-<component>.<operation>[.<sub-operation>]
-```
-
-**Examples**:
-
-- `tx.receive` - Transaction received from peer
-- `consensus.phase.establish` - Consensus establish phase
-- `rpc.command.server_info` - server_info RPC command
-
-### 2.3.2 Complete Span Catalog
-
-> **Status column.** This catalog is the design inventory; it is not a
-> statement of what currently emits. `Live` means the span is present in the
-> implemented inventory ([09-data-collection-reference.md §1.1](./09-data-collection-reference.md#11-complete-span-inventory)),
-> which is the authoritative list. `Renamed`/`Split` means the concept shipped
-> under a different name than planned here. **Not built** means no span is
-> emitted for it today.
->
-> **"Not built" is not one thing.** All 14 such entries fall into three cases, and the
-> fourth column says which — filing them all as oversights would be wrong:
->
-> - **Superseded by metrics or logs (7)** — a deliberate trade-off: the signal is already
->   carried by a metric or by a log-derived panel, and a span would add per-event volume
->   without adding information. `tx.relay`, `fee.escalate`, `validator.list.fetch`,
->   `validator.manifest`, `shamap.sync`, `job.enqueue`, `job.execute`.
-> - **Gap (6)** — nothing was decided; they were simply never instrumented. The four
->   `peer.*` entries, plus `ledger.replay` and `ledger.delta` — and those last two are the
->   sharpest, because they have **no metric substitute at all**.
-> - **Deferred (1)** — scheduled work: `amendment.vote` (Phase 11).
->
-> The four `peer.*` entries are the peer-span coverage gap: only
-> `peer.proposal.receive` and `peer.validation.receive` exist, so protocol
-> message send/receive and connection lifecycle are untraced. See
-> [09 §6.4](./09-data-collection-reference.md#64-peer-span-coverage-gap-not-implemented).
->
-> `tx.validate` did ship, but renamed and split three ways: the apply pipeline
-> traces `tx.preflight` (stateless checks), `tx.preclaim` (ledger-state checks)
-> and `tx.transactor` (application), each stamped with a `stage` attribute.
-> Names come from `TxApplySpanNames.h:90,94,99`. The spans are created in two
-> different files, not one: `tx.preflight` and `tx.preclaim` come from
-> `applySteps.cpp` (`invokePreflight()` at `:211-212`, `invokePreclaim()` at
-> `:258-261`, both via the shared `makeStageSpan()` helper at `:89-126`), while
-> `tx.transactor` is created in `Transactor::operator()()`
-> (`Transactor.cpp:1601-1605`). Query them with
-> `name=~"tx\.(preflight|preclaim|transactor)"` — a **single** backslash; RE2
-> reads `\\.` as a literal backslash followed by any character, which matches
-> nothing here — never `name="tx.validate"`.
-
-| Span name                      | Description                             | Status                                                           | Why not built / where the signal lives instead                                                                                                                                                                                                                                                             |
-| ------------------------------ | --------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `tx.receive`                   | Transaction received from network       | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `tx.validate`                  | Transaction signature/format validation | Renamed + split → `tx.preflight`, `tx.preclaim`, `tx.transactor` | —                                                                                                                                                                                                                                                                                                          |
-| `tx.process`                   | Full transaction processing             | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `tx.relay`                     | Transaction relay to peers              | **Not built**                                                    | **Superseded by metrics.** Relay volume is carried by the overlay traffic counters (`total_bytes_in/out`, `total_messages_in/out`, per-`TrafficCount` category). Relay is also per-peer fan-out, so one span per relay multiplies by peer count for data the counters already aggregate.                   |
-| `tx.apply`                     | Apply transaction to ledger             | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `consensus.round`              | Complete consensus round                | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `consensus.phase.open`         | Open phase - collecting transactions    | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `consensus.phase.establish`    | Establish phase - reaching agreement    | Renamed `consensus.establish`                                    | —                                                                                                                                                                                                                                                                                                          |
-| `consensus.phase.accept`       | Accept phase - applying consensus       | Renamed `consensus.accept`                                       | —                                                                                                                                                                                                                                                                                                          |
-| `consensus.proposal.receive`   | Receive peer proposal                   | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `consensus.proposal.send`      | Send our proposal                       | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `consensus.validation.receive` | Receive peer validation                 | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `consensus.validation.send`    | Send our validation                     | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `rpc.request`                  | HTTP/WebSocket request handling         | Split into `rpc.http_request` / `rpc.ws_message`                 | —                                                                                                                                                                                                                                                                                                          |
-| `rpc.command.*`                | Specific RPC command (dynamic)          | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `peer.connect`                 | Peer connection establishment           | **Not built**                                                    | **Gap, scoped as its own change** — see [09 §6.4](./09-data-collection-reference.md#64-peer-span-coverage-gap-not-implemented). Adding these changes the span-family count and the entries in `expected_spans.json`.                                                                                       |
-| `peer.disconnect`              | Peer disconnection                      | **Not built**                                                    | **Gap.** Partially observable: the aggregate count via the `Overlay.Peer_Disconnects` insight gauge and resource-charge drops via `server_info{metric="peer_disconnects_resources"}`, but not per-reason. Disconnect reasons are only recoverable from `debug.log` (the `log-derived-insights` dashboard). |
-| `peer.message.send`            | Send protocol message                   | **Not built**                                                    | **Gap.** Of the 13 protocol message families only `mtGET_OBJECTS` has native instrumentation (`getobject_*`); byte/message volume is aggregated by `TrafficCount` category, not traced per message.                                                                                                        |
-| `peer.message.receive`         | Receive protocol message                | **Not built**                                                    | **Gap.** Same as `peer.message.send`.                                                                                                                                                                                                                                                                      |
-| `ledger.acquire`               | Ledger acquisition from network         | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `ledger.build`                 | Build new ledger                        | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `ledger.validate`              | Ledger validation                       | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `ledger.close`                 | Close ledger                            | Renamed `consensus.ledger_close`                                 | —                                                                                                                                                                                                                                                                                                          |
-| `ledger.replay`                | Ledger replay executed                  | **Not built**                                                    | **Gap, no substitute.** `LedgerReplayer.cpp` and `LedgerReplayTask.cpp` contain zero `SpanGuard` uses and no metric covers the replay path. A real hole, not a trade-off.                                                                                                                                  |
-| `ledger.delta`                 | Delta-based ledger acquired             | **Not built**                                                    | **Gap, no substitute.** `LedgerDeltaAcquire.cpp` contains zero `SpanGuard` uses. The `acquire_*` stats cover whole-ledger acquisition, not the delta path.                                                                                                                                                 |
-| `pathfind.request`             | Path request initiated                  | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `pathfind.compute`             | Path computation executed               | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `txq.enqueue`                  | Transaction queued                      | Live                                                             | —                                                                                                                                                                                                                                                                                                          |
-| `txq.apply`                    | Queued transaction applied              | Renamed `txq.apply_direct` / `txq.accept_tx`                     | —                                                                                                                                                                                                                                                                                                          |
-| `fee.escalate`                 | Fee escalation triggered                | **Not built**                                                    | **Superseded by metrics + existing spans.** Escalation state is `txq_metrics{metric=…}` and `load_factor_metrics{metric=…}`; the queueing path that triggers it is already traced by the six `txq.*` spans. An event span would restate a gauge.                                                           |
-| `validator.list.fetch`         | UNL list fetched                        | **Not built**                                                    | **Superseded by metrics.** `validator_health{metric="unl_expiry_days"}`, `{metric="unl_blocked"}` and `{metric="validation_quorum"}` carry the outcome. A fetch span would fire on a slow timer and tell an operator nothing the gauges do not.                                                            |
-| `validator.manifest`           | Manifest update processed               | **Not built**                                                    | **Superseded by logs.** Per-master-key manifest dispositions are on the `log-derived-insights` dashboard (`ManifestCache` partition, requires `log_level ManifestCache debug`).                                                                                                                            |
-| `amendment.vote`               | Amendment voting executed               | **Not built**                                                    | **Deferred to Phase 11.** `validator_health{metric="amendment_blocked"}` covers the blocked state in the meantime.                                                                                                                                                                                         |
-| `shamap.sync`                  | State tree synchronization              | **Not built**                                                    | **Superseded by metrics.** Covered by the nine `acquire_*` stats, `nodestore_state{metric=…}` and the five `getobject_*` families. Per-node-fetch spans would be prohibitive volume.                                                                                                                       |
-| `job.enqueue`                  | Job added to queue                      | **Not built**                                                    | **Superseded by metrics.** `job_queued_total` and `job_queued_us{job_type}` plus the 105 per-job-type `jobq_*` gauges. A span per enqueue is one span per unit of daemon work, for latency the histogram already records exactly.                                                                          |
-| `job.execute`                  | Job execution                           | **Not built**                                                    | **Superseded by metrics.** `job_started_total`, `job_finished_total`, `job_running_us{job_type}`. Same volume argument as `job.enqueue`.                                                                                                                                                                   |
-
-### 2.3.3 Attribute Naming Conventions
-
-Span **names** follow §2.3.1 (dotted `<component>.<operation>`). Span
-**attribute keys** follow the rules below. The constants in the `*SpanNames.h`
-headers are the single source of truth; the collector, Tempo, the Grafana
-dashboards, and the runbook all consume these exact keys, so every layer must
-agree with the code. A CI check enforces this end to end.
-
-1. **Per-span unique attribute** → bare field name, allowed when the field is
-   recorded by a single span/workflow so the span name already supplies the
-   domain (e.g. `command`, `version`, `local` on `rpc.command`).
-2. **Shared attribute (same concept on more than one span)** → ONE key, reused
-   verbatim on every span that records it; the span name tells the occurrences
-   apart, so no per-emitter prefix is added. Name it by the field's meaning: a
-   property of a domain object keeps that object's bare field name (`ledger_hash`,
-   `ledger_seq`, `tx_hash`, `peer_id`, `full_validation`); a field already
-   qualified by a sub-kind keeps that qualifier on every emitter (`proposal_trusted`
-   on both `consensus.proposal.receive` and `peer.proposal.receive`;
-   `validation_trusted` likewise). Defined once in the base `SpanNames.h`
-   `namespace attr` block and re-exported (`using`) by each domain header.
-3. **Collision qualifier** → `<domain>_<field>`, only when a bare name would
-   collide with a DIFFERENT concept in the shared span_metrics label space or with
-   the OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`,
-   `consensus_phase`, `consensus_round`, `consensus_mode`). This disambiguates
-   distinct concepts that share a word; it is NOT used to tag the same concept
-   with its emitting workflow — that is rule 2 (one shared name).
-4. **Resource attribute** → dotted `xrpl.<subsystem>.<field>`, reserved ONLY
-   for process/network identity set once at startup (`xrpl.network.id`,
-   `xrpl.network.type`). Span attributes are never dotted in the `xrpl.` form —
-   it blurs the resource/span scope boundary and parses awkwardly in TraceQL.
-5. **Span names** use `<subsystem>[.<component>]` (dotted, per §2.3.1). Only
-   attribute _keys_ follow rules 1–4.
-
-Standard OpenTelemetry semantic-convention keys keep their canonical dotted
-form (e.g. `service.*` resource attributes, `http.*` span attributes); the
-"no dotted form" rule applies to xrpl-custom keys only.
-
-The same rules are recorded in `CONTRIBUTING.md` (the permanent home, since
-`OpenTelemetryPlan/` is removed once the rollout completes). The attribute
-examples in §2.4 below follow these rules.
-
----
-
-## 2.4 Attribute Schema
-
-> **TxQ** = Transaction Queue | **UNL** = Unique Node List | **OTLP** = OpenTelemetry Protocol
-
-### 2.4.1 Resource Attributes (Set Once at Startup)
-
-Resource attributes identify the process and are set once at startup. They use
-the standard OpenTelemetry semantic conventions plus custom dotted `xrpl.*`
-keys (the dotted form is reserved for resource scope per §2.3.3).
-
-Five are set, by `Telemetry.cpp:380-387` (tracer resource) and the matching
-block in `initMetrics()` (metrics resource); the custom key constants are
-`SpanNames.h:117-118`.
-
-| Key                   | Type / value                                                   | Description                    | Status                                                                                                                                                                                                                                                                                          |
-| --------------------- | -------------------------------------------------------------- | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `service.name`        | `"xrpld"`                                                      | Standard `SERVICE_NAME`        | Set                                                                                                                                                                                                                                                                                             |
-| `service.version`     | `build_info::getVersionString()`                               | Standard `SERVICE_VERSION`     | Set                                                                                                                                                                                                                                                                                             |
-| `service.instance.id` | node public key (base58), or `[telemetry] service_instance_id` | Standard `SERVICE_INSTANCE_ID` | Set — but the node-key fallback reaches traces only; see [05 §5.1.1](./05-configuration-reference.md)                                                                                                                                                                                           |
-| `xrpl.network.id`     | network id (e.g. 0 for mainnet)                                | Network identifier             | Set                                                                                                                                                                                                                                                                                             |
-| `xrpl.network.type`   | `"mainnet"` \| `"testnet"` \| `"devnet"` \| `"unknown"`        | Network kind                   | Set                                                                                                                                                                                                                                                                                             |
-| `xrpl.node.type`      | `"validator"` \| `"stock"` \| `"reporting"`                    | Node role                      | **Not implemented** — no constant, no set-site. Node role is therefore not queryable from a trace. (Dashboards do offer an `$xrpl_node_role` filter, but it matches a Prometheus label stamped by the external perf-iac deployment — `check_otel_naming.py:872` — not by anything in this repo) |
-| `xrpl.node.cluster`   | cluster name                                                   | Cluster name, if clustered     | **Not implemented** — no constant, no set-site                                                                                                                                                                                                                                                  |
-
-The collector adds two more resource attributes of its own (`deployment.environment`
-and, when the node did not stamp it, `xrpl.network.type`) via the
-`resource/tier` processor, and deletes the SDK-injected `telemetry.sdk.*` trio
-via `resource/stripsdk`. See [05 §5.5.1](./05-configuration-reference.md).
-
-### 2.4.2 Span Attributes by Category
-
-> Span attribute keys use the underscore form from §2.3.3 (shared/qualified
-> keys are `<domain>_<field>`; per-span unique keys are bare). The dotted form
-> is reserved for the resource attributes in §2.4.1 above. This catalog lists
-> the planned attribute set by category; the exact emitted key **and its type**
-> for each implemented span is defined by the `*SpanNames.h` constants and their
-> set-sites, which win where the two differ. The types in the tables below are
-> the ones originally planned and are **not** all what shipped — `peer_id` is
-> the notable case (planned as a base58 string, shipped as an int64). §2.4.3
-> is the implemented view.
-
-#### Transaction Attributes
-
-| Key                  | Type   | Description                                                                                                                                                   |
-| -------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `tx_hash`            | string | Transaction hash (hex)                                                                                                                                        |
-| `tx_type`            | string | `"Payment"`, `"OfferCreate"`, etc.                                                                                                                            |
-| `tx_account`         | string | Sending account, raw r-address                                                                                                                                |
-| `tx_<field>`         | string | One per account-typed top-level field the transaction carries (`tx_destination`, `tx_owner`, `tx_issuer`, ...), raw r-address; keys in `TxAccountSpanNames.h` |
-| `tx_sequence`        | int64  | Account sequence number                                                                                                                                       |
-| `tx_fee`             | int64  | Fee in drops                                                                                                                                                  |
-| `tx_result`          | string | `"tesSUCCESS"`, `"tecPATH_DRY"`, etc.                                                                                                                         |
-| `current_ledger_seq` | int64  | Open ledger the transaction targeted                                                                                                                          |
-| `relay_count`        | int64  | Peers the transaction was relayed to                                                                                                                          |
-
-> **Note:** `current_ledger_seq` and `ledger_seq` are the same concept — a ledger's sequence number — but they name different ledgers, so the design keeps two keys rather than one. `current_ledger_seq` is the open or in-flight ledger a transaction's work was applied into; it is named after the RPC field `ledger_current_index`. `ledger_seq` (see [Ledger & Job Attributes](#ledger--job-attributes)) is a closed or validated ledger, set by the ledger and consensus spans. Neither is spelled `ledger_index`: per rule 2 of [Telemetry span attribute naming](../CONTRIBUTING.md#telemetry-span-attribute-naming), one concept gets one key reused verbatim, and a different referent is disambiguated with a prefix rather than a synonym.
-
-#### Consensus Attributes
-
-| Key                  | Type    | Description                         |
-| -------------------- | ------- | ----------------------------------- |
-| `consensus_round`    | int64   | Round number                        |
-| `consensus_phase`    | string  | `"open"`, `"establish"`, `"accept"` |
-| `consensus_mode`     | string  | `"proposing"`, `"observing"`, etc.  |
-| `proposers`          | int64   | Number of proposers                 |
-| `prev_ledger_prefix` | string  | Previous ledger hash prefix         |
-| `ledger_seq`         | int64   | Ledger sequence                     |
-| `tx_count`           | int64   | Transactions in consensus set       |
-| `round_time_ms`      | float64 | Round duration                      |
-
-Establish-phase gap fill and cross-node correlation attributes (Phase 4a):
-
-| Key                   | Type   | Description                                               |
-| --------------------- | ------ | --------------------------------------------------------- |
-| `consensus_round_id`  | int64  | Consensus round number                                    |
-| `consensus_ledger_id` | string | `previousLedger.id()` — shared across nodes               |
-| `trace_strategy`      | string | `"deterministic"` or `"random"`                           |
-| `converge_percent`    | int64  | Convergence % (0-100+)                                    |
-| `establish_count`     | int64  | Number of establish iterations                            |
-| `disputes_count`      | int64  | Active disputed transactions                              |
-| `agree_count`         | int64  | Peers that agree (haveConsensus)                          |
-| `disagree_count`      | int64  | Peers that disagree                                       |
-| `threshold_percent`   | int64  | Close-time consensus threshold (`avCT_CONSENSUS_PCT`=75%) |
-| `consensus_result`    | string | `"yes"`, `"no"`, `"moved_on"`, `"expired"`                |
-| `mode_old`            | string | Previous consensus mode                                   |
-| `mode_new`            | string | New consensus mode                                        |
-
-#### RPC Attributes
-
-| Key           | Type    | Description                                                                   |
-| ------------- | ------- | ----------------------------------------------------------------------------- |
-| `command`     | string  | Command name (per-span unique on `rpc.command`)                               |
-| `version`     | int64   | API version                                                                   |
-| `rpc_role`    | string  | `"admin"` or `"user"` (qualified — `role` is generic)                         |
-| `params`      | string  | Sanitized parameters (optional)                                               |
-| `rpc_status`  | string  | Response status: `success` \| `error` (qualified — `status` is OTel-reserved) |
-| `duration_ms` | float64 | Request duration in milliseconds                                              |
-
-#### Peer & Message Attributes
-
-| Key                  | Type    | Description                                                               |
-| -------------------- | ------- | ------------------------------------------------------------------------- |
-| `peer_id`            | string  | Peer public key (base58) — **planned only; shipped as int64, see §2.4.3** |
-| `peer_address`       | string  | IP:port                                                                   |
-| `peer_latency_ms`    | float64 | Measured latency                                                          |
-| `peer_cluster`       | string  | Cluster name if clustered                                                 |
-| `message_type`       | string  | Protocol message type name                                                |
-| `message_size_bytes` | int64   | Message size                                                              |
-| `message_compressed` | bool    | Whether compressed                                                        |
-
-#### Ledger & Job Attributes
-
-| Key                         | Type    | Description                      |
-| --------------------------- | ------- | -------------------------------- |
-| `ledger_hash`               | string  | Ledger hash                      |
-| `ledger_seq`                | int64   | Closed/validated ledger sequence |
-| `close_time_ripple_epoch_s` | int64   | Close time (XRPL epoch seconds)  |
-| `ledger_tx_count`           | int64   | Transaction count                |
-| `job_type`                  | string  | Job type name                    |
-| `job_queue_ms`              | float64 | Time spent in queue              |
-| `job_worker`                | int64   | Worker thread ID                 |
-
-#### PathFinding Attributes
-
-| Key                        | Type   | Description                                                            |
-| -------------------------- | ------ | ---------------------------------------------------------------------- |
-| `pathfind_source_account`  | string | Source r-address, raw                                                  |
-| `pathfind_dest_account`    | string | Destination r-address, raw                                             |
-| `pathfind_source_currency` | string | Source currency code                                                   |
-| `pathfind_dest_currency`   | string | Destination asset: `XRP`, `<issuer>/<currency>`, or an MPT issuance id |
-| `pathfind_path_count`      | int64  | Number of paths found                                                  |
-| `pathfind_cache_hit`       | bool   | RippleLineCache hit                                                    |
-
-#### TxQ Attributes
-
-| Key                   | Type   | Description                 |
-| --------------------- | ------ | --------------------------- |
-| `txq_queue_depth`     | int64  | Current queue depth         |
-| `txq_fee_level`       | int64  | Fee level of transaction    |
-| `txq_eviction_reason` | string | Why transaction was evicted |
-
-#### Fee Attributes
-
-| Key                    | Type  | Description               |
-| ---------------------- | ----- | ------------------------- |
-| `fee_load_factor`      | int64 | Current load factor       |
-| `fee_escalation_level` | int64 | Fee escalation multiplier |
-
-#### Validator Attributes
-
-| Key                      | Type  | Description               |
-| ------------------------ | ----- | ------------------------- |
-| `validator_list_size`    | int64 | UNL size                  |
-| `validator_list_age_sec` | int64 | Seconds since last update |
-
-#### Amendment Attributes
-
-| Key                | Type   | Description                            |
-| ------------------ | ------ | -------------------------------------- |
-| `amendment_name`   | string | Amendment name                         |
-| `amendment_status` | string | `"enabled"`, `"vetoed"`, `"supported"` |
-
-#### SHAMap Attributes
-
-| Key                    | Type    | Description                                   |
-| ---------------------- | ------- | --------------------------------------------- |
-| `shamap_type`          | string  | `"transaction"`, `"state"`, `"account_state"` |
-| `shamap_missing_nodes` | int64   | Number of missing nodes during sync           |
-| `shamap_duration_ms`   | float64 | Sync duration                                 |
-
-### 2.4.3 Data Collection Summary
-
-§2.4.2 above is the _planned_ catalogue; this table is the **implemented** one.
-Its left column lists the keys of the `attr` namespaces of the `*SpanNames.h`
-headers; every key shown has at least one live `attr::` set-site in
-non-test code. The right column lists keys this document once claimed were
-collected but which have no constant and no set-site at all.
-
-**This table is a category-level roll-up, not the authority.** The
-authoritative per-span breakdown — which span carries which attribute — is
-[09-data-collection-reference.md §1.2](./09-data-collection-reference.md#12-complete-attribute-inventory-bareunderscore-keys),
-and the exact key _spelling_ is owned by the `*SpanNames.h` constants. Where
-this table disagrees with either, they win.
-
-> **Known divergence (documented, not resolved here).** 09 §1.2's Consensus
-> subsection lists fewer keys than
-> `include/xrpl/consensus/ConsensusSpanNames.h` defines in its `attr` namespace
-> (its own `makeStr` constants plus `using` re-exports of the shared keys in
-> `SpanNames.h`). Some of the missing keys — `open_duration_ms`,
-> `peer_positions_at_close`, `position_hash_prefix`, `prev_ledger_prefix`,
-> `disputes_resolved_count` — are emitted but absent from 09 §1.2's consensus
-> table; two more, `proposal_trusted` and `validation_trusted`, are
-> documented in 09 §1.2's Peer subsection instead (they are shared keys set on
-> both the `peer.*` and the `consensus.*` receive spans — `PeerImp.cpp:1953`
-> and `:2027` for the proposal pair, `:2591` and `:2635` for the validation
-> pair). Fixing 09 is tracked separately; the header holds the full list.
-
-| Category        | Attributes emitted (from `*SpanNames.h`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | Named here but NOT emitted                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | Purpose                      |
-| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- |
-| **Transaction** | `tx_hash`, `tx_type`, `ter_result`, `fee`, `sequence`, `current_ledger_seq`, `current_ledger_hash`, `local`, `path`, `tx_status`, `peer_version`, `peer_id`, `stage`, `applied`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        | `tx_result` (renamed → `ter_result`), `tx_fee` (→ `fee`), `ledger_index` (→ `current_ledger_seq`), `relay_count`. **`ledger_seq` is not a `tx.*` key**: no `tx.*` span sets it — the receive and apply-stage spans stamp `current_ledger_seq` (`NetworkOPs.cpp:1422`, `PeerImp.cpp:1337`, `Transactor.cpp:1613`, `applySteps.cpp:115`) and, where a view exists, `current_ledger_hash` (`Transactor.cpp:1615`, `applySteps.cpp:121`)                                                                                                                                                                                      | Trace transaction lifecycle  |
-| **Consensus**   | Most keys in `ConsensusSpanNames.h`'s `attr` namespace (own `makeStr` keys and `using` re-exports), each with a set-site: `consensus_ledger_id`, `consensus_round`, `consensus_round_id`, `consensus_phase`, `consensus_mode`, `consensus_state`, `consensus_result`, `consensus_stalled`, `proposers`, `proposers_finished`, `previous_proposers`, `previous_ledger_seq`, `previous_round_time_ms`, `round_time_ms`, `open_duration_ms`, `quorum`, `proposing`, `is_bow_out`, `trace_strategy`, `converge_percent`, `establish_count`, `tx_count`, `tx_count_open`, `tx_id`, `disputes_count`, `disputes_resolved_count`, `dispute_our_vote`, `dispute_yays`, `dispute_nays`, `agree_count`, `disagree_count`, `threshold_percent`, `avalanche_threshold`, `close_time_threshold`, `have_close_time_consensus`, `close_time_resolution_ms`, `close_time_self_ripple_epoch_s`, `close_time_vote_bins`, `resolution_direction`, `parent_close_time_ripple_epoch_s`, `peer_positions_at_close`, `prev_ledger_prefix`, `position_hash_prefix`, `mode_old`, `mode_new`, `validation_sign_time`, `proposal_trusted`, `validation_trusted`; re-exported shared keys `ledger_seq`, `ledger_hash`, `full_validation`, `close_time_ripple_epoch_s`, `close_time_correct`, `close_resolution_ms` | —                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | Analyze consensus timing     |
-| **RPC**         | `command`, `version`, `rpc_role`, `rpc_status`, `request_payload_size`, `is_batch`, `batch_size`, `load_type`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | `duration_ms` (span duration is a TraceQL intrinsic — query `duration`), `params`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | Monitor RPC performance      |
-| **Peer**        | `peer_id` (**int64**, the process-local `Peer::id_` slot number — not a key of any kind; also set on `tx.receive`), `proposal_trusted`, `validation_trusted`, `ledger_hash`, `full_validation`. (`peer_version` is **not** a peer-span key: the constant lives in `TxSpanNames.h:79` and its only set-site is `PeerImp.cpp:1342` on the `tx.receive` span — see the Transaction row)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | `peer_address`, `peer_latency_ms`, `peer_cluster`, `message_type`, `message_size_bytes`, `message_compressed` — the peer-span coverage gap (§2.3.2)                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | Network topology analysis    |
-| **Ledger**      | `ledger_seq`, `tx_count`, `tx_failed`, `validations`, `acquire_reason`, `timeouts`, `peer_count`, `outcome`, `close_time_ripple_epoch_s`, `close_time_correct`, `close_resolution_ms`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  | `ledger_index` (→ `ledger_seq`), `ledger_tx_count` (→ `tx_count`). `ledger_hash` is a live key, but **no `ledger.*` span sets it** — only `consensus.validation.send` (`RCLConsensus.cpp:977`; that span is the one returned by `createValidationSpan()`, which names `cs::validationSend` at `RCLConsensus.cpp:1365,1373`) and `peer.validation.receive` (`PeerImp.cpp:2573`) do. The `LedgerSpanNames.h:41` `using` alias has zero uses. `consensus.ledger_close` sets **no** hash: its four attributes are `ledger_seq`, `consensus_mode`, `tx_count_open` and `close_time_resolution_ms` (`RCLConsensus.cpp:354-361`) | Ledger progression tracking  |
-| **gRPC**        | `method`, `grpc_role`, `grpc_status`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | —                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | gRPC surface monitoring      |
-| **Job**         | — (no job spans exist)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 | `job_type`, `job_queue_ms`, `job_worker`. JobQueue is observed via **metrics**, not spans — but by **two disjoint families**, and only one of them has a `job_type` label. See the note below the table                                                                                                                                                                                                                                                                                                                                                                                                                   | JobQueue performance         |
-| **PathFinding** | `pathfind_fast`, `pathfind_search_level`, `pathfind_num_paths`, `pathfind_ledger_index`, `pathfind_num_requests`, `pathfind_num_source_assets`, `pathfind_dest_currency`, `pathfind_source_account` (hashed), `pathfind_dest_account` (hashed)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | `pathfind_source_currency`, `pathfind_path_count`, `pathfind_cache_hit`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | Payment path analysis        |
-| **TxQ**         | `txq_status`, `fee_level_paid`, `required_fee_level`, `queue_size`, `ledger_changed`, `expired_count`, `ter_code`, `retries_remaining`, `num_cleared`, `tx_type`, plus the re-exported shared keys `tx_hash`, `ledger_seq`, `current_ledger_seq`, `current_ledger_hash`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | `txq_queue_depth` (→ `queue_size`), `txq_fee_level` (→ `fee_level_paid`), `txq_eviction_reason`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           | Queue depth and fee tracking |
-| **Fee**         | — (no `fee.escalate` span, §2.3.2)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | `fee_load_factor`, `fee_escalation_level`. Fee escalation is dashboarded from metrics (`fee-market`), not spans                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           | Fee escalation monitoring    |
-| **Validator**   | — (no `validator.*` span, §2.3.2)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      | `validator_list_size`, `validator_list_age_sec`. UNL health is dashboarded from metrics (`validator-health`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              | UNL health monitoring        |
-| **Amendment**   | — (no `amendment.vote` span, §2.3.2)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | `amendment_name`, `amendment_status`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      | Protocol upgrade tracking    |
-| **SHAMap**      | — (no `shamap.sync` span, §2.3.2)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      | `shamap_type`, `shamap_missing_nodes`, `shamap_duration_ms`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | State tree sync performance  |
-
-The right-hand column is the honest gap list: every key in it appears in the
-§2.4.2 design catalogue but has **zero set-sites** in the code. Where a rename
-happened the live name is given in parentheses; where the concept shipped as a
-metric rather than a span that is stated. Do not build a dashboard panel, an
-alert rule, or a TraceQL query against anything in that column — the query will
-return empty, and (per the PromQL/TraceQL asymmetry) a `=~".*"` matcher on an
-absent attribute silently blanks a TraceQL panel while quietly passing in
-PromQL.
-
-> **JobQueue metrics: two families, one label.** The Job row above has no span
-> attributes, and the metrics that replace them do **not** all carry a
-> `job_type` label. Getting this wrong produces a panel that renders but is
-> wrong, so treat the two families as separate query surfaces:
->
-> | Family                                                                                                                   | Where the job type lives                                                | Source                                                                                      |
-> | ------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- |
-> | Native `XRPL_METRIC_*`: `job_queued_total`, `job_started_total`, `job_finished_total`, `job_queued_us`, `job_running_us` | In a **`job_type` label**                                               | `MetricsRegistry.cpp:360-362` (counters), `:94-95` (histogram names), `:101` (label key)    |
-> | `beast::insight` `jobq` group: `jobq_<jobtype>_waiting` / `_running` / `_deferred` / `_q`                                | In the **metric name itself** — there is **no** `job_type` label at all | `JobTypeData.h:29-32` (naming contract), `:35-38` (suffixes), `Application.cpp:392` (group) |
->
-> **The trap:** `sum by (job_type)(jobq_…)` collapses every job type into a
-> single series with an empty `job_type`, because an absent PromQL label is
-> equivalent to `""` — the query returns a plausible-looking number rather than
-> an error. Aggregate the `jobq_*` family with a name matcher
-> (`{__name__=~"jobq_.*_waiting"}`) and reserve `by (job_type)` for the
-> `job_*_total` / `job_*_us` family.
-
-### 2.4.4 Privacy & Sensitive Data Policy
-
-> **PII** = Personally Identifiable Information
-
-OpenTelemetry instrumentation is designed to collect **operational metadata only**, never sensitive content.
-
-#### Data NOT Collected
-
-The following data is explicitly **excluded** from telemetry collection:
-
-| Excluded Data           | Reason                                                                                                                                                                                                                                                |
-| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Private Keys**        | Never exposed; not relevant to tracing                                                                                                                                                                                                                |
-| **Account Balances**    | Financial data; privacy sensitive                                                                                                                                                                                                                     |
-| **Transaction Amounts** | Financial data; privacy sensitive                                                                                                                                                                                                                     |
-| **Raw TX Payloads**     | May contain sensitive memo/data fields                                                                                                                                                                                                                |
-| **Personal Data**       | No PII collected                                                                                                                                                                                                                                      |
-| **IP Addresses**        | **Never in spans** — no span sets an address attribute (`peer_address` has zero set-sites); peer spans identify peers by `peer_id`, an int64 process-local slot number. **But the log pipeline is a different story** — see the note below this table |
-
-> **Peer IPs DO leave the node — via the log pipeline, not via spans.** The
-> "IP Addresses" row above is scoped to spans, and only to spans. This same
-> document describes a log pipeline (§2.6.5) that carries peer addresses:
->
-> 1. `PeerImp`'s constructor logs the peer's `remoteAddress_` — an `IP:port` —
->    at `info` severity (`PeerImp.h:837-842`), and other overlay call sites log
->    addresses too. These land in the ordinary `debug.log` stream.
-> 2. The collector's `file_log` receiver tails exactly that file
->    (`otel-collector-config.yaml:38-47`, `include: [/var/log/xrpld/*/debug.log]`)
->    and the `logs` pipeline exports it to Loki (`:236-239`).
->
-> So a deployment running the shipped stack **does** ship peer IPs off-box, as
-> log bodies. There is no attribute to drop and no span-level switch to flip,
-> because the IPs are inside free-text log messages rather than in structured
-> fields — a `delete` action on an attribute key would not touch them.
->
-> **The control points are therefore log-side, not trace-side:** Loki
-> retention and access control on the log store; the `file_log` receiver's
-> `include` list (dropping it disables log↔trace correlation entirely); or a
-> collector-side transform on the log body. Do not describe the telemetry
-> pipeline as IP-free without qualifying it to traces.
-
-#### Privacy Protection Mechanisms
-
-| Mechanism                     | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Account Addresses**         | Emitted raw (`pathfind_source_account`, `pathfind_dest_account`). An account address is a public ledger identifier; hashing it protects nothing and breaks the join against explorers, RPC and logs                                                                                                                                                                                                                                                                                                                                                                                                                              |
-| **Collector Tail Sampling**   | **Optional, and OFF in the base stack.** xrpld head sampling is fixed at 1.0 (`Telemetry.h:234` `static constexpr double samplingRatio = 1.0;`), so 100% of traces leave the node. `docker/telemetry/otel-collector-config.yaml` has **no** `tail_sampling` processor either, so the local stack stores 100%. The only shipped policy is in the Grafana Cloud overlay (`otel-collector-config.grafanacloud.yaml:60-67`, wired at `:261`): one `probabilistic` policy at **0.5%**, on the trace-storage branch only so spanmetrics still see every span. Treat sampling as a cost control you opt into — not as a privacy control |
-| **Local Control**             | Node operators have full control over what gets exported                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
-| **No Raw Payloads**           | Transaction content is never recorded, only metadata (hash, type, result)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| **Collector-Level Filtering** | Available for a future genuinely sensitive attribute via an `attributes` processor. None is shipped, and none must be added for account addresses                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
-
-#### Account Addresses
-
-Account addresses are emitted **raw**, at every layer:
-
-1. **SDK-side** (this node): the path-finding RPC handlers set
-   `pathfind_source_account` / `pathfind_dest_account` to the request's
-   r-address, only when it parses as one, and `pathfind_dest_currency` to `to_string(Asset)`,
-   which carries the IOU issuer's r-address. The rationale sits on the attribute
-   constants in `PathFindSpanNames.h`.
-2. **Collector-side**: no collector configuration in this repository hashes
-   or deletes these attributes.
-
-Why raw: an r-address is a public, enumerable identifier on the ledger. An
-unsalted hash of it is reversible by table lookup, so it protects nothing, and
-it breaks the one thing the attribute is for: joining a span to the account as
-explorers, RPC responses and logs show it. The helper `redactAccount()`
-(`xrpl::telemetry`, `Redaction.h`) remains available for a value that is
-genuinely private, but it is applied to no span.
-
-#### Collector-Level Data Protection
-
-No hashing or redaction processor is shipped: account addresses are public
-identifiers and stay as emitted.
-
-**No `peer_address` or `params` scrubbing rule is needed on the trace pipeline,
-and none is shipped.** Earlier drafts prescribed `delete` actions for both.
-Neither attribute is ever emitted: `peer_address` has zero set-sites in the code
-(peer spans carry `peer_id`, an int64 process-local slot number — not an IP and
-not a key), and no span sets a `params` attribute — RPC spans carry `command`,
-`version`, `rpc_role`, `rpc_status`, `request_payload_size`, `is_batch`,
-`batch_size` and `load_type`, never the request body. Adding delete rules for
-absent keys would be harmless but misleading: it would imply the node emits IPs
-and request parameters in spans when it does not.
-
-This says nothing about the **log** pipeline, which is where peer IPs actually
-do leave the node (see the note under "Data NOT Collected" above). An
-`attributes` processor cannot help there — the addresses are inside free-text
-log bodies, not in structured attributes.
-
-If a future span _does_ introduce an IP-bearing or payload-bearing attribute,
-the `attributes` processor is the right place to strip it — and the attribute
-should be added to the §2.4 catalogue in the same change.
-
-#### Configuration Options for Privacy
-
-In `xrpld.cfg`, operators control data collection granularity through the
-`[telemetry]` section. Besides `enabled`, per-component toggles
-(`trace_transactions`, `trace_consensus`, `trace_rpc`, `trace_peer` — the last
-often disabled due to high volume) select which spans are emitted. There is no
-redaction setting: account addresses are public and are emitted raw.
-
-> **Key Principle**: Telemetry collects **operational metadata** (timing, counts, hashes, public identifiers) — never **sensitive content** (keys, balances, amounts, raw payloads).
-
----
-
-## 2.5 Context Propagation Design
-
-> **WS** = WebSocket
-
-### 2.5.0 Deterministic Trace ID Strategy
-
-Both transaction and consensus tracing use **deterministic trace IDs** derived from
-a globally known hash, so all nodes handling the same workflow independently produce
-spans under the same `trace_id`. This is combined with protobuf `span_id` propagation
-for parent-child relay ordering when available.
-
-#### Transactions — `trace_id = txHash[0:16]`
-
-Every node that handles a transaction knows its `txID` (the `uint256` transaction
-hash). The first 16 bytes of this hash are used as the OTel `trace_id`:
-
-```
-uint256 txHash:  A1B2C3D4 E5F6A7B8 C9D0E1F2 A3B4C5D6  E7F8A9B0 C1D2E3F4 A5B6C7D8 E9F0A1B2
-                 |---------- trace_id (16 bytes) ---------|  (remaining 16 bytes unused)
-```
-
-Each node generates a **random 8-byte `span_id`** so its span is unique within the shared trace. When the incoming `TMTransaction` carries a protobuf `TraceContext` whose `trace_id` is `txID[0:16]`, the sender's `span_id` becomes the parent. This keeps the relay chain as a parent-child tree. A parent and its child share one trace. A context that names another trace is not used. When the context is absent (older peers, first hop from client) or not used, the span appears as a root in the same trace. Correlation is preserved. Only the tree structure degrades.
-
-```
-Node A (submitter)        Node B (relay)          Node C (relay)
-trace_id: A1B2...         trace_id: A1B2...       trace_id: A1B2...
-span_id:  1234 (random)   span_id:  5678 (random) span_id:  9ABC (random)
-parent:   (none)          parent:   1234 (proto)   parent:   5678 (proto)
-                               ↑                        ↑
-                       protobuf propagation      protobuf propagation
-```
-
-If protobuf propagation fails at Node B (old peer):
-
-```
-Node A                    Node B (old peer)       Node C
-trace_id: A1B2...         trace_id: A1B2...       trace_id: A1B2...
-span_id:  1234            span_id:  5678          span_id:  9ABC
-parent:   (none)          parent:   (none)        parent:   5678 (proto)
-                          ↑ no parent, but same trace_id — still grouped
-```
-
-#### Consensus — `trace_id = prevLedgerHash[0:16]`
-
-All validators in the same consensus round share the same `previousLedger.id()`.
-The first 16 bytes are used as trace_id. See [Phase 4a implementation status](./06-implementation-phases.md)
-and `createDeterministicContext()` in `RCLConsensus.cpp` for the implementation.
-
-Switchable via `consensus_trace_strategy` config:
-`"deterministic"` (default) or `"random"` (random trace_id, correlation via attribute queries).
-`"random"` is experimental and not used: it would break cross-node trace correlation.
-
-#### Why Not Random IDs with Propagation Only?
-
-Random trace IDs require **unbroken context propagation** across every hop. In a
-mixed-version network (common during upgrades), older peers silently drop the
-`trace_context` protobuf field. The trace splits and downstream spans become
-impossible to find. Deterministic IDs make correlation **propagation-resilient** — the trace
-backend groups all spans for the same transaction/round regardless of whether
-propagation succeeded.
-
-#### Why Keep Protobuf Propagation?
-
-Deterministic trace IDs alone provide correlation (all spans grouped) but not
-**causality** (which node relayed to which). Protobuf `span_id` propagation adds
-parent-child ordering that shows the exact relay path. The two mechanisms complement
-each other:
-
-| Mechanism                    | Provides                    | Fails when                             |
-| ---------------------------- | --------------------------- | -------------------------------------- |
-| Deterministic trace_id       | Cross-node correlation      | Never (hash is always known)           |
-| Protobuf span_id propagation | Parent-child relay ordering | Older peer drops `trace_context` field |
-
-#### Implementation Reference
-
-The utility function `createDeterministicTxContext(uint256 const& txHash)` follows
-the same pattern as `createDeterministicContext(uint256 const& ledgerId)` in
-`RCLConsensus.cpp`. See [Phase 3 Task 3.9](./Phase3_taskList.md) for the full spec.
-
-### 2.5.1 Propagation Boundaries
-
-```mermaid
-flowchart TB
-    subgraph http["HTTP/WebSocket (RPC)"]
-        w3c["W3C Trace Context Headers:<br/>traceparent:<br/>00-trace_id-span_id-flags<br/>tracestate: xrpld=..."]
-    end
-
-    subgraph protobuf["Protocol Buffers (P2P)"]
-        proto["message TraceContext {<br/>  bytes trace_id = 1;  // 16 bytes<br/>  bytes span_id = 2;   // 8 bytes<br/>  uint32 trace_flags = 3;<br/>  reserved 4;  // trace_state, later<br/>}"]
-    end
-
-    subgraph jobqueue["JobQueue / Coroutines (Internal Async)"]
-        job["CoroAwareContextStorage<br/>(RuntimeContextStorage override)<br/><br/>Per-coroutine context stack,<br/>installed globally at startup.<br/>Job itself carries no context."]
-    end
-
-    style http fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style protobuf fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style jobqueue fill:#bf360c,stroke:#8c2809,color:#ffffff
-```
-
-**Reading the diagram:**
-
-- **HTTP/WebSocket - RPC (blue)**: For client-facing RPC requests, trace context is propagated using the W3C `traceparent` header. This is the standard approach and works with any OTel-compatible client.
-- **Protocol Buffers - P2P (green)**: For peer-to-peer messages between xrpld nodes, trace context is embedded as a protobuf `TraceContext` message carrying trace_id, span_id, flags, and optional trace_state.
-- **JobQueue / Coroutines - Internal Async (red)**: For asynchronous work within a single node, the ambient OTel context follows the coroutine rather than being carried on the work item. `include/xrpl/core/Job.h` has **no** telemetry include and no `traceContext_` member — an earlier draft of this diagram showed one, and that was never built. Instead `xrpl::telemetry::CoroAwareContextStorage` (`include/xrpl/telemetry/CoroAwareContextStorage.h:84`) overrides the SDK's `RuntimeContextStorage` with a per-coroutine context stack, and is installed as the global storage in `Telemetry::start()` (`Telemetry.cpp:416-419`) before the tracer provider and before the first span. That fixes the wrong-thread scope pop across coroutine yield/resume and keeps log↔trace correlation intact. The storage is never reset — tearing it down while spans may still exist is undefined behaviour in the SDK — so it lives for the process lifetime.
-
----
-
-## 2.6 Integration with Existing Observability
-
-> **OTLP** = OpenTelemetry Protocol | **WS** = WebSocket
-
-### 2.6.1 Existing Frameworks Comparison
-
-xrpld already has two observability mechanisms. OpenTelemetry complements (not replaces) them:
-
-| Aspect                | PerfLog                       | Beast Insight (StatsD)       | OpenTelemetry             |
-| --------------------- | ----------------------------- | ---------------------------- | ------------------------- |
-| **Type**              | Logging                       | Metrics                      | Distributed Tracing       |
-| **Data**              | JSON log entries              | Counters, gauges, histograms | Spans with context        |
-| **Scope**             | Single node                   | Single node                  | **Cross-node**            |
-| **Output**            | `perf.log` file               | StatsD server                | OTLP Collector            |
-| **Question answered** | "What happened on this node?" | "How many? How fast?"        | "What was the journey?"   |
-| **Correlation**       | By timestamp                  | By metric name               | By `trace_id`             |
-| **Overhead**          | Low (file I/O)                | Low (UDP packets)            | Low-Medium (configurable) |
-
-### 2.6.2 What Each Framework Does Best
-
-#### PerfLog
-
-- **Purpose**: Detailed local event logging for RPC and job execution
-- **Strengths**:
-  - Rich JSON output with timing data
-  - Already integrated in RPC handlers
-  - File-based, no external dependencies
-- **Limitations**:
-  - Single-node only (no cross-node correlation)
-  - No parent-child relationships between events
-  - Manual log parsing required
-
-A PerfLog entry is a JSON object with fields such as `time`, `method`,
-`duration_us`, and `result`.
-
-#### Beast Insight (StatsD)
-
-- **Purpose**: Real-time metrics for monitoring dashboards
-- **Strengths**:
-  - Aggregated metrics (counters, gauges, histograms)
-  - Low overhead (UDP, fire-and-forget)
-  - Good for alerting thresholds
-- **Limitations**:
-  - No request-level detail
-  - No causal relationships
-  - Single-node perspective
-  - Aggregation happens on the StatsD server, not in the process
-
-In xrpld, Beast Insight is used through `increment` (counters), `gauge`
-(point-in-time values), and `timing` (durations) calls. A `timing` call sends
-each measured value as its own raw `|ms` sample, so the histogram a dashboard
-reads is built by the StatsD server from that stream of values.
-
-#### OpenTelemetry (NEW)
-
-- **Purpose**: Distributed request tracing across nodes
-- **Strengths**:
-  - **Cross-node correlation** via `trace_id`
-  - Parent-child span relationships
-  - Rich attributes per span
-  - A `Histogram` instrument that aggregates **at the point of measure**
-  - Industry standard (CNCF)
-- **Limitations**:
-  - Requires collector infrastructure
-  - Higher complexity than logging
-
-A span is created via `startSpan` (e.g. `"tx.relay"`), annotated with
-attributes such as `tx_hash` and `peer_id`, and is automatically linked to its
-parent through the active context.
-
-OpenTelemetry is not only spans. The same SDK offers a `Histogram` instrument,
-and a `Record()` call folds the value straight into bucket counts inside the
-process — no per-event record is shipped and no server-side aggregation step is
-needed. That is what makes it affordable in a hot loop where one span per event
-would not be, and it is the one thing Beast Insight cannot do, because its
-`timing` path ships raw values and aggregates them on the StatsD server.
-
-### 2.6.3 When to Use Each
-
-| Scenario                                | PerfLog    | StatsD | OpenTelemetry |
-| --------------------------------------- | ---------- | ------ | ------------- |
-| "How many TXs per second?"              | ❌         | ✅     | ✅            |
-| "What's the p99 RPC latency?"           | ❌         | ✅     | ✅            |
-| "Why was this specific TX slow?"        | ⚠️ partial | ❌     | ✅            |
-| "Which node delayed consensus?"         | ❌         | ❌     | ✅            |
-| "What happened on node X at time T?"    | ✅         | ❌     | ✅            |
-| "Show me the TX journey across 5 nodes" | ❌         | ❌     | ✅            |
-| "p99 NodeStore fetch latency?"          | ❌         | ❌     | ✅            |
-
-The last row is the case a span cannot answer. One `TMGetObjectByHash` message
-requests up to `tuning::kHardMaxReplyNodes` objects, so a span per NodeStore
-fetch is not affordable in that loop. Instead the fetch loop's wall time is
-recorded once per message into an OpenTelemetry `Histogram`
-(`getobject_lookup_us`), and the quantile is read off its buckets. StatsD is
-marked ❌ because that instrument is recorded on the native OpenTelemetry metrics
-path, not through Beast Insight.
-
-### 2.6.4 Coexistence Strategy
-
-> **Note**: Phase 7 **added** a native OTel Metrics export path alongside the
-> StatsD bridge; it did not replace it. The diagram below shows the Phase 6
-> state, which is still reachable today via `[insight] server=statsd`. See
-> [Phase7_taskList.md](./Phase7_taskList.md) for the design.
-
-```mermaid
-flowchart TB
-    subgraph xrpld["xrpld Process"]
-        perflog["PerfLog<br/>(JSON to file)"]
-        insight["Beast Insight<br/>(StatsD)"]
-        otel["OpenTelemetry<br/>(Tracing)"]
-    end
-
-    perflog --> perffile["perf.log"]
-    insight --> statsd["StatsD Server"]
-    otel --> collector["OTLP Collector"]
-
-    perffile --> grafana["Grafana<br/>(Unified UI)"]
-    statsd --> grafana
-    collector --> grafana
-
-    style xrpld fill:#212121,stroke:#0a0a0a,color:#ffffff
-    style grafana fill:#bf360c,stroke:#8c2809,color:#ffffff
-```
-
-**Reading the diagram:**
-
-- **xrpld Process (dark gray)**: The single xrpld node running all three observability frameworks side by side. Each framework operates independently with no interference.
-- **PerfLog to perf.log**: PerfLog writes JSON-formatted event logs to a local file. Grafana can ingest these via Loki or a file-based datasource.
-- **Beast Insight to StatsD Server**: Insight sends aggregated metrics (counters, gauges) over UDP to a StatsD server. Grafana reads from StatsD-compatible backends like Graphite or Prometheus (via StatsD exporter).
-- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/HTTP to a Collector, which then forwards to a trace backend (Tempo). (OTLP/gRPC is future work — §2.2.2.)
-- **Grafana (red, unified UI)**: All three data streams converge in Grafana, enabling operators to correlate logs, metrics, and traces in a single dashboard.
-
-**Phase 7 outcome (as shipped)**: Beast Insight gained an `OTelCollector`
-`Collector` implementation that rides the global MeterProvider and exports via
-OTLP/HTTP to the same collector as traces. It is selected with
-`[insight] server=otel`.
-
-The three back ends are **co-equal branches of one `if/else` chain** in
-`makeCollectorManager()` (`CollectorManager.cpp:37-75`), not a migration path:
-
-| `[insight] server=`    | Collector         | Status                                                                                                                                   |
-| ---------------------- | ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------- |
-| `otel`                 | `OTelCollector`   | OTLP/HTTP to the OTel Collector — the recommended setting                                                                                |
-| `statsd`               | `StatsDCollector` | Unchanged from before Phase 7. **Not deprecated**: no warning is logged, no removal is scheduled, and the code path is not marked legacy |
-| absent / anything else | `NullCollector`   | **The default.** A node with no `[insight]` section emits no metrics at all                                                              |
-
-Two corrections to earlier drafts, both of which matter operationally: StatsD
-is not a "deprecated fallback", and `otel` is not the default — you must set it
-explicitly. See [06-implementation-phases.md §6.8](./06-implementation-phases.md),
-[Phase7_taskList.md](./Phase7_taskList.md), and
-[05 §5.8.6](./05-configuration-reference.md) for which `[insight]` keys are live
-under `server=otel` (most are inert).
-
-### 2.6.5 Correlation with Logs
-
-**Shipped in Phase 8 — and not the way this section originally planned it.**
-The design here was a `setTraceId` hook on PerfLog, fed from the
-`rpc.command.<method>` span in `RPCHandler.cpp`. That hook was never built:
-`setTraceId` has zero occurrences in **source** — the only hits in the tree are
-in these plan documents, describing the design that was dropped — and PerfLog's
-JSON output carries no trace ID.
-
-What shipped instead is broader and needs no per-call-site wiring: the **journal
-sink** stamps the IDs onto _every_ log line written while a span is active.
-`Logs::format()` (`src/libxrpl/basics/Log.cpp:304-338`, inside
-`#ifdef XRPL_ENABLE_TELEMETRY`) reads the thread-local OTel context, and when
-the active span context is valid it prefixes the message with
-`trace_id=<32 hex> span_id=<16 hex>`. It inspects the context value directly
-rather than calling `GetSpan()`, so the common no-span path costs no heap
-allocation.
-
-Because the IDs land in the ordinary `debug.log` stream, correlation is
-end-to-end without touching PerfLog: the collector's `file_log` receiver parses
-`trace_id`/`span_id` as optional capture groups and ships the lines to Loki, and
-Grafana links both directions (Tempo `tracesToLogs` → Loki, Loki derived fields
-→ Tempo). Details in [05 §5.8.5](./05-configuration-reference.md).
-
-RPC spans still exist and still set status (OK on success, error with the
-recorded exception on failure) — that part of the original design is intact.
-Only the PerfLog-stamping mechanism was replaced.
-
----
-
-_Previous: [Architecture Analysis](./01-architecture-analysis.md)_ | _Next: [Implementation Strategy](./03-implementation-strategy.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_

OpenTelemetryPlan/03-implementation-strategy.md

@@ -1,561 +0,0 @@
-# Implementation Strategy
-
-> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md)
-> **Related**: [Configuration Reference](./05-configuration-reference.md)
-
----
-
-## 3.1 Directory Structure
-
-The telemetry implementation follows xrpld's existing code organization
-pattern. The tree below lists the files in the three telemetry directories.
-The two libxrpl directories hold the tracing wrapper, `MetricsRegistry` and
-the metric headers. The xrpld layer holds `AppMetricGauges` and
-`PeerValidationLoad.h`, beside the tracing helpers. Config structs live inside
-`Telemetry.h`, and context propagation lives in `TraceContextPropagator.h`.
-Attribute constants live in the `*SpanNames.h` headers next to their owning
-class.
-
-```
-include/xrpl/telemetry/            # libxrpl layer: tracing and metrics
-├── Telemetry.h                    # Interface + Setup config struct + factories
-├── SpanGuard.h                    # RAII span management, factory methods, discard()
-├── SpanNames.h                    # StaticStr/join() + shared span & attr constants
-├── MetricNames.h                  # Metric name, label and value constants
-├── DiscardFlag.h                  # Thread-local discard flag
-├── FilteringSpanProcessor.h       # Drops discarded spans; exports each attribute key once
-├── CoroAwareContextStorage.h      # RuntimeContextStorage override for coroutines
-├── DeterministicIdGenerator.h     # trace_id from txHash / prevLedgerHash
-├── HeadSampler.h                  # makeHeadSampler(): parent-based trace-id ratio sampler
-├── TraceContextPropagator.h       # protobuf TraceContext inject/extract (P2P)
-├── TraceContextValidation.h       # Validation of peer-supplied trace context
-├── Redaction.h                    # redactAccount() — hashing helper, applied to no span
-├── TxAccountSpanNames.h           # tx_<account field> keys + accountFieldAttributeKey()
-├── Recording.h                    # Record-only state; empty types when compiled out
-├── MetricsRegistry.h              # MeterProvider + exporter + its own counters/histograms
-├── MetricMacros.h                 # XRPL_METRIC_COUNTER_ADD / _HISTOGRAM_RECORD / ...
-├── PreRegisteredCounters.h        # Startup zero for getobject_rejected_total
-├── HistogramBuckets.h             # Explicit histogram bucket ladders
-├── ValidationTracker.h            # Validation-agreement tracking
-├── RpcMetricNames.h               # RPC request-count histogram names
-├── PeerValidationLoadMetricNames.h
-└── GetObjectMetricNames.h         # getobject_* metric name constants
-
-src/libxrpl/telemetry/
-├── Telemetry.cpp                  # TelemetryImpl + initMetrics()
-├── FilteringSpanProcessor.cpp     # Span processor + the recordable that merges repeated keys
-├── TelemetryConfig.cpp            # [telemetry] section parsing (makeTelemetrySetup)
-├── SpanGuard.cpp                  # Span/scope guard implementation
-├── CoroAwareContextStorage.cpp
-├── DeterministicIdGenerator.cpp
-├── Redaction.cpp
-├── TxAccountSpanNames.cpp         # SField code -> attribute key table
-├── MetricsRegistry.cpp            # Export pipeline, registry instruments, histogram views
-├── detail/ValidationTracker.cpp
-└── NullTelemetry.cpp              # No-op impl — ALWAYS compiled (in-source #ifdef)
-
-src/xrpld/telemetry/               # xrpld layer: observable gauges + tracing helpers
-├── AppMetricGauges.h / .cpp       # Observable gauges that sample live server state
-├── PeerValidationLoad.h           # Rates, ranks and warning throttle for peer_validation_load
-├── ConsensusReceiveTracing.h      # Peer proposal/validation receive spans
-├── MacroCounterPreRegistration.h  # Startup zeros for call-site counters with fixed label sets
-├── PropagationHelpers.h           # Context inject/extract call-site helpers
-├── TxSpanNames.h                  # tx.* span + attribute constants
-└── TxTracing.h                    # Transaction span helpers
-```
-
-Per-class span-name headers deliberately live next to their owning class rather
-than in `telemetry/` — see `ConsensusSpanNames.h`, `TxApplySpanNames.h`,
-`LedgerSpanNames.h`, `RpcSpanNames.h`, `PathFindSpanNames.h`,
-`PeerSpanNames.h`, `TxQSpanNames.h`, `GrpcSpanNames.h`. `TxAccountSpanNames.h` is
-the exception and sits in `telemetry/`: its key table is checked against `TxFormats`
-by a libxrpl test, which cannot include a daemon header.
-
----
-
-## 3.2 Implementation Approach
-
-<div align="center">
-
-```mermaid
-%%{init: {'flowchart': {'nodeSpacing': 20, 'rankSpacing': 30}}}%%
-flowchart TB
-    subgraph phase1["Phase 1: Core"]
-        direction LR
-        sdk["SDK Integration"] ~~~ interface["Telemetry Interface"] ~~~ config["Configuration"]
-    end
-
-    subgraph phase2["Phase 2: RPC"]
-        direction LR
-        http["HTTP Context"] ~~~ rpc["RPC Handlers"]
-    end
-
-    subgraph phase3["Phase 3: P2P"]
-        direction LR
-        proto["Protobuf Context"] ~~~ tx["Transaction Relay"]
-    end
-
-    subgraph phase4["Phase 4: Consensus"]
-        direction LR
-        consensus["Consensus Rounds"] ~~~ proposals["Proposals"]
-    end
-
-    phase1 --> phase2 --> phase3 --> phase4
-
-    style phase1 fill:#1565c0,stroke:#0d47a1,color:#ffffff
-    style phase2 fill:#2e7d32,stroke:#1b5e20,color:#ffffff
-    style phase3 fill:#e65100,stroke:#bf360c,color:#ffffff
-    style phase4 fill:#c2185b,stroke:#880e4f,color:#ffffff
-```
-
-</div>
-
-### Key Principles
-
-1. **Minimal Intrusion**: Instrumentation should not alter existing control flow
-2. **Zero-Cost When Disabled**: Use compile-time flags and no-op implementations
-3. **Backward Compatibility**: Protocol Buffer extensions use high field numbers
-4. **Graceful Degradation**: Tracing failures must not affect node operation
-
----
-
-## 3.3 Performance Overhead Summary
-
-> **OTLP** = OpenTelemetry Protocol
-
-| Metric        | Overhead   | Notes                                            |
-| ------------- | ---------- | ------------------------------------------------ |
-| CPU           | 1-3%       | Of per-transaction CPU cost (~200μs baseline)    |
-| Memory        | ~10 MB     | SDK statics + batch buffer + worker thread stack |
-| Network       | 10-50 KB/s | Compressed OTLP export to collector              |
-| Latency (p99) | <2%        | With proper sampling configuration               |
-
----
-
-## 3.4 Detailed CPU Overhead Analysis
-
-### 3.4.1 Per-Operation Costs
-
-> **Note on hardware assumptions**: The costs below are based on the official OTel C++ SDK CI benchmarks
-> (969 runs on GitHub Actions 2-core shared runners). On production server hardware (3+ GHz Xeon),
-> expect costs at the **lower end** of each range (~30-50% improvement over CI hardware).
-
-| Operation             | Time (ns) | Frequency              | Impact     |
-| --------------------- | --------- | ---------------------- | ---------- |
-| Span creation         | 500-1000  | Every traced operation | Low        |
-| Span end              | 100-200   | Every traced operation | Low        |
-| SetAttribute (string) | 80-120    | 3-5 per span (typical) | Low        |
-| SetAttribute (int)    | 40-60     | 2-3 per span (typical) | Negligible |
-| AddEvent              | 100-200   | 0-2 per span           | Low        |
-| Context injection     | 150-250   | Per outgoing message   | Low        |
-| Context extraction    | 100-180   | Per incoming message   | Low        |
-| GetCurrent context    | 10-20     | Thread-local access    | Negligible |
-
-> **"3-5 attributes per span" is a typical case, not a bound.** The frequency
-> column above describes the median span (`tx.receive`, `rpc.command.*`). A few
-> spans are deliberately attribute-rich: `consensus.accept.apply` sets **13**
-> attributes (`RCLConsensus.cpp:600-674`), and `consensus.round` /
-> `consensus.establish` are of the same order. Use ~15 as the worst case when
-> sizing per-span attribute cost and memory; the consensus spans that hit it fire
-> once per ~3-second round, so their absolute cost stays in the noise
-> (see §3.4.3).
-
-**Source**: Span creation based on OTel C++ SDK `BM_SpanCreation` benchmark (AlwaysOnSampler +
-SimpleSpanProcessor + InMemoryExporter), median ~1,000 ns on CI hardware. AddEvent includes
-timestamp read + string copy + vector push + mutex acquisition. Context injection/extraction
-confirmed by `BM_SpanCreationWithScope` benchmark delta (~160 ns).
-
-### 3.4.2 Transaction Processing Overhead
-
-<div align="center">
-
-```mermaid
-%%{init: {'pie': {'textPosition': 0.75}}}%%
-pie showData
-    "tx.receive (1400ns)" : 1400
-    "tx.process (1200ns)" : 1200
-    "tx.apply (1200ns)" : 1200
-    "Context inject (200ns)" : 200
-```
-
-**Transaction Tracing Overhead (~4.0μs total)**
-
-</div>
-
-**Overhead percentage**: 4.0 μs / 200 μs (avg tx processing) = **~2.0%**
-
-> **Breakdown**: Each span (tx.receive, tx.process, tx.apply) costs ~1,000 ns for creation plus
-> ~200-400 ns for 3-5 attribute sets. Context injection is ~200 ns (confirmed by benchmarks).
-> On production hardware, expect ~2.6 μs total (~1.3% overhead) due to faster span creation (~500-600 ns).
->
-> This three-span model predates the apply-pipeline instrumentation. The shipped
-> transaction path also emits `tx.preflight`, `tx.preclaim` and `tx.transactor`
-> (the spans planned here as `tx.validate`), and never emits `tx.relay`. Scale
-> the estimate by span count for a current figure: ~6 spans ≈ 7-8 μs on CI
-> hardware, ~4-5 μs on server hardware. The measured end-to-end cost is in
-> §3.5.3 (~3-4% throughput at head sampling 1.0), which supersedes this
-> bottom-up estimate.
-
-### 3.4.3 Consensus Round Overhead
-
-| Operation              | Count | Cost (ns) | Total      |
-| ---------------------- | ----- | --------- | ---------- |
-| consensus.round span   | 1     | ~1200     | ~1.2 μs    |
-| consensus.phase spans  | 3     | ~1100     | ~3.3 μs    |
-| proposal.receive spans | ~20   | ~1100     | ~22 μs     |
-| proposal.send spans    | ~3    | ~1100     | ~3.3 μs    |
-| Context operations     | ~30   | ~200      | ~6 μs      |
-| **TOTAL**              |       |           | **~36 μs** |
-
-> **Why higher**: Each span costs ~1,000 ns creation + ~100-200 ns for 1-2 attributes, totaling ~1,100-1,200 ns.
-> Context operations remain ~200 ns (confirmed by benchmarks). On production hardware, expect ~24 μs total.
->
-> The "1-2 attributes" figure understates the shipped consensus spans, which are
-> the attribute-rich ones: `consensus.accept.apply` alone sets 13
-> (`RCLConsensus.cpp:600-674`). Adding ~1 μs per such span still leaves the
-> round total under ~40 μs against a ~3 s round, so the conclusion below is
-> unaffected. Note also that the `consensus.phase` row covers the shipped names
-> `consensus.phase.open`, `consensus.establish` and `consensus.accept` — see
-> [02 §2.3.2](./02-design-decisions.md).
-
-**Overhead percentage**: 36 μs / 3s (typical round) = **~0.001%** (negligible)
-
-### 3.4.4 RPC Request Overhead
-
-| Operation                                  | Cost (ns)    |
-| ------------------------------------------ | ------------ |
-| `rpc.http_request` / `rpc.ws_message` span | ~1200        |
-| `rpc.command.*` span                       | ~1100        |
-| Context extract                            | ~250         |
-| Context inject                             | ~200         |
-| **TOTAL**                                  | **~2.75 μs** |
-
-> **Why higher**: Each span costs ~1,000 ns creation + ~100-200 ns for attributes (command name,
-> version, role). Context extract/inject costs are confirmed by OTel C++ benchmarks.
-
-- Fast RPC (1ms): 2.75 μs / 1ms = **~0.275%**
-- Slow RPC (100ms): 2.75 μs / 100ms = **~0.003%**
-
----
-
-## 3.5 Memory Overhead Analysis
-
-> **OTLP** = OpenTelemetry Protocol
-
-### 3.5.1 Static Memory
-
-| Component                            | Size        | Allocated  |
-| ------------------------------------ | ----------- | ---------- |
-| TracerProvider singleton             | ~64 KB      | At startup |
-| BatchSpanProcessor (circular buffer) | ~16 KB      | At startup |
-| BatchSpanProcessor (worker thread)   | ~8 MB       | At startup |
-| OTLP/HTTP exporter (client init)     | ~64 KB      | At startup |
-| Propagator registry                  | ~8 KB       | At startup |
-| **Total static**                     | **~8.1 MB** |            |
-
-> **Why higher than earlier estimate**: The BatchSpanProcessor's circular buffer itself is only ~16 KB
-> (2049 x 8-byte `AtomicUniquePtr` entries), but it spawns a dedicated worker thread whose default
-> stack size on Linux is ~8 MB. The OTLP/HTTP exporter allocates a small client and TLS
-> initialization buffer. The worker thread stack dominates the static footprint.
-
-### 3.5.2 Dynamic Memory
-
-| Component            | Size per unit  | Max units  | Peak            |
-| -------------------- | -------------- | ---------- | --------------- |
-| Active span          | ~500-800 bytes | 1000       | ~500-800 KB     |
-| Queued span (export) | ~500 bytes     | 2048       | ~1 MB           |
-| Attribute storage    | ~80 bytes      | 5 per span | Included        |
-| Context storage      | ~64 bytes      | Per thread | ~6.4 KB         |
-| **Total dynamic**    |                |            | **~1.5-1.8 MB** |
-
-> **Why active spans are larger**: An active `Span` object includes the wrapper (~88 bytes: shared_ptr,
-> mutex, unique_ptr to Recordable) plus `SpanData` (~250 bytes: SpanContext, timestamps, name, status,
-> empty containers) plus attribute storage (~200-500 bytes for 3-5 string attributes in a `std::map`).
-> Source: `sdk/src/trace/span.h` and `sdk/include/opentelemetry/sdk/trace/span_data.h`.
-> Queued spans release the wrapper, keeping only `SpanData` + attributes (~500 bytes).
-
-### 3.5.3 Memory Growth Characteristics
-
-```mermaid
----
-config:
-    xyChart:
-        width: 700
-        height: 400
----
-xychart-beta
-    title "Memory Usage vs Span Rate (bounded by queue limit)"
-    x-axis "Spans/second" [0, 200, 400, 600, 800, 1000]
-    y-axis "Memory (MB)" 0 --> 12
-    line [8.5, 9.2, 9.6, 9.9, 10.0, 10.0]
-```
-
-**Notes**:
-
-- Memory increases with span rate but **plateaus at queue capacity** (default 2048 spans)
-- Batch export prevents unbounded growth
-- At queue limit, oldest spans are dropped (not blocked)
-- Maximum memory is bounded: ~8.3 MB static (dominated by worker thread stack) + 2048 queued spans x ~500 bytes (~1 MB) + active spans (~0.8 MB) ≈ **~10 MB ceiling**
-- The worker thread stack (~8 MB) is virtual memory; actual RSS depends on stack usage (typically much less)
-
-> **Measured outcome**: A perf-iac comparison (telemetry compiled-in + enabled vs compiled-out,
-> 9 nodes — validators and client-handlers — under sustained payment load) recorded **no measurable
-> RSS increase over the telemetry-off baseline** (~15 GiB mean / ~18–19 GiB peak on both sides),
-> with no OOM, no swap, and no leak across the run. The ~10 MB ceiling above is therefore a
-> provisioning safety margin (dominated by virtual thread-stack address space), not an expected
-> resident-memory increase. Steady-state cost shows up as throughput (~3–4% at head sampling 1.0),
-> not memory.
-
-### 3.5.4 Performance Data Sources
-
-The overhead estimates in Sections 3.3-3.5 are derived from the following sources:
-
-| Source                                           | What it covers                                        | URL                                                                                                                                        |
-| ------------------------------------------------ | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ |
-| OTel C++ SDK CI benchmarks (969 runs)            | Span creation, context activation, sampler overhead   | [Benchmark Dashboard](https://open-telemetry.github.io/opentelemetry-cpp/benchmarks/)                                                      |
-| `api/test/trace/span_benchmark.cc`               | API-level span creation (~22 ns no-op)                | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/api/test/trace/span_benchmark.cc)                                   |
-| `sdk/test/trace/sampler_benchmark.cc`            | SDK span creation with samplers (~1,000 ns AlwaysOn)  | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/test/trace/sampler_benchmark.cc)                                |
-| `sdk/include/.../span_data.h`                    | SpanData memory layout (~250 bytes base)              | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/include/opentelemetry/sdk/trace/span_data.h)                    |
-| `sdk/src/trace/span.h`                           | Span wrapper memory layout (~88 bytes)                | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/src/trace/span.h)                                               |
-| `sdk/include/.../batch_span_processor_options.h` | Default queue size (2048), batch size (512)           | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/include/opentelemetry/sdk/trace/batch_span_processor_options.h) |
-| `sdk/include/.../circular_buffer.h`              | CircularBuffer implementation (AtomicUniquePtr array) | [Source](https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/include/opentelemetry/sdk/common/circular_buffer.h)             |
-| OTLP proto definition                            | Serialized span size estimation                       | [Proto](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto)                          |
-
----
-
-## 3.6 Network Overhead Analysis
-
-### 3.6.1 Export Bandwidth
-
-> **Bytes per span**: Estimates use ~500 bytes/span (conservative upper bound). OTLP protobuf analysis
-> shows a typical span with 3-5 string attributes serializes to ~200-300 bytes raw; with gzip
-> compression (~60-70% of raw) and batching (amortized headers), ~350 bytes/span is more realistic.
-> The table uses the conservative estimate for capacity planning.
-
-**Node → collector bandwidth is always the 100% row.** Head sampling is a
-`static constexpr` 1.0 (`Telemetry.h:234`) with no config key, so every node
-exports every span and the export bandwidth is not tunable from `xrpld.cfg`.
-
-| Sampling Rate         | Spans/sec | Bandwidth  | Where it applies                                                                                                                    |
-| --------------------- | --------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------- |
-| 100%                  | ~500      | ~250 KB/s  | **The only reachable node→collector figure.** Plan capacity against this row                                                        |
-| 0.5%                  | ~2.5      | ~1.25 KB/s | Collector→backend only, and only with the Grafana Cloud overlay's `tail_sampling` (`otel-collector-config.grafanacloud.yaml:60-67`) |
-| 10% / 1% / error-only | —         | —          | **Not implemented.** No shipped config produces these ratios; treat them as illustrative of what a tail-sampling policy could do    |
-
-The rows below 100% therefore reduce _storage_ cost at the backend, never the
-node's egress. Note also that the shipped 0.5% policy is applied to the
-trace-storage branch only, so the spanmetrics-derived RED metrics still see
-100% of spans and stay exact.
-
-### 3.6.2 Trace Context Propagation
-
-| Message Type           | Context Size | Messages/sec | Overhead    |
-| ---------------------- | ------------ | ------------ | ----------- |
-| TMTransaction          | 25 bytes     | ~100         | ~2.5 KB/s   |
-| TMProposeSet           | 25 bytes     | ~10          | ~250 B/s    |
-| TMValidation           | 25 bytes     | ~50          | ~1.25 KB/s  |
-| **Total P2P overhead** |              |              | **~4 KB/s** |
-
----
-
-## 3.7 Optimization Strategies
-
-### 3.7.1 Sampling Strategies
-
-#### Head Sampling (node) — fixed, not a decision point
-
-There is no sampling decision on the node. `samplingRatio` is a
-`static constexpr double = 1.0` (`Telemetry.h:234`) and `TelemetryConfig.cpp:139`
-records why nothing is parsed: a per-node ratio would let two nodes make
-opposite keep/drop decisions for the same distributed trace, yielding partial
-traces. A span with a remote parent is decided by the same ratio sampler, not by
-the peer's sampled flag (`makeHeadSampler`). The only node-local way to drop a span
-is the explicit, per-call-site `SpanGuard::discard()`, enforced downstream by
-`FilteringSpanProcessor`.
-
-#### Tail Sampling (collector) — aspirational shape
-
-The flowchart below is a **design sketch of a multi-policy tail sampler. It is
-not what ships.** The base collector config has no `tail_sampling` processor at
-all; the Grafana Cloud overlay has exactly one `probabilistic` policy at 0.5%
-with no error or latency carve-outs. Read it as a template for a policy you
-might write, not as a description of this repo — and note that adding
-error/latency policies would need `decision_wait` tuning, since a policy can
-only see spans that arrived within that window.
-
-```mermaid
-flowchart TD
-    trace["New Trace"]
-
-    trace --> errors{"Is Error?"}
-    errors -->|Yes| sample["SAMPLE"]
-    errors -->|No| consensus{"Is Consensus?"}
-
-    consensus -->|Yes| sample
-    consensus -->|No| slow{"Is Slow?"}
-
-    slow -->|Yes| sample
-    slow -->|No| prob{"Probabilistic keep?<br/>(shipped policy: 0.5%)"}
-
-    prob -->|Yes| sample
-    prob -->|No| drop["DROP"]
-
-    style sample fill:#1b5e20,stroke:#0d3d14,color:#fff
-    style drop fill:#b71c1c,stroke:#7f1d1d,color:#fff
-    style trace fill:#0d47a1,stroke:#082f6a,color:#fff
-    style errors fill:#334155,stroke:#1e293b,color:#fff
-    style consensus fill:#334155,stroke:#1e293b,color:#fff
-    style slow fill:#334155,stroke:#1e293b,color:#fff
-    style prob fill:#334155,stroke:#1e293b,color:#fff
-```
-
-### 3.7.2 Batch Tuning Recommendations
-
-| Environment        | Batch Size | Batch Delay | Max Queue |
-| ------------------ | ---------- | ----------- | --------- |
-| Low-latency        | 128        | 1000ms      | 512       |
-| High-throughput    | 1024       | 10000ms     | 8192      |
-| Memory-constrained | 256        | 2000ms      | 512       |
-
-### 3.7.3 Conditional Instrumentation
-
-Instrumentation is gated on two levels. A compile-time feature flag reduces the trace macros to no-ops when telemetry is built out, so disabled builds carry zero cost. At runtime, per-component guards (e.g. `shouldTracePeer()`) skip span creation for components whose tracing is turned off, incurring no overhead beyond a single boolean check.
-
-> The compile-time gate is the macro `XRPL_ENABLE_TELEMETRY`, but that macro is
-> **not** the switch you flip. It is a compile definition added by
-> `CMakeLists.txt` (`add_compile_definitions(XRPL_ENABLE_TELEMETRY)`) when the CMake option `telemetry` is ON.
-> That option is declared ON today (`option(telemetry "Enable OpenTelemetry tracing" ON)`)
-> only so that CI compiles the instrumented build while the telemetry branches are
-> in review; **OFF is the intended default once merged**, flipped in a separate
-> change. Select the value explicitly instead of relying on the default:
-> `-Dtelemetry=ON|OFF` (CMake) or `-o telemetry=True|False` (Conan). See
-> [05 §5.4.2](./05-configuration-reference.md).
-
----
-
-## 3.8 Links to Detailed Documentation
-
-- **[Configuration Reference](./05-configuration-reference.md)**: Configuration options and collector setup
-- **[Implementation Phases](./06-implementation-phases.md)**: Detailed timeline and milestones
-
----
-
-## 3.9 Code Intrusiveness Assessment
-
-> **TxQ** = Transaction Queue
-
-This section provides a detailed assessment of how intrusive the OpenTelemetry integration is to the existing xrpld codebase.
-
-### 3.9.3 Risk Assessment by Component
-
-<div align="center">
-
-**Do First** ↖ ↗ **Plan Carefully**
-
-```mermaid
-quadrantChart
-    title Code Intrusiveness Risk Matrix
-    x-axis Low Risk --> High Risk
-    y-axis Low Value --> High Value
-
-    RPC Tracing: [0.2, 0.55]
-    Transaction Relay: [0.55, 0.85]
-    Consensus Tracing: [0.75, 0.92]
-    Peer Message Tracing: [0.85, 0.35]
-    JobQueue Context: [0.3, 0.42]
-    Ledger Acquisition: [0.48, 0.65]
-    PathFinding: [0.38, 0.72]
-    TxQ and Fees: [0.25, 0.62]
-    Validator Mgmt: [0.15, 0.35]
-```
-
-**Optional** ↙ ↘ **Avoid**
-
-</div>
-
-#### Risk Level Definitions
-
-| Risk Level | Definition                                                       | Mitigation                         |
-| ---------- | ---------------------------------------------------------------- | ---------------------------------- |
-| **Low**    | Additive changes only; no modification to existing logic         | Standard code review               |
-| **Medium** | Minor modifications to existing functions; clear boundaries      | Comprehensive unit tests           |
-| **High**   | Changes to core logic or data structures; potential side effects | Integration tests + staged rollout |
-
-### 3.9.4 Architectural Impact Assessment
-
-| Aspect               | Impact  | Justification                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| -------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Data Flow**        | Minimal | Read-only instrumentation; no modification to consensus or transaction data flow                                                                                                                                                                                                                                                                                                                                                                     |
-| **Threading Model**  | Minimal | Context propagation uses thread-local storage (standard OTel pattern)                                                                                                                                                                                                                                                                                                                                                                                |
-| **Memory Model**     | Low     | Bounded queues prevent unbounded growth; RAII ensures cleanup                                                                                                                                                                                                                                                                                                                                                                                        |
-| **Network Protocol** | Low     | Optional fields in protobuf (high field numbers); backward compatible                                                                                                                                                                                                                                                                                                                                                                                |
-| **Configuration**    | None    | New config section; existing configs unaffected                                                                                                                                                                                                                                                                                                                                                                                                      |
-| **Build System**     | Low     | A single CMake option (`telemetry`) selects the whole feature in or out, and builds work either way (`-Dtelemetry=ON` / `-Dtelemetry=OFF`). It is declared ON today only so CI compiles the instrumented paths; **OFF is the intended default once merged**, so the shipped build is opt-in                                                                                                                                                          |
-| **Dependencies**     | Medium  | `opentelemetry-cpp/1.28.0` is a **conditional** requirement, never a hard one: `conanfile.py:152-153` adds it only `if self.options.telemetry`, and `:238-239` adds the matching `libxrpl` component requirement the same way. The option's declared default is `True` today (`conanfile.py:59`), so a default `conan install` does resolve it; with `-o telemetry=False` it never enters the graph and the null implementation supplies the factory |
-
-### 3.9.5 Backward Compatibility
-
-| Compatibility   | Status  | Notes                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| --------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| **Config File** | ✅ Full | New `[telemetry]` section is optional                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| **Protocol**    | ✅ Full | Optional protobuf fields with high field numbers                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| **Build**       | ✅ Full | `-Dtelemetry=OFF` (or `-o telemetry=False`) produces a binary with all tracing compiled out, whatever the option's declared default happens to be. **Not** `-DXRPL_ENABLE_TELEMETRY=OFF`, which does not disable anything — it is not a CMake option, only a compile definition that `CMakeLists.txt:152` adds inside the `if(telemetry)` block. CMake does flag it (`Manually-specified variables were not used by the project`) at the end of configuration, so it is not literally silent — but the warning is easy to scroll past and the resulting binary still has telemetry compiled in. See [05 §5.4.2](./05-configuration-reference.md) |
-| **Runtime**     | ✅ Full | `enabled=0` produces zero overhead                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
-| **API**         | ✅ Full | No changes to public RPC or P2P APIs                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-
-### 3.9.6 Rollback Strategy
-
-If issues are discovered after deployment:
-
-1. **Immediate**: Set `enabled=0` in `[telemetry]` and restart (zero code change).
-   Also set `[insight] server=` to something other than `otel` if metrics must
-   stop too — `enabled=0` governs tracing, and the metrics pipeline is selected
-   separately ([02 §2.6.4](./02-design-decisions.md)).
-2. **Quick**: Rebuild with `-Dtelemetry=OFF` (CMake) or `-o telemetry=False`
-   (Conan). Pass the flag explicitly — an omitted flag resolves to the option's
-   declared default, which is ON today and OFF once the feature is merged; a
-   build that already has telemetry off needs no rebuild at all.
-   **Do not use `-DXRPL_ENABLE_TELEMETRY=OFF`** — it is not a CMake option, so
-   it is ignored (CMake reports it under `Manually-specified variables were not
-used by the project`) and the rebuilt binary still has telemetry compiled in.
-   This step also drops the `opentelemetry-cpp` dependency, so expect a full
-   rebuild rather than an incremental one.
-3. **Complete**: Revert telemetry commits (clean separation makes this easy)
-
-### 3.9.7 Code Change Examples
-
-**Minimal RPC Instrumentation (Low Intrusiveness):** Instrumenting an RPC handler adds roughly 3-4 lines: one macro to start the span and one or two `setAttribute` calls (command name, status). The span ends automatically via RAII, so the existing control flow — process the request, send the result — is untouched.
-
-**Consensus Instrumentation (Medium Intrusiveness):** Consensus is slightly more intrusive because child spans in later phase transitions need the round's context. Beyond the span-start and attribute macros, this requires **four** new member variables on the adaptor rather than the single `currentRoundContext_` this section originally sketched (`RCLConsensus.h:103,113,123,143`):
-
-- `std::optional<telemetry::SpanGuard> roundSpan_` (`:103`) — the round span
-  itself. It is **created and ended in one place**, `startRoundTracing()`: the
-  previous round's guard is released at `RCLConsensus.cpp:1288-1289`
-  (`if (roundSpan_) roundSpan_.reset();`) and the new one is emplaced a few
-  lines later — at `:1306` or `:1310` on the `"attribute"` strategy, at `:1319`
-  on the default `"deterministic"` one. `preStartRound()` does not create it; it
-  calls `startRoundTracing()` at `:1229`. There is no `reset()` method — the
-  span simply lives until the next round begins. A `SpanGuard` owns no
-  thread-local scope, so emplacing and resetting on different job workers is
-  safe.
-- `telemetry::SpanContext roundSpanContext_` (`:113`) — a lightweight value-type
-  snapshot, captured at the end of `startRoundTracing()` (`:1350`). Child spans
-  link through this, not through an ambient parent, so code running on another
-  worker (e.g. `createValidationSpan()` on `jtACCEPT`) never touches
-  `roundSpan_` cross-thread.
-- `telemetry::SpanContext prevRoundSpanContext_` (`:123`) — the prior round's
-  context, saved at `:1282` **before** the new span overwrites
-  `roundSpanContext_`, so the new round span can carry a follows-from link and
-  consecutive rounds stay navigable.
-- `telemetry::SpanContext acceptSpanContext_` (`:143`) — the current round's
-  accept-span context, set at `:544` and cleared at `:1286` on each new round.
-  `createValidationSpan()` prefers it as the parent and falls back to
-  `roundSpanContext_` (`:1363-1373`), so a stale value must not survive into the
-  next round.
-
-The split is the point: the guard is owned by one thread, the contexts are
-copied freely. The existing round logic itself remains unchanged.
-
----
-
-_Previous: [Design Decisions](./02-design-decisions.md)_ | _Next: [Configuration Reference](./05-configuration-reference.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_

OpenTelemetryPlan/05-configuration-reference.md

@@ -1,671 +0,0 @@
-# Configuration Reference
-
-> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md)
-> **Related**: [Implementation Phases](./06-implementation-phases.md)
-
----
-
-## 5.1 xrpld Configuration
-
-> **OTLP** = OpenTelemetry Protocol | **TxQ** = Transaction Queue
-
-### 5.1.1 Configuration File Section
-
-The authoritative `[telemetry]` example lives in `cfg/xrpld-example.cfg`. Telemetry is disabled by default (`enabled=0`); enabling it turns on distributed tracing for transaction flow, consensus, and RPC calls, with traces exported to an OpenTelemetry Collector over OTLP. Head sampling is intentionally fixed at 1.0 (sample everything) and is not configurable — per-node head-sampling would produce broken/partial distributed traces, so volume reduction is delegated to the collector's tail sampling (see Section 7.4.2). Transaction trace IDs are always deterministic (`trace_id = txHash[0:16]`); there is no strategy switch for the transaction path. The full option reference follows.
-
-> **`service_instance_id` defaults to the node public key for every producer.**
-> `Main.cpp` resolves the identity before `ApplicationImp` is constructed and it
-> is never empty, so all three producers below stamp the node key when the config
-> key is unset. They still build their resources at three different times, which
-> is what decides how each behaves if the identity ever changes mid-run:
->
-> | Producer                                    | Resource built by                            | Unset `service_instance_id` yields |
-> | ------------------------------------------- | -------------------------------------------- | ---------------------------------- |
-> | Traces (and therefore all `span_*` metrics) | `Telemetry::start()`, during `setup()`       | Base58 node public key             |
-> | Native `XRPL_METRIC_*` (`MetricsRegistry`)  | `MetricsRegistry::initExporterAndProvider()` | Base58 node public key             |
-> | `beast::insight` (`[insight] server=otel`)  | `TelemetryImpl` **constructor**              | Base58 node public key             |
->
-> - **Traces**: the tracer resource is built in `Telemetry::start()`, which runs
->   inside `setup()` after `getNodeIdentity()` has persisted the identity, so it
->   is the one resource that can still be corrected by
->   `setServiceInstanceId()`. The `spanmetrics` connector derives
->   `span_calls_total` / `span_duration_milliseconds_*` from those spans, so span
->   metrics inherit the same id.
-> - **Native `XRPL_METRIC_*` metrics** build their **own** MeterProvider resource
->   in `MetricsRegistry::initExporterAndProvider()`, called from the registry's
->   constructor. `makeMetricsRegistryOptions()` in `Application.cpp` supplies the
->   id: the config key when set, else the resolved node key.
-> - **`beast::insight` metrics** use the **global** MeterProvider, whose resource
->   is built in the `TelemetryImpl` **constructor**, because insight instruments
->   are created eagerly in subsystem constructors and would otherwise bind to the
->   noop provider forever. It receives the same resolved key. Note it writes the
->   attribute **unconditionally**, where `MetricsRegistry` guards the write with
->   `if (!options.serviceInstanceId.empty())`, so a deliberately blank
->   `service_instance_id` yields `service.instance.id=""` there and an absent
->   attribute here.
->
-> The one case where the three can disagree: a node whose wallet already holds an
-> identity different from the resolved one (another process wrote it between
-> construction and `setup()`). `setServiceInstanceId()` then corrects the tracer,
-> and both metric resources, already frozen, need a restart to follow. Every
-> shipped Grafana dashboard filters on `service_instance_id=~"$node"`, so that run
-> shows its metrics under the resolved key and its traces under the stored one.
-
-### 5.1.2 Configuration Options Summary
-
-| Option                     | Type   | Default                            | Description                                                                                                                                                                                      |
-| -------------------------- | ------ | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `enabled`                  | 0 or 1 | `0`                                | Enable/disable telemetry                                                                                                                                                                         |
-| `traces_endpoint`          | string | `http://localhost:4318/v1/traces`  | OTLP/HTTP collector endpoint for **traces**                                                                                                                                                      |
-| `metrics_endpoint`         | string | `http://localhost:4318/v1/metrics` | OTLP/HTTP collector endpoint for the native metrics pipeline (`MetricsRegistry`). Read by `makeMetricsRegistryOptions()` in `Application.cpp`                                                    |
-| `use_tls`                  | 0 or 1 | `0`                                | Enable TLS for exporter connection                                                                                                                                                               |
-| `tls_ca_cert`              | string | `""`                               | Path to CA certificate file                                                                                                                                                                      |
-| `tls_client_cert`          | string | `""`                               | Client cert (PEM) for mTLS; empty = one-way; if `enabled=1`, needs key + `use_tls=1` or startup fails                                                                                            |
-| `tls_client_key`           | string | `""`                               | Private key (PEM) for `tls_client_cert`; if set with `enabled=1`, needs the cert + `use_tls=1` or fails                                                                                          |
-| `batch_size`               | uint   | `512`                              | Spans per export batch                                                                                                                                                                           |
-| `batch_delay_ms`           | uint   | `5000`                             | Max delay before sending batch (ms)                                                                                                                                                              |
-| `max_queue_size`           | uint   | `2048`                             | Maximum queued spans                                                                                                                                                                             |
-| `trace_transactions`       | 0 or 1 | `1`                                | Enable transaction tracing                                                                                                                                                                       |
-| `trace_consensus`          | 0 or 1 | `1`                                | Enable consensus tracing                                                                                                                                                                         |
-| `trace_rpc`                | 0 or 1 | `1`                                | Enable RPC tracing                                                                                                                                                                               |
-| `trace_peer`               | 0 or 1 | `1`                                | Enable peer message tracing (high volume)                                                                                                                                                        |
-| `trace_ledger`             | 0 or 1 | `1`                                | Enable ledger tracing                                                                                                                                                                            |
-| `consensus_trace_strategy` | string | `"deterministic"`                  | Consensus trace ID strategy: `"deterministic"` (trace_id = prevLedgerHash[0:16]) or `"random"` (experimental, not used). Rejected at startup if it is neither spelling                           |
-| `service_name`             | string | `"xrpld"`                          | Service name (`service.name`) for traces and metrics                                                                                                                                             |
-| `service_instance_id`      | string | node public key (base58)           | Instance identifier (`service.instance.id`). Traces, span metrics and native `XRPL_METRIC_*` metrics all fall back to the node key; **`beast::insight` metrics do not** — see the note in §5.1.1 |
-
-**Not a config key — deterministic transaction trace IDs are unconditional.**
-Earlier drafts of this document listed a `tx_trace_strategy` option
-(`"deterministic"` \| `"attribute"`). No such key exists: `TelemetryConfig.cpp`
-parses no transaction-strategy key, and the transaction trace ID is always
-derived from the transaction hash. Only the **consensus** path has a
-switchable strategy.
-
-**Planned (not yet implemented)**: the following options appear in the design
-documents but are not parsed by `TelemetryConfig.cpp`. They will be added as
-the corresponding subsystems are instrumented:
-
-| Option            | Planned Phase | Purpose                                  |
-| ----------------- | ------------- | ---------------------------------------- |
-| `exporter`        | Future        | Select between OTLP/HTTP and OTLP/gRPC   |
-| `trace_pathfind`  | Phase 2       | Path computation tracing toggle          |
-| `trace_txq`       | Phase 3       | Transaction queue tracing toggle         |
-| `trace_validator` | Future        | Validator list / manifest update tracing |
-| `trace_amendment` | Future        | Amendment voting tracing                 |
-
-> **`exporter` is not read, so do not set it.** No shipped sample config
-> carries it. OTLP/HTTP is the only transport that exists (§2.2.1), so
-> `traces_endpoint` and `metrics_endpoint` are the only transport knobs until
-> the §2.2.2 gRPC work lands.
-
----
-
-## 5.2 Configuration Parser
-
-> **TxQ** = Transaction Queue
-
-The parser `makeTelemetrySetup()` in `src/libxrpl/telemetry/TelemetryConfig.cpp` reads the `[telemetry]` `Section` and populates a `Telemetry::Setup` struct, applying the defaults listed in Section 5.1.2 via `section.valueOr(...)`. It takes `serviceInstanceId` from the `nodePublicKey` argument when the key is absent, applies one unconditional `traces_endpoint` default (`dflt::tracesEndpoint`) — the parser has no notion of exporter type — and leaves the sampling ratio at its fixed 1.0 default (a `static constexpr` member, so there is nothing to parse). It also rejects two contradictory mTLS configurations outright (`tls_client_cert` without `tls_client_key`, and either without `use_tls=1`) rather than failing open at handshake time.
-
-`metrics_endpoint` reaches `MetricsRegistry` by a second route: `makeMetricsRegistryOptions()` in `Application.cpp` reads it from the same `Section` and passes it to the registry's constructor, because `Telemetry` does not expose the `Setup` it parsed. Both metric exporters resolve to that one key:
-
-| Metric source                              | Exporter built by                            | URL comes from                                                       |
-| ------------------------------------------ | -------------------------------------------- | -------------------------------------------------------------------- |
-| `beast::insight` (`[insight] server=otel`) | `Telemetry::initMetrics()` (global provider) | `metrics_endpoint`, used verbatim                                    |
-| Native `XRPL_METRIC_*` (`MetricsRegistry`) | `MetricsRegistry::initExporterAndProvider()` | `metrics_endpoint`, defaulting to `http://localhost:4318/v1/metrics` |
-
-Setting `traces_endpoint` therefore moves traces only; both metric pipelines follow `metrics_endpoint`.
-
----
-
-## 5.3 Application Integration
-
-### 5.3.1 ApplicationImp Changes
-
-> **Identity before construction**: telemetry stamps the node public key into
-> resources that are immutable once built, and it builds them during
-> `ApplicationImp`'s member initializer list. So `Main.cpp` calls
-> `resolveNodeIdentity()` first, from the config and command line alone, and
-> passes the keypair to `makeApplication()`. It never comes back empty: a
-> configured `[node_seed]` decides it, else the wallet database supplies it if
-> one already exists, else it is minted. Every producer therefore carries the
-> node key from the start — the tracer, the **global** MeterProvider the
-> `beast::insight` metrics use, and `MetricsRegistry`'s separate one.
-> `ApplicationImp::setup()` then calls `getNodeIdentity()`, which stores that
-> keypair when the wallet holds none and otherwise adopts what the wallet holds.
-> Only in that second case can the two differ, and then
-> `setServiceInstanceId()` corrects the tracer alone: both metric resources are
-> frozen once their providers are built, so those metrics need a restart to
-> report the stored key.
-
-`ApplicationImp` (in `src/xrpld/app/main/Application.cpp`) owns a `std::pair<PublicKey, SecretKey> nodeIdentity_`, declared before `std::unique_ptr<telemetry::Telemetry> telemetry_` and `std::unique_ptr<telemetry::MetricsRegistry> metricsRegistry_` so both resources can be built from it. All three are built in the member initializer list, before every subsystem. `setup()` persists the identity, starts tracing with `startTelemetry()` before the first consensus round, and arms the registry's observable gauges with `startTelemetryGauges()` once `overlay_` exists. `run()` stops both observers before any service, then stops telemetry last; `~ApplicationImp` repeats those stops for the paths that never reach `run()`. `getTelemetry()` and `getMetricsRegistry()` return the owned instances.
-
-### 5.3.2 ServiceRegistry Interface Addition
-
-`include/xrpl/core/ServiceRegistry.h` gains a pure-virtual `telemetry::Telemetry& getTelemetry()` (with a forward declaration of `telemetry::Telemetry`), giving every component a uniform accessor for the tracing subsystem.
-
-> **Note:** `Application` extends `ServiceRegistry`, so `getTelemetry()` is
-> available on both. Components that hold a `ServiceRegistry&` (e.g.
-> `NetworkOPsImp`) call `registry_.get().getTelemetry()`. Components that
-> still hold an `Application&` (e.g. `ServerHandler`, `PeerImp`,
-> `RCLConsensus::Adaptor`) call `app_.getTelemetry()` directly.
-
----
-
-## 5.4 CMake Integration
-
-> **OTLP** = OpenTelemetry Protocol
-
-### 5.4.1 Locating the OpenTelemetry SDK
-
-> **Superseded design.** Earlier drafts described a hand-written
-> `cmake/FindOpenTelemetry.cmake` module that aliased `OpenTelemetry::api`,
-> `OpenTelemetry::sdk` and `OpenTelemetry::otlp_grpc_exporter` with a
-> `pkg-config` fallback. That module was never written — it exists in no
-> commit — and the aliasing approach it described does not work with the
-> package the build actually consumes.
-
-The SDK is located by the Conan-generated CMake config package, nothing else:
-
-- `CMakeLists.txt` — `find_package(opentelemetry-cpp CONFIG REQUIRED)`,
-  guarded by the `telemetry` option (§5.4.2). The dependency itself is
-  declared in `conanfile.py:153` (`opentelemetry-cpp/1.28.0`), also guarded —
-  `requirements()` adds it only `if self.options.telemetry` (`:152`), so with
-  the option off the package never enters the dependency graph.
-- Linking goes through the **umbrella** target
-  `opentelemetry-cpp::opentelemetry-cpp`, never the per-component targets.
-  `cmake/XrplCore.cmake:221-225` and `:83-91` record why: the Conan package
-  under-declares its inter-component dependencies, so naming `::api` / `::sdk`
-  individually produces the wrong static-link order and fails at executable
-  link time. The umbrella target supplies both the trace and metrics
-  components with the correct ordering.
-
-### 5.4.2 CMakeLists.txt Changes
-
-The build flag is `telemetry`:
-
-```
-option(telemetry "Enable OpenTelemetry tracing" ON)   # top-level CMakeLists.txt
-```
-
-The declared value is ON **temporarily**, so that CI compiles the telemetry code
-paths while the feature branches are in review. **OFF is the intended default
-once merged**, and the flip is a separate change. Set the value explicitly
-rather than relying on the default:
-
-| To …                      | Use (CMake)       | Use (Conan)          |
-| ------------------------- | ----------------- | -------------------- |
-| Build telemetry in        | `-Dtelemetry=ON`  | `-o telemetry=True`  |
-| Build it out (all no-ops) | `-Dtelemetry=OFF` | `-o telemetry=False` |
-
-When the option is ON, the guarded block below it runs
-`find_package(opentelemetry-cpp CONFIG REQUIRED)` and adds the
-**compile definition** `XRPL_ENABLE_TELEMETRY`.
-
-> **`XRPL_ENABLE_TELEMETRY` is not a CMake option.** It is only ever _added_
-> as a compile definition by `add_compile_definitions(XRPL_ENABLE_TELEMETRY)` in that same block. Passing
-> `-DXRPL_ENABLE_TELEMETRY=OFF` on the CMake command line disables **nothing** —
-> it defines an unused cache variable and telemetry stays compiled in. CMake does
-> report it, at the end of configuration under `Manually-specified variables were
-not used by the project`, so it is not literally silent — but that line is easy
-> to scroll past. Any procedure that relies on it (including the rollback path in
-> [§3.9.6](./03-implementation-strategy.md)) must use `-Dtelemetry=OFF`.
-
-The target is `xrpl.libxrpl.telemetry`, created by `add_module(xrpl telemetry)`
-at `cmake/XrplCore.cmake:231` from `include/xrpl/telemetry/` +
-`src/libxrpl/telemetry/`. There is no `xrpl_telemetry` target.
-
-Selection between the real and the no-op implementation is an **in-source
-`#ifdef`, not a source swap**: `NullTelemetry.cpp` is compiled into the target
-unconditionally (see its header comment, `NullTelemetry.cpp:1-12`). It provides
-the `makeTelemetry()` factory when `XRPL_ENABLE_TELEMETRY` is undefined; when
-the macro is defined, `Telemetry.cpp` provides the factory instead and
-`NullTelemetry`'s virtuals only serve as noop tracer/span fallbacks. Call sites
-compile unchanged either way.
-
----
-
-## 5.5 OpenTelemetry Collector Configuration
-
-> **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring
-
-The authoritative collector config lives in the repo at `docker/telemetry/otel-collector-config.yaml` (with Tempo backend config in `docker/telemetry/tempo.yaml`). The sections below summarize the development and production shapes of that pipeline.
-
-### 5.5.1 Development / Base Configuration
-
-`docker/telemetry/otel-collector-config.yaml` is the base config used by the
-local stack and by CI. It carries **three** pipelines, not one:
-
-| Pipeline  | Receivers              | Processors                                                     | Exporters                                  |
-| --------- | ---------------------- | -------------------------------------------------------------- | ------------------------------------------ |
-| `traces`  | `otlp`                 | `resource/tier`, `resource/stripsdk`, `batch`                  | `debug`, `otlp_grpc/tempo`, `span_metrics` |
-| `metrics` | `otlp`, `span_metrics` | `resource/tier`, `resource/stripsdk`, `batch`                  | `prometheus`                               |
-| `logs`    | `file_log`             | `resource/logs`, `resource/tier`, `resource/stripsdk`, `batch` | `otlp_http/loki`                           |
-
-Component detail:
-
-- **Receivers.** `otlp` on gRPC `0.0.0.0:4317` and HTTP `0.0.0.0:4318` (both
-  traces and native metrics arrive on 4318). `file_log` tails
-  `/var/log/xrpld/*/debug.log` and runs a `regex_parser` that lifts
-  `timestamp`, `partition`, `severity` and the optional `trace_id`/`span_id`
-  emitted by the journal sink (§5.8.5).
-- **Processors.** `batch` (1s timeout, `send_batch_size: 100`);
-  `resource/tier` (`action: upsert` on `deployment.environment`, `action: insert` on
-  `xrpl.network.type` only when absent); `resource/stripsdk` (drops the
-  `telemetry.sdk.*` attributes); `resource/logs` (`action: upsert` on
-  `service.name` and `job` — only the former becomes a Loki stream label, see
-  the known issue in §5.8.5). No processor hashes or drops span attributes:
-  account addresses are public identifiers and are stored as emitted.
-- **Connector.** `span_metrics` with `namespace: "span"`
-  (`otel-collector-config.yaml:114`) — this is why the derived RED metrics are
-  `span_calls_total` / `span_duration_milliseconds_*`. The connector's own
-  default namespace is **empty**, so without this setting the names would be
-  the bare `calls_total` / `duration_milliseconds_*`. The
-  `traces_span_metrics_*` family is **not** the connector's default and is not
-  produced here at all — it comes from a different producer, Tempo's
-  `metrics_generator` `span-metrics` processor (`tempo.yaml:75`), whose
-  `remote_write` is commented out in this repo (see §5.8.6). Histogram
-  `unit: ms`
-  with sub-millisecond buckets from `0.01ms`, plus explicit `2s`–`30s`
-  boundaries for consensus and `ledger.acquire`. ~25 low-cardinality
-  dimensions are promoted to labels (`command`, `rpc_status`, `tx_type`,
-  `ter_result`, `stage`, `consensus_mode`, `outcome`, …).
-- **Exporters.** `debug` (console, `verbosity: detailed`), `otlp_grpc/tempo`
-  (`tempo:4317`, `tls.insecure: true`), `otlp_http/loki`
-  (`http://loki:3100/otlp` — Loki 3.x native OTLP; the old `loki` exporter was
-  removed in collector-contrib v0.147.0), and `prometheus` on
-  `0.0.0.0:8889` with `resource_to_telemetry_conversion.enabled: true` so the
-  tier and instance resource attributes become Prometheus labels.
-- **Extensions.** `health_check` on `0.0.0.0:13133` only. There is **no**
-  `zpages` extension.
-
-Deliberately absent from the base config — do not document them as present:
-no `memory_limiter`, no `tail_sampling`, no Elastic APM exporter, and no
-attribute hashing or redaction rule (account addresses are emitted raw).
-
-### 5.5.2 Production Configuration
-
-There is no separate "production" collector config in this repo. The one
-overlay that exists is `docker/telemetry/otel-collector-config.grafanacloud.yaml`.
-It is **not** the base config plus one processor — it restructures the service
-graph. The full delta:
-
-| Added by the overlay            | Where                    | Purpose                                                                                                       |
-| ------------------------------- | ------------------------ | ------------------------------------------------------------------------------------------------------------- |
-| `basicauth/grafanacloud`        | `extensions`             | Extension; instance id / API token from the container environment                                             |
-| `batch/store`                   | `processors`             | 2048-span batches for `traces/store` only, so Grafana Cloud gets fewer, larger requests                       |
-| `tail_sampling`                 | `processors`             | One `probabilistic` policy at **0.5%**, `decision_wait: 10s`, `num_traces: 300000`, a decision cache          |
-| `transform/cloudlabels`         | `processors`             | Copies four resource attrs onto datapoint labels for Cloud (OTLP) ingest                                      |
-| `otlp/tempo` queue              | `exporters`              | Queue sized in spans (100000), because `batch/store` sends it large batches                                   |
-| `otlp_http/grafanacloud`        | `exporters`              | Metrics and logs to Grafana Cloud over OTLP/HTTP, with the default retry and queue                            |
-| `otlp_http/grafanacloud-traces` | `exporters`              | Stored traces to the same endpoint; queue sized in spans (100000), 20 consumers                               |
-| `metrics_flush_interval`        | `connectors.spanmetrics` | `spanmetrics` flushes every 15s instead of the 60s default                                                    |
-| `telemetry`                     | `service`                | The collector's own metrics, posted to its own OTLP receiver so they reach local Prometheus and Grafana Cloud |
-
-| Removed by the overlay | Consequence                                                       |
-| ---------------------- | ----------------------------------------------------------------- |
-| `debug`                | No console span dump; collector logs alone when diagnosing ingest |
-
-Pipelines go from **three** (`traces`, `metrics`, `logs`) to **five**
-(`service.pipelines`): `traces/metrics`, `traces/store`, `metrics/local`,
-`metrics/cloud`, `logs`. `tail_sampling` is applied in **`traces/store`**
-— the branch feeding Tempo and Grafana Cloud — not in a pipeline
-named `traces`, which does not exist in the overlay. The `traces/metrics`
-branch feeds `spanmetrics` unsampled, so the derived RED metrics stay exact
-while stored traces are ~1/200 of ingested ones.
-
-Hardening a collector for a real deployment (TLS/mTLS on the receiver,
-NetworkPolicy, peer trace-context validation) is covered in
-[Securing the OTel Pipeline](./secure-OTel.md) — not by any config file in
-`docker/telemetry/`.
-
----
-
-## 5.6 Docker Compose Development Environment
-
-> **OTLP** = OpenTelemetry Protocol
-
-The authoritative development stack lives in the repo at `docker/telemetry/docker-compose.yml`. It brings up **six** services on a shared `xrpld-telemetry` bridge network. All images are pinned to exact tags.
-
-| Service          | Image                                          | Published ports        | Role                                                              |
-| ---------------- | ---------------------------------------------- | ---------------------- | ----------------------------------------------------------------- |
-| `otel-collector` | `otel/opentelemetry-collector-contrib:0.158.0` | `4317`, `4318`, `8889` | OTLP ingest, spanmetrics, file_log tail, Prometheus scrape target |
-| `tempo`          | `grafana/tempo:2.9.4`                          | `3200`                 | Trace storage and TraceQL                                         |
-| `loki`           | `grafana/loki:3.7.6`                           | `3100`                 | Log storage for log↔trace correlation                             |
-| `prometheus`     | `prom/prometheus:v3.13.2`                      | `9090`                 | Scrapes the collector's `:8889`                                   |
-| `grafana`        | `grafana/grafana:13.1.2`                       | `3000`                 | Dashboards + provisioned datasources/alerts, anonymous admin      |
-| `renderer`       | `grafana/grafana-image-renderer:v5.12.0`       | none                   | Panel→PNG rendering for image export and alert screenshots        |
-
-Two corrections to earlier drafts:
-
-- **`prometheus` is not optional.** `grafana` lists it in `depends_on` (along
-  with `tempo`, `loki` and `renderer`), and several dashboards query
-  `span_calls_total` from it. Removing it blanks most panels.
-- **Port `13133` is not published.** The collector's `health_check` extension
-  listens on `13133` inside the container, but the base compose file publishes
-  only `4317`, `4318` and `8889`. Health checks from the host must either add a
-  port mapping or run `docker compose exec`.
-
-The collector also bind-mounts the xrpld log root read-only
-(`${XRPLD_LOG_DIR:-./data/logs}` → `/var/log/xrpld`) for the `file_log`
-receiver, and the `grafana` service reads Slack/email alert secrets from an
-optional gitignored `.env.alerting`.
-
----
-
-## 5.7 Configuration Architecture
-
-> **OTLP** = OpenTelemetry Protocol
-
-```mermaid
-flowchart TB
-    subgraph config["Configuration Sources"]
-        cfgFile["xrpld.cfg<br/>[telemetry] section"]
-        cmake["CMake option: telemetry<br/>ON today for CI, OFF once merged<br/>when ON, defines XRPL_ENABLE_TELEMETRY"]
-    end
-
-    subgraph init["Initialization"]
-        parse["makeTelemetrySetup()"]
-        factory["makeTelemetry()"]
-    end
-
-    subgraph runtime["Runtime Components"]
-        tracer["TracerProvider"]
-        exporter["OTLP Exporter"]
-        processor["BatchProcessor"]
-    end
-
-    subgraph collector["Collector Pipeline"]
-        recv["Receivers"]
-        proc["Processors"]
-        exp["Exporters"]
-    end
-
-    cfgFile --> parse
-    cmake -->|"compile flag"| parse
-    parse --> factory
-    factory --> tracer
-    tracer --> processor
-    processor --> exporter
-    exporter -->|"OTLP"| recv
-    recv --> proc
-    proc --> exp
-
-    style config fill:#e3f2fd,stroke:#1976d2
-    style runtime fill:#e8f5e9,stroke:#388e3c
-    style collector fill:#fff3e0,stroke:#ff9800
-```
-
-**Reading the diagram:**
-
-- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, per-component trace toggles) while the CMake `telemetry` option controls whether telemetry is compiled in at all. That option is declared ON today only so CI compiles the instrumented paths; OFF is the intended default once merged, so treat the build gate as something to pass explicitly, and the runtime gate is opt-in either way (`enabled=0` by default). Head sampling is fixed at 1.0 and is not a config option; volume reduction happens via tail sampling in the collector.
-- **Initialization**: `makeTelemetrySetup()` parses config values, then `makeTelemetry()` constructs the provider, processor, and exporter objects.
-- **Runtime Components**: The `TracerProvider` creates spans, the `BatchProcessor` buffers them, and the `OTLP Exporter` serializes and sends them over the wire.
-- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP/HTTP and enters the external Collector pipeline. (OTLP/gRPC is future work — see design decisions §2.2.2.)
-- **Collector Pipeline**: `Receivers` ingest OTLP data, `Processors` apply sampling/filtering/enrichment, and `Exporters` forward traces to storage backends (Tempo, etc.).
-
----
-
-## 5.8 Grafana Integration
-
-> **APM** = Application Performance Monitoring
-
-Step-by-step instructions for integrating xrpld traces with Grafana.
-
-### 5.8.1 Data Source Configuration
-
-Three datasources are provisioned from `docker/telemetry/grafana/provisioning/datasources/`. There is **no** Elastic APM datasource — `elastic-apm.yaml` was described in an earlier draft but never existed. Elastic remains a _possible_ backend (§7.2); nothing in this repo provisions it.
-
-| File              | Type         | URL                      | uid          | Notes                                                                                                                |
-| ----------------- | ------------ | ------------------------ | ------------ | -------------------------------------------------------------------------------------------------------------------- |
-| `tempo.yaml`      | `tempo`      | `http://tempo:3200`      | `tempo`      | `nodeGraph`, `serviceMap`/`tracesToMetrics` → `prometheus`, `tracesToLogs` → `loki`, plus ~30 Explore search filters |
-| `prometheus.yaml` | `prometheus` | `http://prometheus:9090` | `prometheus` | Backs every span-metric and native-metric panel                                                                      |
-| `loki.yaml`       | `loki`       | `http://loki:3100`       | `loki`       | Backs `log-derived-insights`; derived fields jump back to Tempo                                                      |
-
-The Tempo `tracesToLogs` block is configured as `filterByTraceID: true`,
-`filterBySpanID: false`, **`tags: []`**. The empty tag list is deliberate: the
-correlation is by trace ID alone, so no span attribute needs to exist on both
-sides. Earlier drafts claimed `trace_id` + `tx_hash` tags — that is not what
-ships, and adding a tag Tempo cannot resolve blanks the link.
-
-The search-filter list is the practical index of queryable span attributes:
-resource scope (`service.name`, `service.instance.id`, `service.version`,
-`xrpl.network.id`, `xrpl.network.type`), intrinsics (`name`, `status`,
-`duration`), and span scope (`command`, `rpc_status`, `rpc_role`, `tx_hash`,
-`tx_type`, `tx_status`, `tx_account`, `tx_destination`, `tx_owner`, `tx_issuer`,
-`local`, `path`, `peer_version`,
-`consensus_*`, `ledger_seq`, `ledger_hash`, `close_time_correct`,
-`close_resolution_ms`, `proposers`, `mode_old`, `mode_new`, `txq_status`,
-`ter_code`).
-
-### 5.8.2 Dashboard Provisioning
-
-`grafana/provisioning/dashboards/dashboards.yaml` declares a single `file`
-provider named `xrpld-telemetry`, `orgId: 1`, targeting Grafana folder `xrpld`
-from path `/var/lib/grafana/dashboards` (no `/rippled` suffix), with
-`disableDeletion: false`, `editable: true`, `foldersFromFilesStructure: false`.
-It sets **no** poll interval — Grafana's `updateIntervalSeconds` default
-applies; the "every 30s" figure in earlier drafts was invented.
-
-`docker-compose.yml` mounts `./grafana/dashboards` read-only at that path, so
-the JSON files in `docker/telemetry/grafana/dashboards/` are what gets
-provisioned.
-
-### 5.8.3 Shipped Dashboards
-
-The dashboards are Prometheus-first, not TraceQL-first, and their uids are
-bare (no `xrpld-` prefix). The full inventory and per-panel query reference is
-[09-data-collection-reference.md](./09-data-collection-reference.md); the uids
-are:
-
-`consensus-health`, `fee-market`, `job-queue`, `ledger-data-sync`,
-`ledger-operations`, `ledger-sync-health`, `log-derived-insights`,
-`network-traffic`, `node-health`, `overlay-traffic-detail`, `peer-network`,
-`peer-quality`, `rpc-pathfinding`, `rpc-performance`, `transaction-overview`,
-`validator-health`.
-
-> **Panel-count convention used in these docs**: counts are of **data panels
-> only** — `type: "row"` collapsible headers are excluded, because a row is a
-> layout element with no query. A board's raw `panels` array is therefore longer
-> than its stated count (e.g. `rpc-performance` has 19 array entries: 2 rows +
-> 17 data panels).
-
-Two examples described in earlier drafts do not exist and should not be looked
-for: `xrpld-rpc-performance` (the real board is `rpc-performance`, mostly
-Prometheus span metrics) and `xrpld-tx-tracing` (the transaction board is
-`transaction-overview`; its error panel filters
-`span_calls_total{span_name="tx.process",
-ter_result!~"tesSUCCESS|"}`, since no `tx.validate` span was ever built — see
-[02 §2.3.2](./02-design-decisions.md)).
-
-> **Why `!~"tesSUCCESS|"` and not `!="tesSUCCESS"`.** An absent Prometheus label
-> compares equal to the empty string, and `tx.process` can end **without** a
-> `ter_result` attribute: `processTransaction()` returns early when
-> `preProcessTransaction()` rejects the transaction
-> (`NetworkOPs.cpp:1437-1438`) and `doTransactionAsync()` returns early when the
-> transaction is already applying (`:1461-1462`); the only setter runs later, at
-> `:1674`. Those series carry `ter_result=""`, which `!="tesSUCCESS"` counts as
-> an error. The regex form excludes the empty value explicitly (the trailing
-> `|` alternative), which is the form `docs/telemetry-runbook.md:1198` and two
-> of the three `transaction-overview.json` failure panels already use.
-
-Every dashboard exposes a `$node` template variable bound to
-`service_instance_id`; see the §5.1.1 note on why `service_instance_id` must be
-set for metric panels to split per node.
-
-### 5.8.4 TraceQL Query Examples
-
-Common queries for xrpld traces. Every span name and attribute below is one
-that the code actually emits — check against the `*SpanNames.h` constants
-before adding more.
-
-```
-# Find all traces for a specific transaction hash
-{resource.service.name="xrpld" && span.tx_hash="ABC123..."}
-
-# Find slow RPC commands (>100ms)
-{resource.service.name="xrpld" && name=~"rpc.command.*"} | { duration > 100ms }
-
-# Find consensus rounds taking >5 seconds
-{resource.service.name="xrpld" && name="consensus.round"} | { duration > 5s }
-
-# Find failed transaction processing
-{resource.service.name="xrpld" && name="tx.process" && span.ter_result!="tesSUCCESS"}
-
-# Find failed apply-pipeline stages (preflight / preclaim / transactor)
-{resource.service.name="xrpld" && name=~"tx\\.(preflight|preclaim|transactor)" && status=error}
-
-# Find transactions that arrived from a peer rather than a local client.
-# The `local` attribute lives on tx.process, NOT on tx.receive (see the note
-# below).
-{resource.service.name="xrpld" && name="tx.process" && span.local=false}
-
-# Compare latency across nodes
-{resource.service.name="xrpld" && name="rpc.command.account_info"} | avg_over_time(duration) by (resource.service.instance.id)
-```
-
-> Queries in earlier drafts used `tx.validate`, `tx.relay` and
-> `span.relay_count`. None of the three exists: signature/format validation
-> ships as `tx.preflight`/`tx.preclaim`, and no relay span or relay-count
-> attribute was ever built. See [02 §2.3.2](./02-design-decisions.md).
-
-> **TraceQL silently returns nothing for an absent attribute.** Unlike PromQL,
-> where a missing label compares equal to `""`, a TraceQL attribute predicate
-> matches only spans that actually carry the attribute — including negated
-> forms such as `!=` and `=~".*"`. So filtering on the wrong span name yields
-> zero rows with no error. `local` has exactly one set-site,
-> `NetworkOPs.cpp:1417`, and it is on **`tx.process`**: an earlier draft paired
-> it with `name="tx.receive"`, which can never match. Check the attribute's
-> owning span in
-> [09 §1.2](./09-data-collection-reference.md) before combining a `name=` and a
-> `span.` predicate.
-
-### 5.8.5 Correlation with Logs
-
-Log↔trace correlation is **implemented** (Phase 8) and needs no Promtail,
-Fluentd or PerfLog change. Two pieces:
-
-1. **The node stamps the IDs.** The journal sink `Logs::format()`
-   (`src/libxrpl/basics/Log.cpp:304-338`, guarded by `XRPL_ENABLE_TELEMETRY`)
-   reads the thread-local OTel context and, when a valid span is active,
-   prefixes the message with `trace_id=<32 hex> span_id=<16 hex>`. It reads
-   the context value directly rather than calling `GetSpan()` to avoid a heap
-   allocation on the (common) no-span path. This is the ordinary `debug.log`
-   stream — PerfLog is not involved, and the `setTraceId` hook described in
-   earlier drafts was never built.
-2. **The collector ingests them.** The `file_log` receiver tails
-   `/var/log/xrpld/*/debug.log` and its `regex_parser` lifts `trace_id` and
-   `span_id` as optional capture groups (§5.5.1). `resource/logs` applies an
-   `upsert` of `service.name=xrpld`, which Loki promotes to the stream label
-   `service_name`, so the canonical selector is **`{service_name="xrpld"}`**.
-   Logs land in Loki via `otlp_http/loki`.
-
-> **Known issue — the collector's `job` upsert is ineffective for stream
-> selection.** `resource/logs` also applies an `upsert` of a `job=xrpld` attribute
-> (`otel-collector-config.yaml:62-70`) with the stated intent that operators
-> could paste `{job="xrpld"}`. That does not work. On OTLP ingest Loki promotes
-> only an **allow-listed** set of resource attributes to indexed stream labels
-> (`service.name`, `service.namespace`, `service.instance.id`,
-> `deployment.environment`, the `k8s.*`/`cloud.*` keys); `job` is not on that
-> list, and this repo ships no Loki config override — `docker-compose.yml:116`
-> starts Loki with the image's built-in `/etc/loki/local-config.yaml`. `job`
-> therefore lands in **structured metadata**, which cannot appear in a stream
-> selector, so `{job="xrpld"}` returns an empty result rather than an error.
-> Corroboration in-repo: `docs/telemetry-runbook.md:2533` states the same
-> ("`service_name="xrpld"` (not `job="xrpld"`)"), and **every Loki query** in
-> the shipped dashboards (panel targets and Loki-backed template variables)
-> selects on `service_name` — **zero** use `job`. Either drop the `job`
-> upsert or add `job` to Loki's `distributor.otlp_config.resource_attributes`
-> allow-list via a mounted Loki config; until then, use `service_name`.
-
-Grafana then links the two directions: the Tempo datasource's `tracesToLogs`
-(`filterByTraceID: true`, `tags: []`) jumps trace → logs, and `loki.yaml`'s
-derived fields jump log → trace.
-
-### 5.8.6 Correlation with Insight/OTel System Metrics
-
-To correlate traces with Beast Insight system metrics:
-
-**Step 1: Export Insight metrics to Prometheus**
-
-Beast Insight metrics are exported natively via OTLP to the OTel Collector,
-which exposes them on its Prometheus endpoint (`:8889`) alongside spanmetrics.
-Set `server=otel` in the `[insight]` section of `xrpld.cfg`; no separate StatsD
-exporter or Prometheus scrape job is needed.
-
-`makeCollectorManager()` (`src/xrpld/app/main/CollectorManager.cpp`) reads these
-`[insight]` keys:
-
-| Key                   | Read at      | Effect when `server=otel`                                                                                                          |
-| --------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------- |
-| `server`              | `:35`        | **Live.** `statsd` \| `otel` \| anything else. Selects the collector implementation.                                               |
-| `address`             | `:39`        | StatsD only — the UDP endpoint.                                                                                                    |
-| `prefix`              | `:41`, `:53` | **Inert.** Stored on the OTel collector but `formatName()` prepends nothing (`OTelCollector.cpp:855-866`); only StatsD applies it. |
-| `endpoint`            | `:50`        | **Inert.** Logged for diagnostics (`OTelCollector.cpp:730`), then unused.                                                          |
-| `service_instance_id` | `:58`        | **Inert.** `(void)`-discarded (`OTelCollector.cpp:722`).                                                                           |
-| `service_name`        | `:64`        | **Inert.** `(void)`-discarded (`OTelCollector.cpp:723`).                                                                           |
-
-> **Where the identity and endpoint actually come from.** `OTelCollector`
-> deliberately does **not** own a pipeline: it fetches the Meter from the
-> **global** MeterProvider that `Telemetry::initMetrics()` published
-> (`OTelCollector.cpp:726-745`). So the resource attributes — including
-> `service.instance.id`, which every dashboard filters on — and the exporter
-> URL both come from the **`[telemetry]`** section, not `[insight]`. The four
-> inert keys above are back-compat leftovers from the StatsD-era signature;
-> setting them has no effect. Set `[telemetry] service_instance_id` instead
-> (§5.1.1).
-
-> **`server=otel` is not the default.** `CollectorManager.cpp:72-75` falls through
-> to `NullCollector` for any unrecognised or absent `server` value, so a node
-> with no `[insight]` section emits no metrics at all.
-
-**Step 2: Correlate metrics to traces**
-
-Today this is a **time-range** correlation, not a click-through one: note the
-window from the metric panel, then search Tempo over the same window filtered
-by `service.instance.id`.
-
-> **Exemplars are NOT implemented.** Earlier drafts of this section instructed
-> operators to rely on automatic exemplars, set
-> `exemplarTraceIdDestinations` on the Prometheus datasource, and enable
-> `exemplar: true` on panels. None of that is wired up: the string `exemplar`
-> appears **nowhere** in `src/libxrpl/telemetry/`, `src/xrpld/telemetry/`, or
-> `docker/telemetry/`. Concretely, three things are missing —
->
-> 1. the SDK's exemplar filter is left at its default and no reservoir is
->    configured in `Telemetry::initMetrics()` or `MetricsRegistry`;
-> 2. the collector's `prometheus` exporter has no exemplar settings;
-> 3. `grafana/provisioning/datasources/prometheus.yaml` has no
->    `exemplarTraceIdDestinations` block.
->
-> Note also that the query used as an example, `rpc_duration_seconds_bucket`,
-> does not exist — RPC latency histograms are `span_duration_milliseconds_bucket`
-> (spanmetrics, `unit: ms`) and `rpc_method_us` (native). Wiring exemplars end
-> to end is genuine open work; until it lands, do not document a click-through
-> that operators cannot perform.
-
-**Step 3: Jump the other way instead**
-
-Trace → metrics is available now: the Tempo datasource sets
-`tracesToMetrics.datasourceUid: prometheus` with a ±1h time shift, so the
-span-metric queries it builds resolve against the `span_*` families the
-collector's `spanmetrics` connector produces. Trace → logs and log → trace are
-both live (§5.8.5).
-
-> **Known gap — Service Map is configured but inactive.** The Tempo datasource
-> declares `serviceMap.datasourceUid: prometheus`, and `tempo.yaml:70-76`
-> enables the `service-graphs` metrics-generator processor, but the generator
-> has nowhere to write: its `remote_write` block is **commented out**
-> (`tempo.yaml:53-56`), and `prometheus.yml:6-9` defines a single scrape job
-> against `otel-collector:8889` — it never scrapes or accepts writes from
-> Tempo. `traces_service_graph_request_total` and its siblings are therefore
-> never stored, so the Service Map / Node Graph tab renders empty. The same gap
-> means Tempo's `span-metrics` processor never lands
-> `traces_spanmetrics_*` either (§5.5.1) — every span metric the dashboards use
-> comes from the collector's connector instead. Closing it needs both halves:
-> uncomment `remote_write` in `tempo.yaml` **and** enable
-> `--web.enable-remote-write-receiver` on the Prometheus service (or add a
-> scrape job for Tempo).
-
----
-
-_Previous: [Implementation Strategy](./03-implementation-strategy.md)_ | _Next: [Implementation Phases](./06-implementation-phases.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_

OpenTelemetryPlan/07-observability-backends.md

@@ -1,571 +0,0 @@
-# Observability Backend Recommendations
-
-> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md)
-> **Related**: [Implementation Phases](./06-implementation-phases.md) | [Appendix](./08-appendix.md)
-
----
-
-## 7.1 Development/Testing Backends
-
-> **OTLP** = OpenTelemetry Protocol
-
-| Backend    | Pros                                | Cons                   | Use Case            |
-| ---------- | ----------------------------------- | ---------------------- | ------------------- |
-| **Tempo**  | Cost-effective, Grafana integration | Requires Grafana stack | Local dev, CI, Prod |
-| **Zipkin** | Simple, lightweight                 | Basic features         | Quick prototyping   |
-
-### Quick Start with Tempo
-
-```bash
-# Start Tempo with OTLP support.
-# Version pinned to match docker/telemetry/docker-compose.yml:55 — keep the
-# two in step, since Tempo config keys change between minor releases.
-#
-# Only 4317 (OTLP/gRPC) is published: docker/telemetry/tempo.yaml:28-33
-# declares a single distributor receiver, `otlp.protocols.grpc` on
-# 0.0.0.0:4317. There is no `http` protocol block, so nothing listens on 4318
-# and publishing it would give you a port that silently refuses connections.
-# 3200 is Tempo's HTTP API/query port (tempo.yaml:17-18), not an ingest port.
-docker run -d --name tempo \
-    -p 3200:3200 \
-    -p 4317:4317 \
-    grafana/tempo:2.9.4
-```
-
-> Note that xrpld itself exports OTLP/**HTTP** only (§2.2.1), so it cannot send
-> to this container directly — the collector is what bridges HTTP ingest to
-> Tempo's gRPC receiver (`otlp/tempo` → `tempo:4317`). A bare Tempo container is
-> useful for replaying traces from another OTLP/gRPC producer, not as an xrpld
-> endpoint.
-
-> In practice, prefer the full stack —
-> `docker compose -f docker/telemetry/docker-compose.yml up -d` — over a bare
-> Tempo container. Most shipped dashboards query Prometheus span metrics, which
-> need the collector and Prometheus services too. See
-> [05 §5.6](./05-configuration-reference.md).
-
----
-
-## 7.2 Production Backends
-
-> **APM** = Application Performance Monitoring
-
-| Backend           | Pros                                      | Cons                   | Use Case                    |
-| ----------------- | ----------------------------------------- | ---------------------- | --------------------------- |
-| **Grafana Tempo** | Cost-effective, Grafana integration       | Requires Grafana stack | Most production deployments |
-| **Elastic APM**   | Full observability stack, log correlation | Resource intensive     | Existing Elastic users      |
-| **Honeycomb**     | Excellent query, high cardinality         | SaaS cost              | Deep debugging needs        |
-| **Datadog APM**   | Full platform, easy setup                 | SaaS cost              | Enterprise with budget      |
-
-### Backend Selection Flowchart
-
-```mermaid
-flowchart TD
-    start[Select Backend] --> budget{Budget<br/>Constraints?}
-
-    budget -->|Yes| oss[Open Source]
-    budget -->|No| saas{Prefer<br/>SaaS?}
-
-    oss --> existing{Existing<br/>Stack?}
-    existing -->|Grafana| tempo[Grafana Tempo]
-    existing -->|Elastic| elastic[Elastic APM]
-    existing -->|None| tempo
-
-    saas -->|Yes| enterprise{Enterprise<br/>Support?}
-    saas -->|No| oss
-
-    enterprise -->|Yes| datadog[Datadog APM]
-    enterprise -->|No| honeycomb[Honeycomb]
-
-    tempo --> final[Configure Collector]
-    elastic --> final
-    honeycomb --> final
-    datadog --> final
-
-    style start fill:#0f172a,stroke:#020617,color:#fff
-    style budget fill:#334155,stroke:#1e293b,color:#fff
-    style oss fill:#1e293b,stroke:#0f172a,color:#fff
-    style existing fill:#334155,stroke:#1e293b,color:#fff
-    style saas fill:#334155,stroke:#1e293b,color:#fff
-    style enterprise fill:#334155,stroke:#1e293b,color:#fff
-    style final fill:#0f172a,stroke:#020617,color:#fff
-    style tempo fill:#1b5e20,stroke:#0d3d14,color:#fff
-    style elastic fill:#bf360c,stroke:#8c2809,color:#fff
-    style honeycomb fill:#0d47a1,stroke:#082f6a,color:#fff
-    style datadog fill:#4a148c,stroke:#2e0d57,color:#fff
-```
-
-**Reading the diagram:**
-
-- **Budget Constraints? (Yes)**: Leads to open-source options. If you already run Grafana or Elastic, pick the matching backend; otherwise default to Grafana Tempo.
-- **Budget Constraints? (No) → Prefer SaaS?**: If you want a managed service, choose between Datadog (enterprise support) and Honeycomb (developer-focused). If not, fall back to open-source.
-- **Terminal nodes (Tempo / Elastic / Honeycomb / Datadog)**: Each represents a concrete backend choice, all of which feed into the same final step.
-- **Configure Collector**: Regardless of backend, you always finish by configuring the OTel Collector to export to your chosen destination.
-
----
-
-## 7.3 Recommended Production Architecture
-
-> **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring | **HA** = High Availability
-
-```mermaid
-flowchart TB
-    subgraph validators["Validator Nodes"]
-        v1[xrpld<br/>Validator 1]
-        v2[xrpld<br/>Validator 2]
-    end
-
-    subgraph stock["Stock Nodes"]
-        s1[xrpld<br/>Stock 1]
-        s2[xrpld<br/>Stock 2]
-    end
-
-    subgraph collector["OTel Collector Cluster"]
-        c1[Collector<br/>DC1]
-        c2[Collector<br/>DC2]
-    end
-
-    subgraph backends["Storage Backends"]
-        tempo[(Grafana<br/>Tempo)]
-        elastic[(Elastic<br/>APM)]
-        archive[(S3/GCS<br/>Archive)]
-    end
-
-    subgraph ui["Visualization"]
-        grafana[Grafana<br/>Dashboards]
-    end
-
-    v1 -->|OTLP| c1
-    v2 -->|OTLP| c1
-    s1 -->|OTLP| c2
-    s2 -->|OTLP| c2
-
-    c1 --> tempo
-    c1 --> elastic
-    c2 --> tempo
-    c2 --> archive
-
-    tempo --> grafana
-    elastic --> grafana
-
-    %% Note: simplified single-collector-per-DC topology shown for clarity
-
-    style validators fill:#b71c1c,stroke:#7f1d1d,color:#ffffff
-    style stock fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style collector fill:#bf360c,stroke:#8c2809,color:#ffffff
-    style backends fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style ui fill:#4a148c,stroke:#2e0d57,color:#ffffff
-```
-
-**Reading the diagram:**
-
-- **Validator / Stock Nodes**: All xrpld nodes emit trace data via OTLP. Validators and stock nodes are grouped separately because they may reside in different network zones.
-- **Collector Cluster (DC1, DC2)**: Regional collectors receive OTLP from nodes in their datacenter, apply processing (sampling, enrichment), and fan out to multiple backends. Enrichment includes deployment-tier tagging: each collector stamps `deployment.environment` and (as a fallback) `xrpl.network.type` so one Grafana stack can filter data from many collectors by tier.
-- **Storage Backends**: Tempo and Elastic provide queryable trace storage; S3/GCS Archive provides long-term cold storage for compliance or post-incident analysis.
-- **Grafana Dashboards**: The single visualization layer that queries both Tempo and Elastic, giving operators a unified view of all traces.
-- **Data flow direction**: Nodes → Collectors → Storage → Grafana. Each arrow represents a network hop; minimizing collector-to-backend hops reduces latency.
-
-> **Note**: Production deployments should use multiple collector instances behind a load balancer for high availability. The diagram shows a simplified single-collector topology for clarity.
-
----
-
-## 7.4 Architecture Considerations
-
-### 7.4.1 Collector Placement
-
-| Strategy      | Description          | Pros                     | Cons                    |
-| ------------- | -------------------- | ------------------------ | ----------------------- |
-| **Sidecar**   | Collector per node   | Isolation, simple config | Resource overhead       |
-| **DaemonSet** | Collector per host   | Shared resources         | Complexity              |
-| **Gateway**   | Central collector(s) | Centralized processing   | Single point of failure |
-
-**Recommendation**: Use **Gateway** pattern with regional collectors for xrpld networks:
-
-- One collector cluster per datacenter/region
-- Tail-based sampling at collector level
-- Multiple export destinations for redundancy
-
-### 7.4.2 Sampling Strategy
-
-An earlier version of this section described a three-policy tail sampler (keep
-all errors / keep anything >5s / keep 10% of the rest). **No such sampler
-exists in this repo.** What ships is below.
-
-```mermaid
-flowchart LR
-    subgraph head["Head Sampling (Node) — fixed"]
-        hs["TraceIdRatio 1.0 for root and remote-parent spans<br/>local-parent spans follow the parent<br/>samplingRatio is static constexpr<br/>no config key exists<br/>100% of spans exported"]
-    end
-
-    subgraph tail["Tail Sampling (Collector) — opt-in"]
-        base["Base config:<br/>NO tail_sampling processor<br/>100% of traces stored"]
-        cloud["grafanacloud overlay only:<br/>one probabilistic policy<br/>sampling_percentage: 0.5"]
-    end
-
-    head --> tail
-    base --> final["Stored Traces"]
-    cloud --> final
-
-    style head fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style tail fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style hs fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style base fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style cloud fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style final fill:#bf360c,stroke:#8c2809,color:#ffffff
-```
-
-**Reading the diagram:**
-
-- **Head Sampling (Node)** — fixed at 100% and genuinely not configurable:
-  `Telemetry.h:234` declares `static constexpr double samplingRatio = 1.0;` and
-  `TelemetryConfig.cpp:139` records that there is nothing to parse. This is
-  intentional: a per-node ratio would let different nodes make divergent
-  keep/drop decisions for the same distributed trace, producing broken/partial
-  traces. A span inheriting a remote parent is decided by the same ratio
-  sampler, not by the peer's sampled flag.
-- **Tail Sampling (Collector)** — the base config
-  (`docker/telemetry/otel-collector-config.yaml`) has **no** `tail_sampling`
-  processor, so the local and CI stacks keep 100% of traces. The only shipped
-  policy lives in the `tail_sampling` processor of
-  `otel-collector-config.grafanacloud.yaml`, wired into the **`traces/store`**
-  pipeline — the overlay has no pipeline
-  named `traces`; it splits the trace stream into `traces/metrics` (unsampled,
-  feeds `spanmetrics`) and `traces/store` (sampled, feeds Tempo and Grafana
-  Cloud). See [05 §5.5.2](./05-configuration-reference.md) for the full overlay
-  delta. The policy is a single `probabilistic` at **0.5%**,
-  `decision_wait: 10s`, `num_traces: 300000`, with a decision cache. There
-  are no error or latency carve-outs.
-- **Why 0.5% does not damage the dashboards**: the policy is applied on the
-  trace-storage branch only. The `spanmetrics` connector runs on a separate
-  branch that still sees every span, so `span_calls_total` and
-  `span_duration_milliseconds_*` remain exact. Sampling costs you individual
-  example traces in Tempo, not metric accuracy.
-- **If you want the error/latency policies**: they are a reasonable thing to
-  add, but they must be written — and `decision_wait` sized so a trace's spans
-  have all arrived before the policy evaluates it.
-
-#### Companion guard: `memory_limiter` (recommended, not configured)
-
-Tail sampling bounds what the collector **stores**; it does not bound what the
-collector **buffers**. `tail_sampling` is the opposite of cheap here — it holds
-up to `num_traces` (300 000) traces in memory for `decision_wait` before
-deciding (a few KB each, about 1 GB when full) — and the `spanmetrics` connector keeps a live series cache on top of
-that. A production gateway collector should therefore also run a
-[`memory_limiter`](https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/memorylimiterprocessor/README.md)
-processor as an OOM guard: it applies backpressure (refusing new data with a
-retryable error, which the node's `sending_queue` will retry) instead of letting
-the process be killed and losing every buffered trace.
-
-> **Not currently configured anywhere in this repo.** Neither
-> `otel-collector-config.yaml` nor
-> `otel-collector-config.grafanacloud.yaml` declares a `memory_limiter`, and
-> neither compose file sets a container memory limit — so today a traffic spike
-> is bounded only by host RAM. This is a recommendation for real deployments,
-> recorded here because [05 §5.5.1](./05-configuration-reference.md) lists
-> `memory_limiter` among the processors deliberately **absent** from the shipped
-> config and that must not be read as "not needed". Placement rules if you add
-> it: it must be the **first** processor in every pipeline (ahead of `batch`),
-> and `limit_mib` must sit below the container/cgroup limit with headroom for
-> the sampling and spanmetrics caches.
-
-### 7.4.3 Data Retention
-
-| Environment                 | Hot Storage | Warm Storage | Cold Archive | Source                                                       |
-| --------------------------- | ----------- | ------------ | ------------ | ------------------------------------------------------------ |
-| Development (local stack)   | **1 hour**  | N/A          | N/A          | `tempo.yaml:40` — `compactor.compaction.block_retention: 1h` |
-| Staging (recommendation)    | 7 days      | N/A          | N/A          | Not configured in this repo                                  |
-| Production (recommendation) | 7 days      | 30 days      | many years   | Not configured in this repo                                  |
-
-> **The local stack keeps traces for 1 hour, not 24.** `block_retention: 1h`
-> is deliberate — it bounds disk for a long-running dev node — but it means a
-> trace you found this morning is gone by lunchtime. Raise
-> `block_retention` in `docker/telemetry/tempo.yaml` before starting any
-> investigation that needs to span a working day. The staging and production
-> rows are recommendations only; nothing in this repo provisions them.
-
----
-
-## 7.5 Integration Checklist
-
-- [ ] Choose primary backend (Tempo recommended for cost/features)
-- [ ] Deploy collector cluster with high availability
-- [ ] Configure tail-based sampling for error/latency traces
-- [ ] Set up Grafana dashboards for trace visualization
-- [ ] Configure alerts for trace anomalies
-- [ ] Establish data retention policies
-- [ ] Test trace correlation with logs and metrics
-
----
-
-## 7.6 Grafana Dashboards and Alerts
-
-> **Superseded.** This section was written in Phase 1a, before any dashboard
-> shipped, and described three hypothetical boards (`xrpld-consensus-health`,
-> `xrpld-node-overview`, `xrpld-unified`) and three TraceQL alert rules in a
-> group called `xrpld-tracing-alerts`. **None of those uids or rule names exist
-> anywhere in the repo.** What actually ships is the dashboards and alert rules
-> below, and both are Prometheus-first rather than TraceQL-first. The
-> authoritative references are:
->
-> | For                                              | See                                                                                                       |
-> | ------------------------------------------------ | --------------------------------------------------------------------------------------------------------- |
-> | Dashboard and panel inventory, per-panel queries | [09-data-collection-reference.md](./09-data-collection-reference.md)                                      |
-> | Alert catalogue, thresholds and response steps   | `docs/telemetry-runbook.md`                                                                               |
-> | Files on disk                                    | `docker/telemetry/grafana/dashboards/*.json`, `docker/telemetry/grafana/provisioning/alerting/rules.yaml` |
->
-> The rest of this section records only the facts a reader needs so as not to
-> chase the removed names.
-
-### 7.6.1 Shipped Dashboards
-
-The JSON dashboards are provisioned into Grafana folder `xrpld`. The uids are
-bare — there is no `xrpld-` prefix:
-
-`consensus-health`, `fee-market`, `job-queue`, `ledger-data-sync`,
-`ledger-operations`, `ledger-sync-health`, `log-derived-insights`,
-`network-traffic`, `node-health`, `overlay-traffic-detail`, `peer-network`,
-`peer-quality`, `rpc-pathfinding`, `rpc-performance`, `transaction-overview`,
-`validator-health`.
-
-> **Panel-count convention** (shared with [05 §5.8.3](./05-configuration-reference.md)):
-> counts are of **data panels only**. `type: "row"` collapsible headers are
-> excluded because a row carries no query, so a board's raw `panels` array is
-> longer than its stated count.
-
-`consensus-health.json` is a useful calibration for how far this section drifted:
-where the removed text described "four TraceQL panels", the real board carries **22
-data panels** in 4 rows (26 `panels` array entries) — 19 Prometheus targets
-against `${DS_PROMETHEUS}` and 9 TraceQL targets against `${DS_TEMPO}`. Tempo is
-used for trace _drill-down_; the time series come from span metrics.
-
-### 7.6.2 Shipped Alert Rules
-
-`docker/telemetry/grafana/provisioning/alerting/rules.yaml` provisions the
-rules in the groups below, all in folder `xrpld`, all `interval: 1m`, and all
-**PromQL** — there are zero TraceQL alert rules.
-
-| Group              | Rules                                                                                                   |
-| ------------------ | ------------------------------------------------------------------------------------------------------- |
-| `xrpld-consensus`  | `LedgerHistoryMismatch`, `LedgerCloseStalled`, `ValidatedLedgerStale`                                   |
-| `xrpld-validator`  | `ValidationsMissed`, `ValidationsNotChecked`                                                            |
-| `xrpld-jobqueue`   | `JobQueueTxOverflow`, `JobQueueLatencyHigh`, `IOEventLoopLatencyHigh`                                   |
-| `xrpld-node-state` | `NodeStateFlapping`, `NodeNotFull`                                                                      |
-| `xrpld-overlay`    | `ManifestJobQueueConvoy`, `ManifestFloodInbound`, `PeerResourceDisconnects`, `UntrustedValidationFlood` |
-
-> Two placements are worth noting because they are not what the rule names
-> suggest. `ValidatedLedgerStale` is grouped under `xrpld-consensus`, not
-> `xrpld-validator` — it fires on any node whose validated-ledger sequence stops
-> advancing, which is a chain-progress symptom rather than a validator-identity
-> one. `IOEventLoopLatencyHigh` sits in `xrpld-jobqueue` because, like
-> `JobQueueLatencyHigh`, it measures how long queued work waits before it runs.
-
-Thresholds, measured baselines and response procedures are in the runbook's
-alert catalogue, not here.
-
-### 7.6.3 Writing New Rules: the metric name
-
-If you add a span-metric alert, the metric is **`span_calls_total`**. This stack
-sets the `span_metrics` connector's `namespace: "span"`
-(`otel-collector-config.yaml:114`); the connector's own default namespace is
-**empty**, so without that setting the names would be the bare `calls_total` /
-`duration_milliseconds_*`. Several dashboards already query the `span_` names.
-Durations are likewise `span_duration_milliseconds_bucket`.
-
-> **`traces_spanmetrics_*` is a different producer, not the connector's
-> default.** That family is emitted by **Tempo's** `metrics_generator`
-> `span-metrics` processor (`tempo.yaml:70-76`), which is a separate
-> implementation from the collector connector. It does not exist in this stack
-> either: the generator's `remote_write` is commented out (`tempo.yaml:53-56`)
-> and `prometheus.yml:6-9` scrapes only `otel-collector:8889`, so nothing stores
-> what Tempo generates. Do not write a rule against `traces_spanmetrics_*` and
-> do not describe `namespace: "span"` as overriding it.
-
-An RPC error-rate rule, written against the real metric name, looks like this.
-Note that error _rate_ is a ratio, so it must divide the error-span rate by the
-total-span rate — a bare rate returns calls/second and would fire on traffic
-volume alone:
-
-```
-sum(rate(span_calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m]))
-/
-sum(rate(span_calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m]))
-> 0.05
-```
-
-> **Prefer PromQL over TraceQL for alerting.** TraceQL aggregates
-> (`avg(duration)`, `rate()`) need Tempo 2.3+ with TraceQL metrics enabled, are
-> slower, and are distorted by any tail sampling in the path (§7.4.2). Span
-> metrics are computed pre-sampling and cost nothing extra to query. That is
-> why every shipped rule is PromQL.
-
----
-
-## 7.7 PerfLog and Insight Correlation
-
-> **OTLP** = OpenTelemetry Protocol
-
-How to correlate OpenTelemetry traces with existing xrpld observability.
-
-### 7.7.1 Correlation Architecture
-
-There is **one** collection agent, not three. Earlier drafts of this diagram
-routed logs through "Promtail/Fluentd" and metrics through a "StatsD Exporter";
-neither exists in this stack. Logs are read by the OTel Collector's own
-`file_log` receiver, and `beast::insight` metrics arrive at the same collector
-over OTLP (`[insight] server=otel`). The single-agent shape is the point: one
-process, one config file, one place to add redaction or tier tagging.
-
-```mermaid
-flowchart TB
-    subgraph xrpld["xrpld Node"]
-        otel["OpenTelemetry Spans"]
-        journal["Journal debug.log<br/>trace_id= span_id= prefix<br/>(Log.cpp:304-338)"]
-        insight["Beast Insight + XRPL_METRIC_*<br/>native OTLP metrics"]
-    end
-
-    otelc["OTel Collector<br/>receivers: otlp, file_log<br/>connector: spanmetrics<br/>3 pipelines"]
-
-    subgraph storage["Storage"]
-        tempo[("Tempo")]
-        loki[("Loki")]
-        prom[("Prometheus")]
-    end
-
-    dashboards["Grafana<br/>Tempo to Loki via tracesToLogs<br/>Loki to Tempo via derived fields"]
-
-    otel -->|"OTLP/HTTP :4318"| otelc
-    journal -->|"file_log tails<br/>/var/log/xrpld"| otelc
-    insight -->|"OTLP/HTTP :4318"| otelc
-
-    otelc -->|"otlp/tempo"| tempo
-    otelc -->|"otlp_http/loki"| loki
-    otelc -->|"prometheus :8889"| prom
-
-    tempo --> dashboards
-    loki --> dashboards
-    prom --> dashboards
-
-    style xrpld fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style storage fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style otel fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style journal fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style insight fill:#0d47a1,stroke:#082f6a,color:#ffffff
-    style otelc fill:#bf360c,stroke:#8c2809,color:#ffffff
-    style tempo fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style loki fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style prom fill:#1b5e20,stroke:#0d3d14,color:#ffffff
-    style dashboards fill:#4a148c,stroke:#2e0d57,color:#ffffff
-```
-
-**Reading the diagram:**
-
-- **xrpld Node (three signals, one transport)**: spans and metrics both leave over OTLP/HTTP on port 4318. Logs do not leave the node at all — the node just writes `debug.log`, and the journal sink prefixes `trace_id=`/`span_id=` whenever a span is active (`Log.cpp:304-338`).
-- **OTel Collector (single agent)**: an `otlp` receiver takes spans and metrics; a `file_log` receiver tails `/var/log/xrpld/*/debug.log` and regex-parses the trace/span IDs out of each line. A `spanmetrics` connector derives RED metrics from the trace stream and feeds them into the metrics pipeline. Three pipelines, three exporters — see [05 §5.5.1](./05-configuration-reference.md).
-- **PerfLog is not in this picture.** It still writes `perf.log`, but nothing collects it and it carries no trace ID; the `setTraceId` hook once planned for it was never built ([02 §2.6.5](./02-design-decisions.md)).
-- **StatsD is not in this picture either.** It remains a supported `[insight] server=` choice, but selecting it takes metrics _out_ of this pipeline and requires a StatsD receiver you would have to add yourself — the compose file's StatsD port mapping is commented out.
-- **Grafana**: correlation is bidirectional and configured in the datasources, not in a bespoke panel — Tempo's `tracesToLogs` (`filterByTraceID: true`) jumps trace → logs, and `loki.yaml`'s derived fields jump log → trace.
-
-### 7.7.2 Correlation Fields
-
-| Source          | Field                 | Link To | Status                                                                                                                                                                                                                                                            |
-| --------------- | --------------------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Trace**       | `trace_id`            | Logs    | **Live.** Tempo `tracesToLogs`, `filterByTraceID: true`                                                                                                                                                                                                           |
-| **Trace**       | `tx_hash`             | —       | Live as a span attribute for search; **not** used as a cross-signal join key (`tags: []`)                                                                                                                                                                         |
-| **Trace**       | `ledger_seq`          | —       | Live as a span attribute; not a join key                                                                                                                                                                                                                          |
-| **Journal log** | `trace_id`, `span_id` | Traces  | **Live.** Emitted by `Log.cpp:304-338` into `debug.log`, parsed by the collector's `file_log` receiver, jumped via `loki.yaml` derived fields                                                                                                                     |
-| **PerfLog**     | `trace_id`            | Traces  | **Not implemented.** PerfLog output has no trace ID; the planned `setTraceId` hook was never built. Use the journal log instead                                                                                                                                   |
-| **Insight**     | `exemplar.trace_id`   | Traces  | **Not implemented.** No exemplar configuration exists anywhere in the code or collector config — no `exemplar_filter` on the SDK side, no `exemplarTraceIdDestinations` on the Prometheus datasource. Metric spike → trace jumps must be done by time range today |
-
-### 7.7.3 Example: Debugging a Slow Transaction
-
-**Step 1: Find the trace**
-
-```
-# In Grafana Explore with Tempo
-{resource.service.name="xrpld" && span.tx_hash="ABC123..."}
-```
-
-**Step 2: Get the trace_id from the trace view**
-
-```
-Trace ID: 4bf92f3577b34da6a3ce929d0e0e4736
-```
-
-**Step 3: Find related log lines**
-
-```
-# In Grafana Explore with Loki. `service_name` is the promoted stream label;
-# do NOT use {job="xrpld"} — see the note below.
-{service_name="xrpld"} |= "4bf92f3577b34da6a3ce929d0e0e4736"
-```
-
-These are journal (`debug.log`) lines, not PerfLog lines — see §7.7.2.
-
-> **Known issue — `{job="xrpld"}` does not select anything.** The collector's
-> `resource/logs` processor does upsert a `job=xrpld` resource attribute
-> (`otel-collector-config.yaml:62-70`), explicitly so that operators could paste
-> `{job="xrpld"}`. Loki does not cooperate: on OTLP ingest it promotes only an
-> **allow-listed** set of resource attributes to indexed stream labels
-> (`service.name`, `service.namespace`, `service.instance.id`,
-> `deployment.environment`, `k8s.*`, `cloud.*`), and `job` is not on it. This
-> repo mounts no Loki config override (`docker-compose.yml:116` uses the image's
-> built-in `local-config.yaml`), so `job` lands in **structured metadata** —
-> queryable only with a `|` filter after a selector, never as the selector
-> itself. A `{job="xrpld"}` query returns empty with no error, which is why this
-> is easy to miss. `docs/telemetry-runbook.md:2533` says the same, and every
-> Loki query in the shipped dashboards (panel targets and template
-> variables) selects on `service_name` — zero use `job`. Fix options:
-> drop the ineffective `job` upsert, or mount a Loki config adding `job` to
-> `distributor.otlp_config.resource_attributes`.
-
-**Step 4: Check metrics for the time window**
-
-```
-# In Grafana with Prometheus. Span-derived RED metrics for the transaction
-# pipeline (namespace "span" — see 7.6.3):
-sum(rate(span_calls_total{span_name="tx.process"}[1m])) by (service_instance_id)
-
-# Error share of the same pipeline. Note !~"tesSUCCESS|" — NOT
-# !="tesSUCCESS" — so spans that carry no ter_result are excluded:
-sum(rate(span_calls_total{span_name="tx.process", ter_result!~"tesSUCCESS|"}[5m]))
-/
-sum(rate(span_calls_total{span_name="tx.process"}[5m]))
-```
-
-> **Why the regex form.** An absent Prometheus label is indistinguishable from
-> the empty string, and `tx.process` can end **without** a `ter_result`: the span
-> is opened at `NetworkOPs.cpp:1416`, but `processTransaction()` returns early
-> when `preProcessTransaction()` rejects the transaction (`:1437-1438`), and
-> `doTransactionAsync()` returns early when the transaction is already applying
-> (`:1461-1462`) — both before the only setter, at `:1674`. Those series arrive
-> with `ter_result=""`, which `!="tesSUCCESS"` happily counts as a failure and
-> inflates the ratio. `!~"tesSUCCESS|"` excludes the empty value via the trailing
-> `|` alternative. This is the form `docs/telemetry-runbook.md:1198` and the
-> `transaction-overview.json` stage-failure panels already use; apply it to any
-> new `ter_result` predicate.
-
-> Earlier drafts used `rate(xrpld_tx_applied_total[1m])` and
-> `rate(xrpld_tx_received_total[5m])`. **Neither metric exists** — there is no
-> `xrpld_`-prefixed metric family at all, because `OTelCollector::formatName()`
-> deliberately prepends no prefix (`OTelCollector.cpp:855-866`); the OTel
-> resource `service.name` identifies the service instead. Use the `span_*`
-> families above (verified in `transaction-overview.json` and
-> `rpc-performance.json`) or the native `XRPL_METRIC_*` instrument names listed
-> in [09-data-collection-reference.md](./09-data-collection-reference.md).
-
-### 7.7.4 Unified Dashboard
-
-> **Superseded.** No `xrpld-unified` dashboard exists. The single-pane view it
-> described is instead delivered by two things that did ship: the
-> **`log-derived-insights`** dashboard (all Loki-backed) plus the
-> bidirectional datasource links (Tempo `tracesToLogs` → Loki, `loki.yaml`
-> derived fields → Tempo), which let you cross signals from _any_ board rather
-> than only from one dedicated dashboard.
->
-> The correlation fields those links rely on — and which of them are actually
-> implemented — are in §7.7.2. For the full board inventory see
-> [09-data-collection-reference.md](./09-data-collection-reference.md).
-
----
-
-_Previous: [Implementation Phases](./06-implementation-phases.md)_ | _Next: [Appendix](./08-appendix.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_

OpenTelemetryPlan/08-appendix.md

@@ -1,220 +0,0 @@
-# Appendix
-
-> **Parent Document**: [OpenTelemetryPlan.md](./OpenTelemetryPlan.md)
-> **Related**: [Observability Backends](./07-observability-backends.md)
-
----
-
-## 8.1 Glossary
-
-> **OTLP** = OpenTelemetry Protocol | **TxQ** = Transaction Queue
-
-| Term                  | Definition                                                 |
-| --------------------- | ---------------------------------------------------------- |
-| **Span**              | A unit of work with start/end time, name, and attributes   |
-| **Trace**             | A collection of spans representing a complete request flow |
-| **Trace ID**          | 128-bit unique identifier for a trace                      |
-| **Span ID**           | 64-bit unique identifier for a span within a trace         |
-| **Context**           | Carrier for trace/span IDs across boundaries               |
-| **Propagator**        | Component that injects/extracts context                    |
-| **Sampler**           | Decides which traces to record                             |
-| **Exporter**          | Sends spans to backend                                     |
-| **Collector**         | Receives, processes, and forwards telemetry                |
-| **OTLP**              | OpenTelemetry Protocol (wire format)                       |
-| **W3C Trace Context** | Standard HTTP headers for trace propagation                |
-| **Baggage**           | Key-value pairs propagated across service boundaries       |
-| **Resource**          | Entity producing telemetry (service, host, etc.)           |
-| **Instrumentation**   | Code that creates telemetry data                           |
-
-### xrpld-Specific Terms
-
-| Term              | Definition                                                    |
-| ----------------- | ------------------------------------------------------------- |
-| **Overlay**       | P2P network layer managing peer connections                   |
-| **Consensus**     | XRP Ledger consensus algorithm (RCL)                          |
-| **Proposal**      | Validator's suggested transaction set for a ledger            |
-| **Validation**    | Validator's signature on a closed ledger                      |
-| **HashRouter**    | Component for transaction deduplication                       |
-| **JobQueue**      | Thread pool for asynchronous task execution                   |
-| **PerfLog**       | Existing performance logging system in xrpld                  |
-| **Beast Insight** | Existing metrics framework in xrpld                           |
-| **PathFinding**   | Payment path computation engine for cross-currency payments   |
-| **TxQ**           | Transaction queue managing fee-based prioritization           |
-| **LoadManager**   | Dynamic fee escalation based on network load                  |
-| **SHAMap**        | SHA-256 hash-based map (Merkle trie variant) for ledger state |
-
-### Phase 9–11 Terms
-
-| Term                        | Definition                                                                |
-| --------------------------- | ------------------------------------------------------------------------- |
-| **MetricsRegistry**         | Centralized class for OTel async gauge registrations (Phase 9)            |
-| **ObservableGauge**         | OTel Metrics SDK async instrument polled via callback at fixed intervals  |
-| **PeriodicMetricReader**    | OTel SDK component that invokes gauge callbacks at configurable intervals |
-| **CountedObject**           | xrpld template that tracks live instance counts via atomic counters       |
-| **TxQ**                     | Transaction queue managing fee escalation and ordering                    |
-| **Load Factor**             | Combined multiplier affecting transaction cost (local, cluster, network)  |
-| **OTel Collector Receiver** | Custom Go plugin that polls xrpld RPC and emits OTel metrics (Phase 11)   |
-
----
-
-## 8.2 Span Hierarchy Visualization
-
-The authoritative span-flow diagrams — a master overview plus per-stage
-flowcharts (ingress, the shared apply pipeline, the consensus round, ledger
-finalize, and the pathfinding / ledger-acquire side flows) — live in the operator
-runbook. They map every span onto the **real xrpld control flow and XRPL protocol
-order** (verified against code and `docs/consensus.md`, with file:line evidence),
-label every node and branch with the span that represents that state or
-transition, and call out where the OpenTelemetry span parent links diverge from
-that flow.
-
-> **See**: [docs/telemetry-runbook.md § Protocol Span Flow](../docs/telemetry-runbook.md#protocol-span-flow).
-
-The full span inventory (names, attributes, parents as instrumented) is in
-[09-data-collection-reference.md §1](./09-data-collection-reference.md#1-opentelemetry-spans).
-
----
-
-## 8.3 References
-
-> **OTLP** = OpenTelemetry Protocol
-
-### OpenTelemetry Resources
-
-1. [OpenTelemetry C++ SDK](https://github.com/open-telemetry/opentelemetry-cpp)
-2. [OpenTelemetry Specification](https://opentelemetry.io/docs/specs/otel/)
-3. [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/)
-4. [OTLP Protocol Specification](https://opentelemetry.io/docs/specs/otlp/)
-
-### Standards
-
-5. [W3C Trace Context](https://www.w3.org/TR/trace-context/)
-6. [W3C Baggage](https://www.w3.org/TR/baggage/)
-7. [Protocol Buffers](https://protobuf.dev/)
-
-### xrpld Resources
-
-8. [xrpld Source Code](https://github.com/XRPLF/rippled)
-9. [XRP Ledger Documentation](https://xrpl.org/docs/)
-10. [xrpld Overlay README](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/README.md)
-11. [xrpld RPC README](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/README.md)
-12. [xrpld Consensus README](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/README.md)
-
----
-
-## 8.4 Version History
-
-| Version | Date       | Author | Changes                                                        |
-| ------- | ---------- | ------ | -------------------------------------------------------------- |
-| 1.0     | 2026-02-12 | -      | Initial implementation plan                                    |
-| 1.1     | 2026-02-13 | -      | Refactored into modular documents                              |
-| 1.2     | 2026-03-09 | -      | Added Phases 9–11 (future enhancement plans)                   |
-| 1.3     | 2026-03-24 | -      | Review fixes: accuracy corrections, cross-document consistency |
-
----
-
-## 8.5 Document Index
-
-### Plan Documents
-
-| Document                                                             | Description                                  |
-| -------------------------------------------------------------------- | -------------------------------------------- |
-| [OpenTelemetryPlan.md](./OpenTelemetryPlan.md)                       | Master overview and executive summary        |
-| [00-tracing-fundamentals.md](./00-tracing-fundamentals.md)           | Distributed tracing concepts and OTel primer |
-| [01-architecture-analysis.md](./01-architecture-analysis.md)         | xrpld architecture and trace points          |
-| [02-design-decisions.md](./02-design-decisions.md)                   | SDK selection, exporters, span conventions   |
-| [03-implementation-strategy.md](./03-implementation-strategy.md)     | Directory structure, performance analysis    |
-| [05-configuration-reference.md](./05-configuration-reference.md)     | xrpld config, CMake, Collector configs       |
-| [06-implementation-phases.md](./06-implementation-phases.md)         | Timeline, tasks, risks, success metrics      |
-| [07-observability-backends.md](./07-observability-backends.md)       | Backend selection and architecture           |
-| [08-appendix.md](./08-appendix.md)                                   | Glossary, references, version history        |
-| [09-data-collection-reference.md](./09-data-collection-reference.md) | Span/metric/dashboard inventory              |
-
-### Task Lists
-
-| Document                                                                   | Description                                    |
-| -------------------------------------------------------------------------- | ---------------------------------------------- |
-| [Phase2_taskList.md](./Phase2_taskList.md)                                 | RPC layer trace instrumentation                |
-| [Phase3_taskList.md](./Phase3_taskList.md)                                 | Peer overlay & consensus tracing               |
-| [Phase4_taskList.md](./Phase4_taskList.md)                                 | Transaction lifecycle tracing                  |
-| [Phase5_taskList.md](./Phase5_taskList.md)                                 | Ledger processing & advanced tracing           |
-| [Phase5_IntegrationTest_taskList.md](./Phase5_IntegrationTest_taskList.md) | Observability stack integration tests          |
-| [Phase7_taskList.md](./Phase7_taskList.md)                                 | Native OTel metrics migration                  |
-| [Phase8_taskList.md](./Phase8_taskList.md)                                 | Log-trace correlation                          |
-| [Phase9_taskList.md](./Phase9_taskList.md)                                 | Internal metric instrumentation gap fill       |
-| [Phase10_taskList.md](./Phase10_taskList.md)                               | Synthetic workload generation & validation     |
-| [Phase11_taskList.md](./Phase11_taskList.md)                               | Third-party data collection pipelines (future) |
-
-> **Only Phase 11 is still "future".** Phase 9 ships on
-> `pratik/otel-phase9-metric-gap-fill` (18 task entries, 9.1–9.17 plus 9.7a) and
-> Phase 10 on `pratik/otel-phase10-workload-validation` (7 tasks). Their task
-> lists are present on every branch from those points forward, so a reader on a
-> later branch sees plans that are already implemented, not proposals. Phase 11
-> (13 tasks) has no implementation branch.
-
-> **Note**: Phases 1 and 6 do not have separate task list files. Phase 1 tasks are documented in [06-implementation-phases.md §6.2](./06-implementation-phases.md). Phase 6 tasks are documented in [06-implementation-phases.md §6.7](./06-implementation-phases.md).
-
----
-
-## 8.6 Phase 9–11 Cross-Reference Guide
-
-This guide maps Phase 9–11 content to its location across the documentation.
-
-### Phase 9: Internal Metric Instrumentation Gap Fill
-
-| Content                                           | Location                                                                                                                                                                                                     |
-| ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| Plan & architecture                               | [06-implementation-phases.md §6.8.2](./06-implementation-phases.md)                                                                                                                                          |
-| Task list (18 entries, 9.1–9.17)                  | [Phase9_taskList.md](./Phase9_taskList.md)                                                                                                                                                                   |
-| Metric definitions                                | [09-data-collection-reference.md §5b](./09-data-collection-reference.md)                                                                                                                                     |
-| New classes: `MetricsRegistry`, `AppMetricGauges` | `include/xrpl/telemetry/MetricsRegistry.h` + `src/libxrpl/telemetry/MetricsRegistry.cpp` (pipeline, counters, histograms) and `src/xrpld/telemetry/AppMetricGauges.h/.cpp` (observable gauges) — **shipped** |
-| New dashboards (4)                                | `fee-market`, `job-queue`, `peer-quality`, `validator-health` — **shipped**                                                                                                                                  |
-| Updated dashboards (2)                            | `node-health`, `rpc-performance`                                                                                                                                                                             |
-| Provisioned alert rules                           | `docker/telemetry/grafana/provisioning/alerting/rules.yaml` — 14 rules in 5 groups ([07 §7.6.2](./07-observability-backends.md))                                                                             |
-
-> **Task numbering**: `Phase9_taskList.md` carries 18 `## Task 9.x` headings —
-> 9.1 through 9.17 plus the inserted 9.7a (`push_metrics.py` parity). The "10
-> tasks" figure in earlier revisions predates 9.7a and 9.11–9.17. Tasks 9.8 and
-> 9.11–9.13 together produce the four new dashboards; Task 9.17 (peer span
-> coverage) is explicitly **deferred to Phase 11**.
-
-**Metric categories**: NodeStore I/O, Cache Hit Rates, TxQ, PerfLog Per-RPC, PerfLog Per-Job, Counted Objects, Fee Escalation & Load Factors.
-
-### Phase 10: Synthetic Workload Generation & Telemetry Validation
-
-| Content              | Location                                                                 |
-| -------------------- | ------------------------------------------------------------------------ |
-| Plan & architecture  | [06-implementation-phases.md §6.8.3](./06-implementation-phases.md)      |
-| Task list (7 tasks)  | [Phase10_taskList.md](./Phase10_taskList.md)                             |
-| Branch               | `pratik/otel-phase10-workload-validation`                                |
-| Validation inventory | [09-data-collection-reference.md §5c](./09-data-collection-reference.md) |
-| Test harness         | `docker/telemetry/docker-compose.workload.yaml` (phase-10 branch)        |
-| CI workflow          | `.github/workflows/telemetry-validation.yml` (phase-10 branch)           |
-
-**Validates** (Phase-10 harness inventory): span types, required span attributes, metric entries, dashboards, log-trace correlation.
-
-> **The counts live in the harness manifests, not in this file, because they change from branch to branch.** The manifests (`docker/telemetry/workload/expected_spans.json`, `expected_metrics.json`) are added on the phase-10 branch. In `expected_spans.json`, `total_span_types` is the number of span entries and `total_unique_attributes` is the number of distinct keys across their `required_attributes` lists. `test_contract_totals_count_its_span_entries` in `docker/telemetry/workload/test_validate_telemetry.py` fails when either total does not match the entries. `expected_metrics.json` lists the dashboards the harness checks under `grafana_dashboards.uids`. The full emitted inventory is in [09-data-collection-reference.md §1.1](./09-data-collection-reference.md#11-complete-span-inventory) and [§5c](./09-data-collection-reference.md#validated-telemetry-inventory).
-
-### Phase 11: Third-Party Data Collection Pipelines
-
-| Content                           | Location                                                                 |
-| --------------------------------- | ------------------------------------------------------------------------ |
-| Plan & architecture               | [06-implementation-phases.md §6.8.4](./06-implementation-phases.md)      |
-| Task list (13 tasks)              | [Phase11_taskList.md](./Phase11_taskList.md)                             |
-| External metric definitions (~30) | [09-data-collection-reference.md §5d](./09-data-collection-reference.md) |
-| Custom OTel Collector receiver    | `docker/telemetry/otel-rippled-receiver/` (planned)                      |
-| Prometheus alerting rules (11)    | [09-data-collection-reference.md §5d](./09-data-collection-reference.md) |
-| New dashboards (4)                | Validator Health, Network Topology, Fee Market (External), DEX & AMM     |
-
-> **Two of those names now collide with shipped Phase-9 boards.** Phase 9
-> already ships `validator-health` and `fee-market`, both built from the node's
-> **own** telemetry. The Phase-11 entries are the third-party-data variants
-> (network-wide validator agreement, external fee/DEX feeds via the custom
-> receiver). They need distinct uids, or they will overwrite the Phase-9 boards
-> on provisioning.
-
-**Consumer categories**: Exchanges, Payment Processors, DeFi/AMM, NFT Marketplaces, Analytics Providers, Wallets, Compliance, Academic Researchers, Institutional Custody, CBDC Bridge Operators.
-
----
-
-_Previous: [Observability Backends](./07-observability-backends.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_

OpenTelemetryPlan/OpenTelemetryPlan.md

@@ -1,239 +0,0 @@
-# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld
-
-## Executive Summary
-
-> **OTLP** = OpenTelemetry Protocol
-
-This document provides a comprehensive implementation plan for integrating OpenTelemetry distributed tracing into the xrpld XRP Ledger node software. The plan addresses the unique challenges of a decentralized peer-to-peer system where trace context must propagate across network boundaries between independent nodes.
-
-### Key Benefits
-
-- **End-to-end transaction visibility**: Track transactions from submission through consensus to ledger inclusion
-- **Consensus round analysis**: Understand timing and behavior of consensus phases across validators
-- **RPC performance insights**: Identify slow handlers and optimize response times
-- **Network topology understanding**: Visualize message propagation patterns between peers
-- **Incident debugging**: Correlate events across distributed nodes during issues
-
-### Estimated Performance Overhead
-
-| Metric        | Overhead   | Notes                                            |
-| ------------- | ---------- | ------------------------------------------------ |
-| CPU           | 1-3%       | Span creation and attribute setting              |
-| Memory        | <10 MB     | SDK statics + batch buffer + worker thread stack |
-| Network       | 10-50 KB/s | Compressed OTLP export to collector              |
-| Latency (p99) | <2%        | With proper sampling configuration               |
-
----
-
-## Document Structure
-
-This implementation plan is organized into modular documents for easier navigation:
-
-<div align="center">
-
-```mermaid
-flowchart TB
-    overview["📋 OpenTelemetryPlan.md<br/>(This Document)"]
-
-    subgraph fundamentals["Fundamentals"]
-        fund["00-tracing-fundamentals.md"]
-    end
-
-    subgraph analysis["Analysis & Design"]
-        arch["01-architecture-analysis.md"]
-        design["02-design-decisions.md"]
-    end
-
-    subgraph impl["Implementation"]
-        strategy["03-implementation-strategy.md"]
-        config["05-configuration-reference.md"]
-    end
-
-    subgraph deploy["Deployment & Planning"]
-        phases["06-implementation-phases.md"]
-        backends["07-observability-backends.md"]
-        appendix["08-appendix.md"]
-        dataref["09-data-collection-reference.md"]
-    end
-
-    overview --> fundamentals
-    overview --> analysis
-    overview --> impl
-    overview --> deploy
-
-    fund --> arch
-    arch --> design
-    design --> strategy
-    strategy --> config
-    config --> phases
-    phases --> backends
-    backends --> appendix
-    appendix --> dataref
-
-    style overview fill:#1b5e20,stroke:#0d3d14,color:#fff,stroke-width:2px
-    style fundamentals fill:#00695c,stroke:#004d40,color:#fff
-    style fund fill:#00695c,stroke:#004d40,color:#fff
-    style analysis fill:#0d47a1,stroke:#082f6a,color:#fff
-    style impl fill:#bf360c,stroke:#8c2809,color:#fff
-    style deploy fill:#4a148c,stroke:#2e0d57,color:#fff
-    style arch fill:#0d47a1,stroke:#082f6a,color:#fff
-    style design fill:#0d47a1,stroke:#082f6a,color:#fff
-    style strategy fill:#bf360c,stroke:#8c2809,color:#fff
-    style config fill:#bf360c,stroke:#8c2809,color:#fff
-    style phases fill:#4a148c,stroke:#2e0d57,color:#fff
-    style backends fill:#4a148c,stroke:#2e0d57,color:#fff
-    style appendix fill:#4a148c,stroke:#2e0d57,color:#fff
-    style dataref fill:#4a148c,stroke:#2e0d57,color:#fff
-```
-
-</div>
-
----
-
-## Table of Contents
-
-| Section | Document                                                       | Description                                                            |
-| ------- | -------------------------------------------------------------- | ---------------------------------------------------------------------- |
-| **0**   | [Tracing Fundamentals](./00-tracing-fundamentals.md)           | Distributed tracing concepts, span relationships, context propagation  |
-| **1**   | [Architecture Analysis](./01-architecture-analysis.md)         | xrpld component analysis, trace points, instrumentation priorities     |
-| **2**   | [Design Decisions](./02-design-decisions.md)                   | SDK selection, exporters, span naming, attributes, context propagation |
-| **3**   | [Implementation Strategy](./03-implementation-strategy.md)     | Directory structure, key principles, performance optimization          |
-| **5**   | [Configuration Reference](./05-configuration-reference.md)     | xrpld config, CMake integration, Collector configurations              |
-| **6**   | [Implementation Phases](./06-implementation-phases.md)         | 11-phase timeline, tasks, risks, success metrics                       |
-| **7**   | [Observability Backends](./07-observability-backends.md)       | Backend selection guide and production architecture                    |
-| **8**   | [Appendix](./08-appendix.md)                                   | Glossary, references, version history                                  |
-| **9**   | [Data Collection Reference](./09-data-collection-reference.md) | Complete inventory of spans, attributes, metrics, and dashboards       |
-
-> Note there is no document 4: `04-code-samples.md` was removed during the
-> rollout, and the numbering was left as-is rather than renumbering every
-> cross-reference in the chain.
-
----
-
-## 0. Tracing Fundamentals
-
-This document introduces distributed tracing concepts for readers unfamiliar with the domain. It covers what traces and spans are, how parent-child and follows-from relationships model causality, how context propagates across service boundaries, and how sampling controls data volume. It also maps these concepts to xrpld-specific scenarios like transaction relay and consensus.
-
-➡️ **[Read Tracing Fundamentals](./00-tracing-fundamentals.md)**
-
----
-
-## 1. Architecture Analysis
-
-> **WS** = WebSocket | **TxQ** = Transaction Queue
-
-The xrpld node consists of several key components that require instrumentation for comprehensive distributed tracing. The main areas include the RPC server (HTTP/WebSocket), Overlay P2P network, Consensus mechanism (RCLConsensus), JobQueue for async task execution, PathFinding, Transaction Queue (TxQ), fee escalation (LoadManager), ledger acquisition, validator management, and existing observability infrastructure (PerfLog, Insight/StatsD, Journal logging).
-
-Key trace points span across transaction submission via RPC, peer-to-peer message propagation, consensus round execution, ledger building, path computation, transaction queue behavior, fee escalation, and validator health. The implementation prioritizes high-value, low-risk components first: RPC handlers provide immediate value with minimal risk, while consensus tracing requires careful implementation to avoid timing impacts.
-
-➡️ **[Read full Architecture Analysis](./01-architecture-analysis.md)**
-
----
-
-## 2. Design Decisions
-
-> **OTLP** = OpenTelemetry Protocol | **CNCF** = Cloud Native Computing Foundation
-
-The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/HTTP to an OpenTelemetry Collector, which provides flexible routing and sampling. OTLP/gRPC is planned future work (see design decisions §2.2.2).
-
-Span naming follows a hierarchical `<component>.<operation>` convention (e.g., `rpc.command.server_info`, `tx.process`, `consensus.round`). Context propagation uses W3C Trace Context headers for HTTP and embedded Protocol Buffer fields for P2P messages. The implementation coexists with existing PerfLog and Insight observability systems through correlation IDs.
-
-**Data Collection & Privacy**: Telemetry collects only operational metadata (timing, counts, hashes, public identifiers) — never sensitive content (private keys, balances, amounts, raw payloads). Account addresses are public ledger identifiers and are emitted raw; there is no redaction setting and no collector-side hashing. Trace volume is _not_ reduced on the node (head sampling is fixed at 100%); reduction, where wanted, is a collector-side tail-sampling decision. Node operators control which subsystems are traced via the `[telemetry]` per-component toggles.
-
-➡️ **[Read full Design Decisions](./02-design-decisions.md)**
-
----
-
-## 3. Implementation Strategy
-
-The telemetry code is organized under `include/xrpl/telemetry/` for headers, `src/libxrpl/telemetry/` for implementation, and `src/xrpld/telemetry/` for the native-metrics module added in Phases 7 and 9. Key principles include RAII-based span management via `SpanGuard` (with `discard()` for dropping unwanted spans), a `FilteringSpanProcessor` that intercepts `OnEnd()` to prevent discarded spans from entering the export pipeline, conditional compilation behind the `XRPL_ENABLE_TELEMETRY` compile definition (set by the CMake `telemetry` option, which defaults to **ON** — build it out with `-Dtelemetry=OFF`), and minimal runtime overhead through batch processing.
-
-Performance optimization strategies include head sampling fixed at 100% (intentionally not configurable, so trace keep/drop decisions stay coherent across nodes), optional tail-based sampling at the collector to reduce stored volume (not enabled in the base stack — the only shipped policy is a 0.5% probabilistic one in the Grafana Cloud overlay), batch export to reduce network overhead, and conditional instrumentation that compiles to no-ops when disabled.
-
-➡️ **[Read full Implementation Strategy](./03-implementation-strategy.md)**
-
----
-
-## 5. Configuration Reference
-
-> **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring
-
-Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, TLS/mTLS, batch tuning, and component-level filtering. Exporter selection is _not_ configurable — OTLP/HTTP is the only transport. Head sampling is fixed at 1.0 (not operator-configurable); volume reduction is done by tail sampling in the collector. CMake integration uses the `telemetry` option (default **ON**) for compile-time control.
-
-Endpoints are spread across **three** keys in two sections, not one "traces and metrics" pair:
-
-| Signal                                               | Key                            | Default                            | Source                                              |
-| ---------------------------------------------------- | ------------------------------ | ---------------------------------- | --------------------------------------------------- |
-| Traces                                               | `[telemetry] endpoint`         | `http://localhost:4318/v1/traces`  | `TelemetryConfig.cpp:36,61`                         |
-| Native metrics (`XRPL_METRIC_*` / `MetricsRegistry`) | `[telemetry] metrics_endpoint` | `http://localhost:4318/v1/metrics` | `makeMetricsRegistryOptions()` in `Application.cpp` |
-| `beast::insight` metrics (`server=otel`)             | `[insight] endpoint`           | `http://localhost:4318/v1/metrics` | `CollectorManager.cpp:50`                           |
-
-`[telemetry]` itself has exactly **one** `endpoint` key, and it is traces-only.
-
-The repo ships one collector config (`docker/telemetry/otel-collector-config.yaml`, three pipelines: traces, metrics, logs) plus a Grafana Cloud overlay that adds 0.5% tail sampling. A six-service Docker Compose stack — collector, Tempo, Loki, Prometheus, Grafana, renderer — gives a complete local environment.
-
-➡️ **[View full Configuration Reference](./05-configuration-reference.md)**
-
----
-
-## 6. Implementation Phases
-
-The plan was originally scoped at **13 weeks across 8 phases** — the table below
-is that original scope. As delivered it grew to **11 phases through week 20**;
-Phases 9-11 were added after the original plan was written. See
-[06-implementation-phases.md §6.12.6](./06-implementation-phases.md) for the
-authoritative per-phase status, and treat the eight rows below as the
-originally-planned subset rather than the current timeline:
-
-| Phase | Duration    | Focus                 | Key Deliverables                                          |
-| ----- | ----------- | --------------------- | --------------------------------------------------------- |
-| 1     | Weeks 1-2   | Core Infrastructure   | SDK integration, Telemetry interface, Configuration       |
-| 2     | Weeks 3-4   | RPC Tracing           | HTTP context extraction, Handler instrumentation          |
-| 3     | Weeks 5-6   | Transaction Tracing   | Protocol Buffer context, Relay propagation                |
-| 4     | Weeks 7-8   | Consensus Tracing     | Round spans, Proposal/validation tracing                  |
-| 5     | Week 9      | Documentation         | Runbook, Dashboards, Training                             |
-| 6     | Week 10     | StatsD Metrics Bridge | OTel Collector StatsD receiver, 3 Grafana dashboards      |
-| 7     | Weeks 11-12 | Native OTel Metrics   | OTelCollector impl, OTLP metrics export (StatsD retained) |
-| 8     | Week 13     | Log-Trace Correlation | trace_id in logs, Loki ingestion, Tempo↔Loki linking      |
-
-Delivered beyond the original scope: **Phase 9** (weeks 14-15, internal metric
-instrumentation gap fill), **Phase 10** (weeks 16-17, synthetic workload
-generation and telemetry validation) and **Phase 11** (weeks 18-20, third-party
-data-collection pipelines).
-
-**Total Effort**: 65.1 developer-days with 2 developers, for the eight
-originally-planned phases only.
-
-➡️ **[View full Implementation Phases](./06-implementation-phases.md)**
-
----
-
-## 7. Observability Backends
-
-> **APM** = Application Performance Monitoring | **GCS** = Google Cloud Storage
-
-Grafana Tempo is recommended for all environments due to its cost-effectiveness and Grafana integration, and it is the only backend this repo provisions. Elastic APM remains a reasonable choice for organizations with existing Elastic infrastructure, but nothing here configures it.
-
-The recommended production architecture uses a gateway collector pattern with regional collectors performing tail-based sampling, routing traces to multiple backends (Tempo for primary storage, Elastic for log correlation, S3/GCS for long-term archive). Note that several subsections of doc 7 predate the shipped dashboards and alert rules and are marked superseded in place, pointing at [09-data-collection-reference.md](./09-data-collection-reference.md) and `docs/telemetry-runbook.md`.
-
-➡️ **[View Observability Backend Recommendations](./07-observability-backends.md)**
-
----
-
-## 8. Appendix
-
-The appendix contains a glossary of OpenTelemetry and xrpld-specific terms, references to external documentation and specifications, version history for this implementation plan, and a complete document index.
-
-➡️ **[View Appendix](./08-appendix.md)**
-
----
-
-## 9. Data Collection Reference
-
-A single-source-of-truth reference documenting every piece of telemetry data collected by xrpld: the OpenTelemetry span inventory with per-span attributes, the `beast::insight` and native `XRPL_METRIC_*` instruments (gauges, counters, histograms, overlay traffic), the SpanMetrics-derived Prometheus metrics, and the Grafana dashboards. Includes Tempo search guides and Prometheus query examples. Consult that document rather than this index for any count — it tracks the code, this summary does not.
-
-➡️ **[View Data Collection Reference](./09-data-collection-reference.md)**
-
----
-
-_This document provides a comprehensive implementation plan for integrating OpenTelemetry distributed tracing into the xrpld XRP Ledger node software. For detailed information on any section, follow the links to the corresponding sub-documents._

OpenTelemetryPlan/Phase10_taskList.md

@@ -1,309 +0,0 @@
-# Phase 10: Synthetic Workload Generation & Telemetry Validation — Task List
-
-> **Status**: Future Enhancement
->
-> **Goal**: Build tools that generate realistic XRPL traffic to validate the full Phases 1-9 telemetry stack end-to-end — all spans, attributes, metrics, dashboards, and log-trace correlation — under controlled load.
->
-> **Scope**: Python/shell test harness + multi-node docker-compose environment + automated validation scripts + performance benchmarks.
->
-> **Branch**: `pratik/otel-phase10-workload-validation` (from `pratik/otel-phase9-metric-gap-fill`)
->
-> **Depends on**: Phase 9 (internal metric gap fill) — validates the full metric surface
-
-### Related Plan Documents
-
-| Document                                                             | Relevance                                                       |
-| -------------------------------------------------------------------- | --------------------------------------------------------------- |
-| [06-implementation-phases.md](./06-implementation-phases.md)         | Phase 10 plan: motivation, architecture, exit criteria (§6.8.3) |
-| [09-data-collection-reference.md](./09-data-collection-reference.md) | Defines the full inventory of spans/metrics to validate         |
-| [Phase9_taskList.md](./Phase9_taskList.md)                           | Prerequisite — all internal metrics must be emitting            |
-
-### Why This Phase Exists
-
-Before Phases 1-9 can be considered production-ready, we need proof that:
-
-1. Every emitted span fires with its required attributes under real transaction
-   workloads (the harness takes the span and attribute totals from
-   `expected_spans.json`)
-2. All 255+ StatsD metrics + ~50 Phase 9 metrics appear in Prometheus with non-zero values
-3. Log-trace correlation (Phase 8) produces clickable trace_id links in Loki
-4. Every Grafana dashboard the harness checks renders meaningful data (no empty
-   panels)
-5. Performance overhead stays within bounds (< 3% CPU, < 5MB memory)
-6. The telemetry stack survives sustained load without data loss or queue backpressure
-
----
-
-## Task 10.1: Multi-Node Test Harness
-
-**Objective**: Create a docker-compose environment with 3-5 validator nodes that produces real consensus rounds.
-
-**What to do**:
-
-- Create `docker/telemetry/docker-compose.workload.yaml` — **as shipped this file
-  holds only the observability backend**: `otel-collector`, `tempo`,
-  `prometheus`, `loki`, `grafana`. It contains **no xrpld services**.
-  - Shared network (`workload-net`) with service discovery
-
-- The 5 validators are **native `xrpld` processes**, not containers.
-  `docker/telemetry/workload/run-full-validation.sh` (`NUM_NODES=5`) generates
-  keys, writes a per-node `xrpld.cfg`, and launches each node on
-  `127.0.0.1` with sequential RPC / WS / peer ports. Each node:
-  - Gets its validator key from `generate-validator-keys.sh`
-  - Lists the other 4 nodes in `ips_fixed`
-  - Has all telemetry enabled: `[telemetry] enabled=1`, `[insight] server=otel`
-  - Enables all trace categories including `trace_peer=1`
-  - Writes logs to a file tailed by the OTel Collector filelog receiver
-
-- ❌ **`make telemetry-workload-up` / `make telemetry-workload-down` were never
-  implemented.** There is no `Makefile` anywhere in the repository. The entry
-  point is `run-full-validation.sh` (with `--profile`, `--nodes`,
-  `--skip-loki`, `--skip-regression`, `--with-benchmark`). The node-count flag is
-  spelled `--nodes`, **not** `--num-nodes` — `run-full-validation.sh:80` (usage)
-  and `:100` (the `case` arm). `NUM_NODES` is the internal shell variable it
-  assigns to.
-
-**Key files**:
-
-- New: `docker/telemetry/docker-compose.workload.yaml` (backend only)
-- New: `docker/telemetry/workload/generate-validator-keys.sh`
-- New: `docker/telemetry/workload/run-full-validation.sh` — writes each node's
-  cfg **inline** via a heredoc at `run-full-validation.sh:242`
-  (`cat >"$NODE_DIR/xrpld.cfg" <<EOCFG`)
-- New: `docker/telemetry/workload/xrpld-validator.cfg.template` (96 lines) — it
-  **was** created and is tracked on the Phase 10 branch, but it is **unused**:
-  nothing reads it, and its `{{NODE_INDEX}}` / `{{RPC_PORT}}` / `{{OTEL_ENDPOINT}}`
-  placeholders are never substituted, because the inline heredoc above supersedes
-  it. Either wire the script to the template or delete the template — keeping both
-  guarantees they drift.
-
----
-
-## Task 10.2: RPC Load Generator
-
-**Objective**: Configurable tool that fires all traced RPC commands at controlled rates.
-
-**What to do**:
-
-- Create `docker/telemetry/workload/rpc_load_generator.py`:
-  - Connects to one or more xrpld WebSocket endpoints
-  - Fires all RPC commands that have trace spans: `server_info`, `ledger`, `tx`, `account_info`, `account_lines`, `fee`, `submit`, etc.
-  - Configurable parameters: rate (RPS), duration, command distribution weights
-  - Injects `traceparent` HTTP headers to test W3C context propagation
-  - Logs progress and errors to stdout
-
-- Command distribution should match realistic production ratios:
-  - 40% `server_info` / `fee` (health checks)
-  - 30% `account_info` / `account_lines` / `account_objects` (wallet queries)
-  - 15% `ledger` / `ledger_data` (explorer queries)
-  - 10% `tx` / `account_tx` (transaction lookups)
-  - 5% `book_offers` / `amm_info` (DEX queries)
-
-**Key files**:
-
-- New: `docker/telemetry/workload/rpc_load_generator.py`
-- New: `docker/telemetry/workload/requirements.txt`
-
----
-
-## Task 10.3: Transaction Submitter
-
-**Objective**: Generate diverse transaction types to exercise `tx.*` and `ledger.*` spans.
-
-**What to do**:
-
-- Create `docker/telemetry/workload/tx_submitter.py`:
-  - Pre-funds test accounts from genesis account
-  - Submits a mix of transaction types:
-    - `Payment` (XRP and issued currencies) — exercises `tx.process`, `tx.apply`
-    - `OfferCreate` / `OfferCancel` — DEX activity
-    - `TrustSet` — trust line creation for issued currencies
-    - `NFTokenMint` / `NFTokenCreateOffer` / `NFTokenAcceptOffer` — NFT activity
-    - `EscrowCreate` / `EscrowFinish` — escrow lifecycle
-    - `AMMCreate` / `AMMDeposit` / `AMMWithdraw` — AMM pool operations (if amendment enabled)
-  - Configurable: TPS target, transaction mix weights, duration
-  - Monitors submission results and tracks success/failure rates
-
-- The transaction mix ensures the telemetry captures the full range of ledger activity that third parties care about.
-
-**Key files**:
-
-- New: `docker/telemetry/workload/tx_submitter.py`
-- New: `docker/telemetry/workload/test_accounts.json` (pre-generated keypairs)
-
----
-
-## Task 10.4: Telemetry Validation Suite
-
-**Objective**: Automated scripts that verify all expected telemetry data exists after a workload run.
-
-**What to do**:
-
-- Create `docker/telemetry/workload/validate_telemetry.py`:
-
-  **Span validation** (queries Tempo API):
-  - Assert every required span name in `expected_spans.json` appears in traces.
-    Conditional spans — `grpc.*`, `ledger.acquire`, `txq.*`,
-    `consensus.mode_change`, `rpc.process` — are marked `optional` and skipped
-    when the workload does not exercise them.
-  - Assert each span has its required attributes (bare/underscore keys; dotted
-    `xrpl.*` is reserved for resource attributes)
-  - Assert parent-child relationships are correct. The two live RPC trees are:
-    - HTTP: `rpc.http_request` -> `rpc.process` -> `rpc.command.*`
-    - WebSocket: `rpc.ws_message` -> `rpc.command.*` — **there is no
-      `rpc.process` on the WS path**. `rpc.process` is created only in
-      `ServerHandler::processRequest()` (`ServerHandler.cpp:718`), reached from
-      `processSession(Session, coro)`, i.e. HTTP only. Under WS-only load
-      `rpc.process` never appears, and `rpc.command.*` parents directly to
-      `rpc.ws_message`.
-  - Assert span durations are reasonable (> 0, < 60s)
-
-  **Metric validation** (queries Prometheus API):
-  - Assert all SpanMetrics-derived metrics are non-zero: `span_calls_total`,
-    `span_duration_milliseconds_bucket` (the connector's `namespace` is `span`,
-    not `traces_span_metrics` — `otel-collector-config.yaml:113-114`)
-  - Assert the insight-sourced metrics are non-zero: `ledgermaster_validated_ledger_age`,
-    `peer_finder_active_{inbound,outbound}_peers`, etc. — all lowercase, no
-    `xrpld_` prefix (`77f35c03db` removed the prefix and lowercased names)
-  - Assert all Phase 9 metrics are non-zero: `nodestore_state`, `cache_metrics`,
-    `txq_metrics`, `rpc_method_{started,finished,errored}_total`, `object_count`,
-    `load_factor_metrics`
-  - Assert metric label cardinality is within bounds
-
-  **Log-trace correlation validation** (queries Loki API):
-  - Assert logs contain `trace_id=` and `span_id=` fields
-  - Pick a random trace_id from Tempo → query Loki for matching logs → assert results exist
-  - Assert Grafana derived field links are functional
-
-  **Dashboard validation**:
-  - For each dashboard, query the dashboard API and assert no panels show "No
-    data". The harness checks every dashboard listed under
-    `grafana_dashboards.uids` in `expected_metrics.json`.
-
-- Output: JSON report with pass/fail per check, suitable for CI.
-
-**Key files**:
-
-- New: `docker/telemetry/workload/validate_telemetry.py`
-- New: `docker/telemetry/workload/expected_spans.json` (span inventory for validation)
-- New: `docker/telemetry/workload/expected_metrics.json` (metric inventory for validation)
-
----
-
-## Task 10.5: Performance Benchmark Suite
-
-**Objective**: Measure CPU/memory/latency overhead of the telemetry stack.
-
-**What to do**:
-
-- Create `docker/telemetry/workload/benchmark.sh`:
-  - **Baseline run**: Start cluster with `[telemetry] enabled=0`, run transaction workload for 5 minutes, record metrics
-  - **Telemetry run**: Start cluster with full telemetry enabled, run identical workload, record metrics
-  - **Comparison**: Calculate deltas for:
-    - CPU usage (per-node average)
-    - Memory RSS (per-node peak)
-    - RPC p99 latency
-    - Transaction throughput (TPS)
-    - Consensus round time p95
-    - Ledger close time p95
-
-- Output: Markdown table comparing baseline vs. telemetry, with pass/fail against targets:
-  - CPU overhead < 3%
-  - Memory overhead < 5MB
-  - RPC latency impact < 2ms p99
-  - Throughput impact < 5%
-  - Consensus impact < 1%
-
-- Store results in `docker/telemetry/workload/benchmark-results/` for historical tracking.
-
-**Key files**:
-
-- New: `docker/telemetry/workload/benchmark.sh`
-- New: `docker/telemetry/workload/collect_system_metrics.sh`
-
----
-
-## Task 10.6: CI Integration
-
-**Objective**: Wire the validation suite into CI for regression detection.
-
-**What to do**:
-
-- Create a CI workflow (GitHub Actions or equivalent) that:
-  1. Builds xrpld with `-DXRPL_ENABLE_TELEMETRY=ON`
-  2. Starts the multi-node workload harness
-  3. Runs the RPC load generator + transaction submitter for 2 minutes
-  4. Runs the validation suite
-  5. Runs the benchmark suite
-  6. Fails the build if any validation check fails or benchmark exceeds thresholds
-  7. Archives the validation report and benchmark results as artifacts
-
-- This should be a separate workflow (not part of the main CI), triggered manually or on telemetry-related branch changes.
-
-**Key files**:
-
-- New: `.github/workflows/telemetry-validation.yml`
-- New: `docker/telemetry/workload/run-full-validation.sh` (orchestrator script)
-
----
-
-## Task 10.7: Documentation
-
-**Objective**: Document the workload tools and validation process.
-
-**What to do**:
-
-- Create `docker/telemetry/workload/README.md`:
-  - Quick start guide for running workload harness
-  - Configuration options for load generator and tx submitter
-  - How to read validation reports
-  - How to run benchmarks and interpret results
-
-- Update `docs/telemetry-runbook.md`:
-  - Add "Validating Telemetry Stack" section
-  - Add "Performance Benchmarking" section
-
-- Update `OpenTelemetryPlan/09-data-collection-reference.md`:
-  - Add "Validation" section with expected metric/span counts
-
----
-
-## Exit Criteria — Delivered in PR #6519
-
-- [x] 5-node validator cluster starts and reaches consensus — as native `xrpld`
-      processes driven by `run-full-validation.sh` (`NUM_NODES=5`), not from
-      docker-compose
-- [x] RPC load generator fires all traced RPC commands at configurable rates
-- [x] Transaction submitter generates 6+ transaction types at configurable TPS
-- [x] Validation suite confirms the full span / attribute / metric inventory
-      (totals computed dynamically from `expected_spans.json` /
-      `expected_metrics.json`)
-- [x] Log-trace correlation validated end-to-end (Loki <-> Tempo) — implemented,
-      and gated in CI: the workflow passes no `--skip-loki`, so
-      `validate_telemetry.py` builds and runs both log-correlation checks
-- [x] Every dashboard in `grafana_dashboards.uids` provisions and loads; the
-      check runs no panel query, so an empty panel is not caught
-- [x] Overhead benchmark (`benchmark.sh`) measures telemetry-off vs telemetry-on
-      deltas
-- [ ] Benchmark shows < 3% CPU overhead, < 5MB memory overhead — needs a
-      measured run
-- [x] CI workflow runs validation on telemetry branch changes
-- [x] Validation report output is CI-parseable (JSON with exit codes)
-- [x] OTel-driven regression gate captures per-span and per-job timings from
-      Prometheus and compares against a committed baseline. Per-RPC timings are
-      **not** gated: `regression-metrics.json` defines only `spans` and
-      `job_queue` groups (FU-4).
-
-## Follow-up Work (tracked in separate PRs)
-
-- [ ] FU-2: Automate baseline persistence across CI runs (artifact uploaded
-      on merge to `develop`, downloaded on PR runs). Current mechanism
-      requires a manual baseline-refresh PR.
-- [ ] FU-4: Replace the proxy measurements in `benchmark.sh` (wall-clock curl
-      p99, ledger-cadence-as-TPS, ledger-cadence-as-consensus-p95) with
-      PromQL quantile queries from the same pipeline the regression gate uses,
-      and add an `rpc_methods` group to `regression-metrics.json` plus a
-      `defaults.rpc_method` block to `regression-thresholds.json` (without both,
-      any `rpc.*` metric resolves to "no threshold configured" and never gates).
-- [ ] FU-6: Grafana dashboard plotting historical baseline values keyed by
-      commit SHA, for triaging noisy regressions.

OpenTelemetryPlan/Phase11_taskList.md

@@ -1,664 +0,0 @@
-# Phase 11: Third-Party Data Collection Pipelines — Task List
-
-> **Status**: Not started — 0 of 13 tasks complete (`grep -c '^## Task 11\.'` = 13:
-> Tasks 11.1 through 11.13). Verified against the tree:
-> no `.go` files exist anywhere, `docker/telemetry/otel-rippled-receiver/` does
-> not exist, `docker/telemetry/prometheus/` does not exist (so no
-> `prometheus/rippled-alerts.yml`), and no `network-topology` / `dex-amm`
-> dashboards are present under `docker/telemetry/grafana/dashboards/`. **No Phase 11 work has
-> been done, so no task box below may be ticked.**
->
-> One **prerequisite** box is ticked, and only one: Task 11.12's
-> "`state_tracking` gauge implemented (Task 7.12)". That is an upstream
-> dependency satisfied by Phase 7/9 code, not Phase 11 work — see the citation
-> there.
->
-> **Goal**: Build a custom OTel Collector receiver that periodically polls xrpld's admin RPCs and exports structured metrics for external consumers — making all XRPL health, validator, peer, fee, and DEX data available as Prometheus/OTLP metrics without xrpld code changes.
->
-> **Scope**: Go-based OTel Collector receiver plugin + Grafana dashboards + Prometheus alerting rules.
->
-> **Branch**: `pratik/otel-phase11-third-party-collection` (from `pratik/otel-phase10-workload-validation`)
->
-> **Depends on**: Phase 10 (validation harness for testing the new receiver)
-
-### Related Plan Documents
-
-| Document                                                             | Relevance                                                       |
-| -------------------------------------------------------------------- | --------------------------------------------------------------- |
-| [06-implementation-phases.md](./06-implementation-phases.md)         | Phase 11 plan: motivation, architecture, exit criteria (§6.8.4) |
-| [09-data-collection-reference.md](./09-data-collection-reference.md) | Defines full metric inventory including third-party metrics     |
-| [Phase10_taskList.md](./Phase10_taskList.md)                         | Prerequisite — validation harness for testing                   |
-
-### Third-Party Consumer Gap Analysis
-
-This phase addresses the cross-cutting gap identified during research: **xrpld has no native Prometheus/OTLP metrics export for data accessible only via RPC**. Every consumer (exchanges, payment processors, analytics providers, validators, researchers, compliance firms, custodians) must build custom JSON-RPC polling and conversion. This receiver centralizes that work.
-
-| Consumer Category          | Data Unlocked by This Phase                                        |
-| -------------------------- | ------------------------------------------------------------------ |
-| **Exchanges**              | Real-time fee estimates, TxQ capacity, server health scores        |
-| **Payment Processors**     | Settlement latency percentiles, corridor health, path availability |
-| **Analytics Providers**    | Validator metrics, network topology, amendment voting status       |
-| **DeFi / AMM**             | AMM pool TVL, DEX order book depth, trade volumes                  |
-| **Validators / Operators** | Per-peer latency, version distribution, UNL health, alerting       |
-| **Compliance**             | Transaction volume trends, network growth metrics                  |
-| **Academic Researchers**   | Consensus performance time-series, decentralization metrics        |
-| **CBDC / Tokenization**    | Token supply tracking, trust line adoption, freeze status          |
-| **Institutional Custody**  | Multi-sig status, escrow tracking, reserve calculations            |
-| **Wallet Providers**       | Server health for node selection, fee prediction data              |
-
----
-
-## Task 11.1: OTel Collector Receiver Scaffold
-
-**Objective**: Create the Go project structure for a custom OTel Collector receiver that polls xrpld JSON-RPC.
-
-**What to do**:
-
-- Create `docker/telemetry/otel-rippled-receiver/`:
-  - `receiver.go` — implements `receiver.Metrics` interface
-  - `config.go` — configuration struct (endpoint, poll interval, enabled RPCs)
-  - `factory.go` — receiver factory registration
-  - `go.mod` / `go.sum` — Go module with OTel Collector SDK dependency
-
-- Configuration model:
-
-  ```yaml
-  xrpld_receiver:
-    endpoint: "http://localhost:5005" # xrpld admin RPC
-    poll_interval: 30s # how often to poll
-    enabled_collectors:
-      - server_info
-      - get_counts
-      - fee
-      - peers
-      - validators
-      - feature
-      - server_state
-    amm_pools: [] # optional: AMM pool IDs to track
-    book_offers_pairs: [] # optional: currency pairs for DEX depth
-  ```
-
-- Build a custom OTel Collector binary that includes this receiver alongside the standard receivers.
-
-**Key files**:
-
-- New: `docker/telemetry/otel-rippled-receiver/receiver.go`
-- New: `docker/telemetry/otel-rippled-receiver/config.go`
-- New: `docker/telemetry/otel-rippled-receiver/factory.go`
-- New: `docker/telemetry/otel-rippled-receiver/go.mod`
-- New: `docker/telemetry/otel-rippled-receiver/Dockerfile`
-
----
-
-## Task 11.2: server_info / server_state Collector
-
-**Objective**: Poll `server_info` and `server_state` and export all fields as OTel metrics.
-
-**What to do**:
-
-- Implement `serverInfoCollector` that calls `server_info` (admin) and extracts:
-
-  **Node Health Gauges:**
-  - `xrpl_server_state` (enum → int: disconnected=0, connected=1, syncing=2, tracking=3, full=4, proposing=5)
-  - `xrpl_server_state_duration_seconds`
-  - `xrpl_uptime_seconds`
-  - `xrpl_io_latency_ms`
-  - `xrpl_amendment_blocked` (0 or 1)
-  - `xrpl_peers_count`
-  - `xrpl_peer_disconnects_total`
-  - `xrpl_peer_disconnects_resources_total`
-  - `xrpl_jq_trans_overflow_total`
-
-  **Consensus Gauges:**
-  - `xrpl_last_close_proposers`
-  - `xrpl_last_close_converge_time_seconds`
-  - `xrpl_validation_quorum`
-
-  **Ledger Gauges:**
-  - `xrpl_validated_ledger_seq`
-  - `xrpl_validated_ledger_age_seconds`
-  - `xrpl_validated_ledger_base_fee_drops`
-  - `xrpl_validated_ledger_reserve_base_drops`
-  - `xrpl_validated_ledger_reserve_inc_drops`
-  - `xrpl_close_time_offset_seconds` (0 when absent)
-
-  **Load Factor Gauges:**
-  - `xrpl_load_factor`
-  - `xrpl_load_factor_server`
-  - `xrpl_load_factor_fee_escalation`
-  - `xrpl_load_factor_fee_queue`
-  - `xrpl_load_factor_local`
-  - `xrpl_load_factor_net`
-  - `xrpl_load_factor_cluster`
-
-  **State Accounting Gauges** (per state: disconnected, connected, syncing, tracking, full):
-  - `xrpl_state_duration_seconds{state="<name>"}`
-  - `xrpl_state_transitions_total{state="<name>"}`
-
-  **Validator Info** (when node is a validator):
-  - `xrpl_validator_list_count`
-  - `xrpl_validator_list_expiration_seconds` (epoch)
-  - `xrpl_validator_list_active` (0 or 1)
-
-**Key files**:
-
-- New: `docker/telemetry/otel-rippled-receiver/collectors/server_info.go`
-
----
-
-## Task 11.3: get_counts Collector
-
-**Objective**: Poll `get_counts` and export internal object counts and NodeStore stats.
-
-**What to do**:
-
-- Implement `getCountsCollector`:
-
-  **Database Gauges:**
-  - `xrpl_db_size_kb{db="total"}`, `xrpl_db_size_kb{db="ledger"}`, `xrpl_db_size_kb{db="transaction"}`
-
-  **NodeStore Gauges:**
-  - `xrpl_nodestore_reads_total`, `xrpl_nodestore_reads_hit`, `xrpl_nodestore_writes_total`
-  - `xrpl_nodestore_read_bytes`, `xrpl_nodestore_written_bytes`
-  - `xrpl_nodestore_read_duration_us`, `xrpl_nodestore_write_load`
-  - `xrpl_nodestore_read_queue`, `xrpl_nodestore_read_threads_running`
-
-  **Cache Gauges:**
-  - `xrpl_cache_hit_rate{cache="SLE"}`, `xrpl_cache_hit_rate{cache="ledger"}`, `xrpl_cache_hit_rate{cache="accepted_ledger"}`
-  - `xrpl_cache_size{cache="treenode"}`, `xrpl_cache_size{cache="fullbelow"}`, `xrpl_cache_size{cache="accepted_ledger"}`
-
-  **Object Count Gauges:**
-  - `xrpl_object_count{type="<name>"}` for each counted object type (Transaction, Ledger, NodeObject, STTx, STLedgerEntry, InboundLedger, Pathfinder, etc.)
-
-  **Rates:**
-  - `xrpl_historical_fetch_per_minute`
-  - `xrpl_local_txs`
-
-**Key files**:
-
-- New: `docker/telemetry/otel-rippled-receiver/collectors/get_counts.go`
-
----
-
-## Task 11.4: Peer Topology Collector
-
-**Objective**: Poll `peers` and export per-peer and aggregate network metrics.
-
-**What to do**:
-
-- Implement `peersCollector`:
-
-  **Aggregate Gauges:**
-  - `xrpl_peers_inbound_count`
-  - `xrpl_peers_outbound_count`
-  - `xrpl_peers_cluster_count`
-
-  **Per-Peer Gauges** (with labels `peer_key` truncated to 8 chars for cardinality control):
-  - `xrpl_peer_latency_ms{peer="<key>", version="<ver>", inbound="<bool>"}`
-  - `xrpl_peer_uptime_seconds{peer="<key>"}`
-  - `xrpl_peer_load{peer="<key>"}`
-
-  **Distribution Gauges** (aggregated across all peers):
-  - `xrpl_peer_latency_p50_ms`, `xrpl_peer_latency_p95_ms`, `xrpl_peer_latency_p99_ms`
-  - `xrpl_peer_version_count{version="<semver>"}` — count of peers per software version
-
-  **Tracking Status:**
-  - `xrpl_peer_diverged_count` — peers with `track=diverged`
-  - `xrpl_peer_unknown_count` — peers with `track=unknown`
-
-**Key files**:
-
-- New: `docker/telemetry/otel-rippled-receiver/collectors/peers.go`
-
-**Cardinality note**: Per-peer metrics use truncated keys. For large peer sets (50+), the aggregate distribution gauges are preferred over per-peer labels.
-
----
-
-## Task 11.5: Validator & Amendment Collector
-
-**Objective**: Poll `validators` and `feature` to export validator health and amendment voting status.
-
-**What to do**:
-
-- Implement `validatorCollector`:
-
-  **From `validators` RPC:**
-  - `xrpl_trusted_validators_count`
-  - `xrpl_validator_signing` (0 or 1 — whether local validator is signing)
-
-  **From `feature` RPC:**
-  - `xrpl_amendment_enabled_count` — total enabled amendments
-  - `xrpl_amendment_majority_count` — amendments with majority but not yet enabled
-  - `xrpl_amendment_vetoed_count` — locally vetoed amendments
-  - `xrpl_amendment_unsupported_majority` (0 or 1) — any unsupported amendment has majority (critical alert)
-
-  **Per-amendment with majority** (limited cardinality — only amendments with `majority` set):
-  - `xrpl_amendment_majority_time{name="<amendment>"}` — epoch time when majority was gained
-  - `xrpl_amendment_votes{name="<amendment>"}` — current vote count
-  - `xrpl_amendment_threshold{name="<amendment>"}` — votes needed
-
-**Key files**:
-
-- New: `docker/telemetry/otel-rippled-receiver/collectors/validators.go`
-
----
-
-## Task 11.6: Fee & TxQ Collector
-
-**Objective**: Poll `fee` RPC and export real-time fee market data.
-
-**What to do**:
-
-- Implement `feeCollector` that calls the public `fee` RPC:
-
-  **Fee Level Gauges:**
-  - `xrpl_fee_current_ledger_size` — transactions in current open ledger
-  - `xrpl_fee_expected_ledger_size` — expected transactions at close
-  - `xrpl_fee_max_queue_size` — maximum transaction queue size
-  - `xrpl_fee_open_ledger_fee_drops` — minimum fee for open ledger inclusion
-  - `xrpl_fee_median_fee_drops` — median fee level
-  - `xrpl_fee_minimum_fee_drops` — base reference fee
-  - `xrpl_fee_queue_size` — current queue depth
-
-- This overlaps with Phase 9's internal TxQ metrics but provides an external-only collection path that doesn't require xrpld code changes.
-
-**Key files**:
-
-- New: `docker/telemetry/otel-rippled-receiver/collectors/fee.go`
-
----
-
-## Task 11.7: DEX & AMM Collector (Optional)
-
-**Objective**: Periodically poll configured AMM pools and order book pairs for DeFi metrics.
-
-**What to do**:
-
-- Implement `dexCollector` (enabled only when `amm_pools` or `book_offers_pairs` are configured):
-
-  **AMM Pool Gauges** (per configured pool):
-  - `xrpl_amm_reserve{pool="<id>", asset="<currency>"}` — pool reserve amount
-  - `xrpl_amm_lp_token_supply{pool="<id>"}` — outstanding LP tokens
-  - `xrpl_amm_trading_fee{pool="<id>"}` — pool trading fee (basis points)
-  - `xrpl_amm_tvl_drops{pool="<id>"}` — total value locked (XRP-denominated)
-
-  **Order Book Gauges** (per configured pair):
-  - `xrpl_orderbook_bid_depth{pair="<base>/<quote>"}` — total bid volume
-  - `xrpl_orderbook_ask_depth{pair="<base>/<quote>"}` — total ask volume
-  - `xrpl_orderbook_spread{pair="<base>/<quote>"}` — best bid-ask spread
-  - `xrpl_orderbook_offer_count{pair="<base>/<quote>", side="bid|ask"}` — number of offers
-
-**Key files**:
-
-- New: `docker/telemetry/otel-rippled-receiver/collectors/dex.go`
-
-**Note**: This is optional because it requires explicit configuration of which pools/pairs to track. Default configuration tracks no DEX data.
-
----
-
-## Task 11.8: Prometheus Alerting Rules
-
-**Objective**: Create production-ready alerting rules for the `xrpl_*` metrics
-exported by this receiver.
-
-> **Scope note — do not duplicate Phase 9.** Phase 9 already ships provisioned
-> **Grafana** alerting at
-> `docker/telemetry/grafana/provisioning/alerting/{rules,contactpoints,policies}.yaml`
-> — 14 rules in 5 groups, 2 contact points (`xrpld-default` Slack,
-> `xrpld-critical` Slack + email), and a nested notification policy keyed on
-> `severity = critical`. Four of the rules below overlap it:
->
-> | Rule here           | Addressed by (Phase 9)                            | Coverage                                                                                                         |
-> | ------------------- | ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
-> | `XRPLServerNotFull` | `NodeNotFull` (group `xrpld-node-state`)          | Full                                                                                                             |
-> | `XRPLLedgerStale`   | `ValidatedLedgerStale` (group `xrpld-consensus`)  | **Partial** — Phase 9: `ledgermaster_validated_ledger_age > 60` for 5m; the external shape is `> 30` for 1m      |
-> | `XRPLHighIOLatency` | `IOEventLoopLatencyHigh` (group `xrpld-jobqueue`) | **Partial** — Phase 9: p95 of `ios_latency_milliseconds_bucket` **> 1000 ms for 10m**; external: **> 50 for 1m** |
-> | `XRPLStateFlapping` | `NodeStateFlapping` (group `xrpld-node-state`)    | Full                                                                                                             |
->
-> The remaining 8 (`XRPLAmendmentBlocked`, `XRPLNoPeers`,
-> `XRPLUnsupportedAmendmentMajority`, `XRPLLowPeerCount`, `XRPLHighLoadFactor`,
-> `XRPLSlowConsensus`, `XRPLValidatorListExpiring`, `XRPLClockDrift`) are
-> genuinely new. Note the two sets watch different metric surfaces — the Phase 9
-> rules fire on xrpld's own OTLP metrics, these on the receiver's `xrpl_*`
-> metrics — so if both are kept, dedupe the notification policy to avoid
-> double-paging on the same underlying condition.
->
-> `docker/telemetry/prometheus/` does not exist today. Prefer extending the
-> Phase 9 Grafana provisioning tree over introducing a second, Prometheus-native
-> alerting mechanism; if a `prometheus/` tree is added anyway, say explicitly in
-> its header which alerts it owns.
-
-**What to do**:
-
-- Create `docker/telemetry/prometheus/rippled-alerts.yml`:
-
-  **Tier 1 — Critical (page immediately):**
-
-  ```yaml
-  - alert: XRPLServerNotFull
-    expr: xrpl_server_state < 4
-    for: 15m
-
-  - alert: XRPLAmendmentBlocked
-    expr: xrpl_amendment_blocked == 1
-    for: 1m
-
-  - alert: XRPLNoPeers
-    expr: xrpl_peers_count == 0
-    for: 5m
-
-  - alert: XRPLLedgerStale
-    expr: xrpl_validated_ledger_age_seconds > 120
-    for: 2m
-
-  - alert: XRPLHighIOLatency
-    expr: xrpl_io_latency_ms > 100
-    for: 5m
-
-  - alert: XRPLUnsupportedAmendmentMajority
-    expr: xrpl_amendment_unsupported_majority == 1
-    for: 1m
-  ```
-
-  **Tier 2 — Warning (investigate within hours):**
-
-  ```yaml
-  - alert: XRPLLowPeerCount
-    expr: xrpl_peers_count < 10
-    for: 15m
-
-  - alert: XRPLHighLoadFactor
-    expr: xrpl_load_factor > 10
-    for: 10m
-
-  - alert: XRPLSlowConsensus
-    expr: xrpl_last_close_converge_time_seconds > 6
-    for: 5m
-
-  - alert: XRPLValidatorListExpiring
-    expr: (xrpl_validator_list_expiration_seconds - time()) < 86400
-    for: 1h
-
-  - alert: XRPLClockDrift
-    expr: xrpl_close_time_offset_seconds > 0
-    for: 5m
-
-  - alert: XRPLStateFlapping
-    expr: rate(xrpl_state_transitions_total{state="full"}[1h]) > 2
-    for: 30m
-  ```
-
-**Key files**:
-
-- New: `docker/telemetry/prometheus/rippled-alerts.yml`
-- Update: `docker/telemetry/prometheus/prometheus.yml` (add rule_files reference)
-
----
-
-## Task 11.9: New Grafana Dashboards
-
-**Objective**: Create 4 new dashboards for the data exported by the receiver.
-
-> **UID COLLISION — pick a different uid.** Phase 9 already ships
-> `docker/telemetry/grafana/dashboards/validator-health.json` with
-> **uid `validator-health`** (backed by xrpld's own
-> `validation_agreement` / `validator_health` / `state_tracking` OTLP metrics).
-> Provisioning a second dashboard with the same uid makes Grafana overwrite one
-> with the other — whichever the provisioner loads last wins, silently. Use a
-> distinct uid such as `validator-health-external` (and a distinct filename), the
-> same way this task already disambiguates Fee Market as
-> `xrpld-fee-market-external` against Phase 9's `fee-market`. Also check
-> `peer-quality`, `fee-market`, `job-queue` and `node-health` before adding any
-> further uid.
-
-**What to do**:
-
-- **Validator Health** (`validator-health-external` — **not** `validator-health`,
-  see the collision note above):
-  - Server state timeline, state duration breakdown
-  - Proposer count trend, converge time trend, validation quorum
-  - Validator list expiration countdown
-  - Amendment voting status (majority/enabled/vetoed)
-
-- **Network Topology** (`xrpld-network-topology`):
-  - Peer count (inbound/outbound/cluster), peer version distribution
-  - Peer latency distribution (p50/p95/p99), diverged peer count
-  - Geographic distribution (if enriched with GeoIP)
-  - Peer uptime distribution
-
-- **Fee Market** (`xrpld-fee-market-external`):
-  - Current fee levels (open ledger, median, minimum), fee escalation timeline
-  - Queue depth vs. capacity, transactions per ledger
-  - Load factor breakdown (server/network/cluster/escalation)
-
-- **DEX & AMM Overview** (`xrpld-dex-amm`) (only populated when DEX collectors are configured):
-  - AMM pool TVL, reserve ratios, LP token supply
-  - Order book depth per pair, spread trends
-  - Trading fee revenue estimates
-
-**Key files**:
-
-- New: `docker/telemetry/grafana/dashboards/validator-health-external.json`
-  (**must not** reuse Phase 9's `validator-health.json` / uid `validator-health`)
-- New: `docker/telemetry/grafana/dashboards/network-topology.json`
-- New: `docker/telemetry/grafana/dashboards/fee-market-external.json`
-  (Phase 9 owns `fee-market.json` / uid `fee-market`)
-- New: `docker/telemetry/grafana/dashboards/dex-amm.json`
-
-> Filenames drop the legacy `dashboards/rippled-*` prefix: `145b1469d6` and
-> `25868f2740` renamed every dashboard to bare names with bare uids, so no
-> `dashboards/rippled-*.json` path exists in the tree.
-
----
-
-## Task 11.10: Integration with Phase 10 Validation
-
-**Objective**: Extend the Phase 10 validation suite to verify this receiver's metrics.
-
-**What to do**:
-
-- Update `docker/telemetry/workload/validate_telemetry.py`:
-  - Add assertions for all `xrpl_*` metrics produced by the receiver
-  - Verify metric labels have expected values
-  - Verify alerting rules fire correctly (inject a "bad" state and check alert)
-
-- Update `docker/telemetry/docker-compose.workload.yaml`:
-  - Add the custom OTel Collector build with the xrpld receiver
-  - Configure the receiver to poll one of the test nodes
-
-**Key files**:
-
-- Update: `docker/telemetry/workload/validate_telemetry.py`
-- Update: `docker/telemetry/docker-compose.workload.yaml`
-- Update: `docker/telemetry/workload/expected_metrics.json`
-
----
-
-## Task 11.11: Documentation
-
-**Objective**: Document the receiver, its metrics, deployment, and alerting.
-
-**What to do**:
-
-- Create `docker/telemetry/otel-rippled-receiver/README.md`:
-  - Architecture overview (how the receiver fits into the OTel Collector)
-  - Configuration reference (all config options with defaults)
-  - Metric reference table (all exported metrics with types and labels)
-  - Deployment guide (building custom collector binary, docker-compose integration)
-
-- Update `OpenTelemetryPlan/09-data-collection-reference.md`:
-  - Add "Third-Party Metrics (OTel Collector Receiver)" section
-  - Add new Grafana dashboard reference (4 dashboards)
-  - Add alerting rules reference
-
-- Update `docs/telemetry-runbook.md`:
-  - Add "Third-Party Metrics Receiver" troubleshooting section
-  - Add alerting playbook (what to do for each Tier 1/Tier 2 alert)
-
----
-
-## Task 11.12: Alert Rules for External Dashboard Parity Metrics
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — 18 alert rules ported from the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
->
-> **Upstream**: Phase 7 Tasks 7.9-7.16 (metrics), Phase 9 Tasks 9.11-9.13 (dashboards).
-> **Downstream**: None — terminal task in the parity chain.
-
-**Objective**: Add Grafana alerting rules for the Phase 7+ parity metrics (validation agreement, validator health, peer quality, state tracking, ledger economy). These complement Task 11.8's `xrpl_*` alerts by covering the internal metrics.
-
-> **4 of the 18 are addressed by Phase 9** — 2 fully, 2 only partially. Extend,
-> do not blindly re-create:
->
-> | Rule here          | Addressed by (Phase 9)                                                                      | Coverage                                                                                                                                                                    |
-> | ------------------ | ------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-> | Unhealthy State    | `NodeNotFull` (group `xrpld-node-state`)                                                    | Full                                                                                                                                                                        |
-> | High IO Latency    | `IOEventLoopLatencyHigh` (group `xrpld-jobqueue`, p95 of `ios_latency_milliseconds_bucket`) | **Partial** — Phase 9 fires at p95 **> 1000 ms for 10m**; the rule below wants **> 50 for 1m** (20× tighter)                                                                |
-> | Job Queue Overflow | `JobQueueTxOverflow` (group `xrpld-jobqueue`, `jq_trans_overflow_total`)                    | Full                                                                                                                                                                        |
-> | Stale Ledger       | `ValidatedLedgerStale` (group `xrpld-consensus`, `ledgermaster_validated_ledger_age`)       | **Partial** — different metric: Phase 9 uses `ledgermaster_validated_ledger_age > 60` for 5m; the rule below uses `ledger_economy{metric="ledger_age_seconds"} > 30` for 1m |
->
-> The two **Partial** rows are not closed. Either re-baseline the Phase 9
-> thresholds or ship the tighter variants here — do not skip them as duplicates.
->
-> Remaining open work is **14 rules**, of which **3** (CPU High, Memory Critical,
-> Disk Warning) need `node_exporter`, which is not in the stack. Nothing else is
-> blocked: "Not Proposing" used to be listed as blocked on an unimplemented
-> `state_tracking` gauge, but that gauge **ships** — see the Exit Criteria note
-> below.
->
-> **Metric-name translation.** Names carry **no** `xrpld_` prefix
-> (`77f35c03db`), so as a rule of thumb read every `xrpld_<name>` below as plain
-> `<name>`. **Two shapes do not follow that rule:**
->
-> - **Multiplexed observable gauges.** Many readings are a `metric` **label
->   value** on a shared instrument, not a metric name. `xrpld_txq_count` is
->   `txq_metrics{metric="txq_count"}`; likewise `load_factor_metrics{…}`,
->   `nodestore_state{…}`, `cache_metrics{…}`. The rows below that already use the
->   `<instrument>{metric="…"}` form (`state_tracking`, `validator_health`,
->   `validation_agreement`, `server_info`, `peer_quality`, `load_factor_metrics`,
->   `ledger_economy`) are correct; only drop the prefix on those.
-> - **Unit-suffixed histograms** from `beast::insight`. `OTelCollectorImp` appends
->   the unit, so `xrpld_ios_latency_bucket` is really
->   `ios_latency_milliseconds_bucket` — the spelling used by
->   `node-health.json:577` and `ledger-data-sync.json:1353`.
-
-**Critical Group** (8 rules, eval interval 10s):
-
-| Rule                | Condition                                                        | For |
-| ------------------- | ---------------------------------------------------------------- | --- |
-| Agreement Below 90% | `xrpld_validation_agreement{metric="agreement_pct_24h"} < 90`    | 30s |
-| Not Proposing       | `xrpld_state_tracking{metric="state_value"} < 6`                 | 10s |
-| Unhealthy State     | `xrpld_state_tracking{metric="state_value"} < 4`                 | 10s |
-| Amendment Blocked   | `xrpld_validator_health{metric="amendment_blocked"} == 1`        | 1m  |
-| UNL Expiring        | `xrpld_validator_health{metric="unl_expiry_days"} < 14`          | 1h  |
-| High IO Latency     | `histogram_quantile(0.95, ios_latency_milliseconds_bucket) > 50` | 1m  |
-| High Load Factor    | `xrpld_load_factor_metrics{metric="load_factor"} > 1000`         | 1m  |
-| Peer Count Critical | `xrpld_server_info{metric="peers"} < 5`                          | 1m  |
-
-**Network Group** (3 rules, eval interval 10s):
-
-| Rule                      | Condition                                                         | For |
-| ------------------------- | ----------------------------------------------------------------- | --- |
-| Peer Drop >10%            | `delta(xrpld_server_info{metric="peers"}[30s]) / ... * 100 < -10` | 30s |
-| Peer Drop >30%            | Same formula, threshold -30                                       | 30s |
-| P90 Latency + Disconnects | `peer_latency_p90_ms > 500 AND rate(disconnects) > 0`             | 2m  |
-
-**Performance Group** (7 rules, eval interval 10s):
-
-| Rule                | Condition                                                    | For |
-| ------------------- | ------------------------------------------------------------ | --- |
-| CPU High            | Per-core CPU > 80% (requires node_exporter)                  | 2m  |
-| Memory Critical     | Memory usage > 90% (requires node_exporter)                  | 1m  |
-| Disk Warning        | Disk usage > 85% (requires node_exporter)                    | 2m  |
-| Job Queue Overflow  | `rate(xrpld_jq_trans_overflow_total[5m]) > 0`                | 1m  |
-| Upgrade Recommended | `xrpld_peer_quality{metric="peers_higher_version_pct"} > 60` | 1m  |
-| TX Rate Drop        | Transaction rate dropped > 50% in 5m window                  | 5m  |
-| Stale Ledger        | `xrpld_ledger_economy{metric="ledger_age_seconds"} > 30`     | 1m  |
-
-**Notification channel templates**: Slack and Email/SMTP already ship in Phase
-9's `contactpoints.yaml` (`xrpld-default`, `xrpld-critical`). Discord and
-PagerDuty templates remain open.
-
-**Key files** — extend the **Phase 9** provisioning tree. The
-`docker/telemetry/grafana/alerting/` directory named in the original spec has
-never existed in any commit; the real location is
-`docker/telemetry/grafana/provisioning/alerting/`:
-
-- Extend: `docker/telemetry/grafana/provisioning/alerting/rules.yaml` (add groups
-  alongside the existing `xrpld-consensus`, `xrpld-validator`, `xrpld-jobqueue`,
-  `xrpld-node-state`, `xrpld-overlay`)
-- Extend: `docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml`
-  (add Discord / PagerDuty receivers)
-- Extend: `docker/telemetry/grafana/provisioning/alerting/policies.yaml`
-  (add routes; the root route and the `severity = critical` child already exist)
-
-**Exit Criteria**:
-
-- [ ] The 14 not-yet-shipped rules evaluate without errors in Grafana alerting UI
-- [ ] The 2 rules **fully** covered by Phase 9 (Unhealthy State, Job Queue
-      Overflow) are not duplicated; the 2 **partially** covered ones (High IO
-      Latency, Stale Ledger) are either re-baselined on the Phase 9 rule or shipped
-      as tighter variants — decision recorded either way
-- [ ] Critical rules fire within expected timeframe when conditions are met
-- [ ] Notification channel templates are documented (not hard-coded to any service)
-- [ ] `node_exporter` decision recorded for the 3 host-level rules (CPU, memory, disk)
-- [x] `state_tracking` gauge implemented (Task 7.12) before adding "Not Proposing"
-      — **prerequisite met upstream**, not Phase 11 work.
-      `MetricsRegistry::registerStateTrackingGauge()`
-      (`src/xrpld/telemetry/MetricsRegistry.cpp:1461-1510`) creates
-      `CreateDoubleObservableGauge("state_tracking", "Node state and mode tracking")`
-      at `:1466` and observes `state_value` (`:1497`) and
-      `time_in_current_state_seconds` (`:1502`). Already queried by
-      `validator-health.json:765,971` and `ledger-data-sync.json:869`, and
-      documented in
-      [09-data-collection-reference.md](./09-data-collection-reference.md)
-      § State Tracking. "Not Proposing" can be written now.
-
----
-
-## Task 11.13: Dual-Datasource Architecture Documentation
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
-
-**Objective**: Document the external dashboard's "fast path" pattern as a future optimization for real-time panels.
-
-**Pattern**: A lightweight Prometheus scrape endpoint (separate from OTLP pipeline) that polls critical metrics every 2-5s, bypassing the 10s OTLP metric reader interval and Prometheus scrape interval.
-
-**Use case**: Real-time state panels (server state, ledger age, peer count) where 10-15s latency is too slow for operational dashboards.
-
-**Decision**: Document as a future option, not implement now. The current 10s interval is acceptable for v1. The external dashboard achieves 2-5s freshness by polling RPC directly, which is what the Phase 11 receiver already does. Adding a separate scrape endpoint to xrpld would only be needed if sub-second metric freshness is required from the internal metrics pipeline.
-
-**What to document**:
-
-- Architecture comparison: OTLP pipeline (10-15s) vs. direct scrape (2-5s) vs. push gateway
-- When to consider: operator feedback indicating 10s is insufficient for alerting SLOs
-- How to implement if needed: add `/metrics` HTTP endpoint to xrpld with Prometheus client library
-- Trade-offs: additional port, additional dependency, duplication with OTLP metrics
-
-**Key files**:
-
-- Update: `OpenTelemetryPlan/09-data-collection-reference.md` (add "Future: Dual-Datasource Architecture" section)
-- Update: `docs/telemetry-runbook.md` (add brief note in performance tuning section)
-
-**Exit Criteria**:
-
-- [ ] Architecture comparison documented with clear trade-offs
-- [ ] Decision rationale recorded (why deferred, when to revisit)
-
----
-
-## Exit Criteria
-
-- [ ] Custom OTel Collector receiver builds and starts without errors
-- [ ] All `xrpl_*` metrics from server_info, get_counts, peers, validators, fee appear in Prometheus
-- [ ] Metrics update at configured poll interval (default 30s)
-- [ ] 4 new Grafana dashboards operational with data, none reusing a Phase 9 uid
-      (`validator-health`, `peer-quality`, `fee-market`, `job-queue`, `node-health`)
-- [ ] Prometheus alerting rules fire correctly for simulated failure conditions
-- [ ] DEX/AMM collector works when configured (optional — not required for base exit criteria)
-- [ ] Phase 10 validation suite passes with receiver metrics included
-- [ ] Receiver handles xrpld restart/unavailability gracefully (no crash, logs warning, retries)
-- [ ] Documentation complete: receiver README, metric reference, alerting playbook
-- [ ] Go receiver has unit tests with >80% coverage
-- [ ] The 14 not-yet-shipped Grafana alert rules for Phase 7+ parity metrics
-      evaluate correctly (Task 11.12); the other 4 of the 18 already ship in Phase 9
-- [ ] Dual-datasource architecture documented with trade-offs (Task 11.13)

OpenTelemetryPlan/Phase2_taskList.md

@@ -1,240 +0,0 @@
-# Phase 2: RPC Tracing Completion Task List
-
-> **Goal**: Complete RPC tracing coverage with unit tests, Grafana search filters, PathFind instrumentation, and config hardening. Build on the Phase 1c SpanGuard factory foundation to achieve production-quality RPC observability.
->
-> **Scope**: Unit tests for core telemetry, Grafana Tempo search filters, PathFind RPC tracing, config validation (`std::clamp`).
->
-> **Branch**: `pratik/otel-phase2-rpc-tracing` (from `pratik/otel-phase1c-rpc-integration`)
-
-### Related Plan Documents
-
-| Document                                                         | Relevance                                                                                                                        |
-| ---------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- |
-| [03-implementation-strategy.md](./03-implementation-strategy.md) | Code structure and instrumentation patterns (replaces the deleted `04-code-samples.md` §4.4.2 / §4.5.3, removed by `d6450631bf`) |
-| [02-design-decisions.md](./02-design-decisions.md)               | W3C Trace Context (§2.5), span attributes (§2.4.2)                                                                               |
-| [06-implementation-phases.md](./06-implementation-phases.md)     | Phase 2 tasks (§6.3), definition of done (§6.11.2)                                                                               |
-
----
-
-## Task 2.1: W3C Trace Context HTTP Header Extraction
-
-**Status**: DEFERRED → Phase 3
-
-**Reason**: W3C context propagation (`traceparent`/`tracestate` headers) requires a consumer — in Phase 2, RPC spans are entirely local to the node. Phase 3 introduces cross-node transaction tracing via protobuf context propagation, which is the first use case for extracted trace context. Implementing it here without a consumer would be dead code.
-
-**Implemented in**: `pratik/otel-phase3-tx-tracing` — `TraceContextPropagator.h/.cpp`
-
----
-
-## Task 2.2: Per-Category Span Creation
-
-**Status**: COMPLETE (superseded by Phase 1c design)
-
-**Original plan**: Add `XRPL_TRACE_PEER` and `XRPL_TRACE_LEDGER` macros.
-
-**Actual implementation**: Phase 1c replaced all tracing macros with the `SpanGuard::span(TraceCategory, prefix, name)` factory pattern. The `TraceCategory` enum (`Rpc`, `Transactions`, `Consensus`, `Peer`, `Ledger`) serves the same conditional-creation purpose without macros. No separate task needed — the factory already supports all categories.
-
----
-
-## Task 2.3: Add shouldTraceLedger() to Telemetry Interface
-
-**Objective**: The `Setup` struct has a `traceLedger` field but there's no corresponding virtual method. Add it for interface completeness.
-
-**What to do**:
-
-- Edit `include/xrpl/telemetry/Telemetry.h`:
-  - Add `virtual bool shouldTraceLedger() const = 0;`
-
-- Update all implementations:
-  - `src/libxrpl/telemetry/Telemetry.cpp` (TelemetryImpl, NullTelemetryOtel)
-  - `src/libxrpl/telemetry/NullTelemetry.cpp` (NullTelemetry)
-
-**Key modified files**:
-
-- `include/xrpl/telemetry/Telemetry.h`
-- `src/libxrpl/telemetry/Telemetry.cpp`
-- `src/libxrpl/telemetry/NullTelemetry.cpp`
-
----
-
-## Task 2.4: Unit Tests for Core Telemetry Infrastructure
-
-**Status**: COMPLETE
-
-**Objective**: Add unit tests for the core telemetry abstractions to validate correctness and catch regressions.
-
-**Implemented**:
-
-- `src/tests/libxrpl/telemetry/TelemetryConfig.cpp`:
-  - Test Setup defaults (all fields have correct initial values)
-  - Test `makeTelemetrySetup` config parser (empty section, full section, edge cases)
-  - Test `samplingRatio` clamping (values outside 0.0-1.0)
-
-- `src/tests/libxrpl/telemetry/SpanGuardFactory.cpp`:
-  - Test null guard methods are safe (setAttribute, setOk, setError, addEvent on null)
-  - Test category span returns null when telemetry disabled
-  - Test child/linked span null when no parent context
-  - Test move construction transfers ownership
-  - Test recordException safe on null guard
-  - Test discard() safe on null guard
-
-- `src/tests/libxrpl/telemetry/main.cpp` — GTest runner
-- `src/tests/libxrpl/CMakeLists.txt` — test target with optional OTel linking
-
----
-
-## Task 2.5: Enhance RPC Span Attributes
-
-**Status**: DEFERRED (low priority)
-
-**Reason**: The high-value attributes (`command`, `version`, `role`, `status`) are already set by Phase 1c. The remaining HTTP transport-level attributes (`http.method`, `net.peer.ip`, `http.status_code`) provide limited additional insight since:
-
-- `http.method` is always POST for JSON-RPC
-- `net.peer.ip` is debug-level info available in logs
-- `duration_ms` is redundant with span duration (OTel captures start/end time natively)
-
-These can be added later if dashboard queries specifically need them. The node health attributes (Task 2.8) provide far more operational value and were prioritized instead.
-
----
-
-## Task 2.6: Build Verification and Performance Baseline
-
-**Objective**: Verify the build succeeds with and without telemetry, and establish a performance baseline.
-
-**What to do**:
-
-1. Build with `telemetry=ON` and verify no compilation errors
-2. Build with `telemetry=OFF` and verify no regressions
-3. Run existing unit tests to verify no breakage
-4. Document any build issues in lessons.md
-
-**Verification Checklist**:
-
-- [ ] `conan install . --build=missing -o telemetry=True` succeeds
-- [ ] `cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -Dtelemetry=ON ..` configures correctly (there is no `default` preset; Conan writes `conan-release`)
-- [ ] Build succeeds with telemetry ON
-- [ ] Build succeeds with telemetry OFF
-- [ ] Existing tests pass with telemetry ON
-- [ ] Existing tests pass with telemetry OFF
-
----
-
-## Task 2.8: RPC Span Attribute Enrichment — Node Health Context
-
-**Status**: DROPPED.
-
-Node health (`amendment_blocked`, `server_state`) is not part of the telemetry surface. Operators consume the same data via the existing `server_info` / `server_state` RPC commands, so duplicating it on traces adds storage and cardinality cost without new value. The OTel C++ SDK 1.18.0 also does not support runtime updates to the resource, ruling out resource-level emission of these dynamic-by-nature flags.
-
----
-
-## Task 2.9: PathFind RPC Instrumentation
-
-**Status**: COMPLETE
-
-**Objective**: Trace the path_find and ripple_path_find RPC handlers to capture request latency and computation cost.
-
-**Spans added**:
-
-- `pathfind.request` — wraps `doPathFind()` and `doRipplePathFind()` RPC handlers
-- `pathfind.compute` — wraps `PathRequest::doUpdate()` (`pathfind_fast` attr)
-- `pathfind.update_all` — wraps `PathRequestManager::updateAll()` on ledger close (`pathfind_ledger_index`, `pathfind_num_requests` attrs; emitted only when active subscriptions exist)
-- `pathfind.discover` — wraps the entire per-source-asset loop in `PathRequest::findPaths()` (`pathfind_search_level`, `pathfind_num_paths` attrs). One span per RPC call instead of N (one per source asset). Trade-off: per-asset breakdown is lost; storage and cardinality bounded.
-
-**Attribute namespacing**: All pathfind attributes use the `pathfind_*` underscore form per the Phase 1c naming-spec rule 5.
-
-**New file**: `src/xrpld/rpc/detail/PathFindSpanNames.h`
-
-**Modified files**:
-
-- `src/xrpld/rpc/handlers/orderbook/PathFind.cpp`
-- `src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp`
-- `src/xrpld/rpc/detail/PathRequest.cpp`
-- `src/xrpld/rpc/detail/PathRequestManager.cpp`
-- `src/xrpld/rpc/detail/Pathfinder.cpp`
-
----
-
-## Task 2.10: RPC and PathFind Span Attribute Gap Fill
-
-**Status**: COMPLETE
-
-**Objective**: Wire up workflow-identifying attributes that enable filtering and grouping traces by request characteristics without drilling into child spans.
-
-**Attributes added**:
-
-| Span                | Attribute                    | Type   | Source                            |
-| ------------------- | ---------------------------- | ------ | --------------------------------- |
-| `rpc.http_request`  | `request_payload_size`       | int64  | `request.body().size()`           |
-| `rpc.process`       | `is_batch`                   | bool   | `method == "batch"` check         |
-| `rpc.process`       | `batch_size`                 | int64  | `params.size()` (only when batch) |
-| `rpc.ws_message`    | `command`                    | string | `jv[command]` or `jv[method]`     |
-| `rpc.command.*`     | `load_type`                  | string | `context.loadType.label()`        |
-| `pathfind.compute`  | `pathfind_dest_currency`     | string | `to_string(saDstAmount_.asset())` |
-| `pathfind.discover` | `pathfind_num_source_assets` | int64  | `sourceAssets.size()`             |
-
-_Note: `pathfind_dest_amount` was removed — the destination amount is a financial value excluded by the privacy policy (design §2.4.4)._
-
-**New attr keys**: `RpcSpanNames.h` (`isBatch`, `batchSize`, `loadType`), `PathFindSpanNames.h` (`destCurrency`, `numSourceAssets`).
-
-**Modified files**:
-
-- `src/xrpld/rpc/detail/RpcSpanNames.h`
-- `src/xrpld/rpc/detail/PathFindSpanNames.h`
-- `src/xrpld/rpc/detail/ServerHandler.cpp`
-- `src/xrpld/rpc/detail/RPCHandler.cpp`
-- `src/xrpld/rpc/detail/PathRequest.cpp`
-
----
-
-## Summary
-
-| Task | Description                                 | Status              | Notes                                                     |
-| ---- | ------------------------------------------- | ------------------- | --------------------------------------------------------- |
-| 2.1  | W3C Trace Context header extraction         | Deferred → Phase 3  | No consumer in Phase 2; needs cross-node tracing          |
-| 2.2  | Per-category span creation                  | Complete (Phase 1c) | Superseded by TraceCategory enum + SpanGuard              |
-| 2.3  | Add shouldTraceLedger() interface method    | Complete (Phase 1c) | Delivered in Phase 1c base branch                         |
-| 2.4  | Unit tests for core telemetry               | Complete            | TelemetryConfig + SpanGuardFactory tests                  |
-| 2.5  | Enhanced RPC span attributes (HTTP-level)   | Deferred            | Low value; span duration covers timing natively           |
-| 2.6  | Build verification and performance baseline | Complete            | Verified in CI on Phase 1c                                |
-| 2.7  | Grafana Tempo search filters                | Complete            | rpc-command, rpc-status, rpc-role filters                 |
-| 2.8  | RPC span attribute enrichment (node health) | Dropped             | Available via `server_info`/`server_state` RPC            |
-| 2.9  | PathFind RPC instrumentation                | Complete            | request, compute, update_all, discover                    |
-| 2.10 | RPC/PathFind span attribute gap fill        | Complete            | Batch detection, payload size, load cost, pathfind params |
-
-**Delivered in this branch**: Tasks 2.4, 2.7, 2.9, 2.10.
-**Deferred with rationale**: Tasks 2.1 (→Phase 3), 2.5 (low priority).
-**Dropped**: Task 2.8 (node health not duplicated on traces).
-**Superseded**: Task 2.2 (Phase 1c SpanGuard factory covers this).
-
----
-
-## Known Issues / Future Work
-
-### Thread safety of TelemetryImpl::stop() vs startSpan()
-
-`TelemetryImpl::stop()` resets `sdkProvider_` (a `std::shared_ptr`) without
-synchronization. `getTracer()` reads the same member from RPC handler threads.
-This is a data race if any thread calls `startSpan()` concurrently with `stop()`.
-
-**Current mitigation**: `Application::stop()` shuts down `serverHandler_`,
-`overlay_`, and `jobQueue_` before calling `telemetry_->stop()`, so no callers
-remain. See comments in `Telemetry.cpp:stop()` and `Application.cpp`.
-
-**TODO**: Add an `std::atomic<bool> stopped_` flag checked in `getTracer()` to
-make this robust against future shutdown order changes.
-
-### Macro incompatibility: XRPL_TRACE_SPAN vs XRPL_TRACE_SET_ATTR
-
-`XRPL_TRACE_SPAN` and `XRPL_TRACE_SPAN_KIND` declare `_xrpl_guard_` as a bare
-`SpanGuard`, but `XRPL_TRACE_SET_ATTR` and `XRPL_TRACE_EXCEPTION` call
-`_xrpl_guard_.has_value()` which requires `std::optional<SpanGuard>`. Using
-`XRPL_TRACE_SPAN` followed by `XRPL_TRACE_SET_ATTR` in the same scope would
-fail to compile.
-
-**Current mitigation**: No call site currently uses `XRPL_TRACE_SPAN` — all
-production code uses the conditional macros (`XRPL_TRACE_RPC`, `XRPL_TRACE_TX`,
-etc.) which correctly wrap the guard in `std::optional`.
-
-**TODO**: Either make `XRPL_TRACE_SPAN`/`XRPL_TRACE_SPAN_KIND` also wrap in
-`std::optional`, or document that `XRPL_TRACE_SET_ATTR` is only compatible with
-the conditional macros.

OpenTelemetryPlan/Phase3_taskList.md

@@ -1,568 +0,0 @@
-# Phase 3: Transaction Tracing Task List
-
-> **Goal**: Trace the full transaction lifecycle from RPC submission through peer relay, including cross-node context propagation via Protocol Buffer extensions. This is the WALK phase that demonstrates true distributed tracing.
->
-> **Scope**: Protocol Buffer `TraceContext` message, context serialization, PeerImp transaction instrumentation, NetworkOPs processing instrumentation, HashRouter visibility, and multi-node relay context propagation.
->
-> **Branch**: `pratik/otel-phase3-tx-tracing` (from `pratik/otel-phase2-rpc-tracing`)
-
-### Related Plan Documents
-
-| Document                                                                                          | Relevance                                                                                                                                                                                                    |
-| ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| [docs/telemetry-runbook.md § Protocol Span Flow](../docs/telemetry-runbook.md#protocol-span-flow) | Authoritative protocol span-flow reference — replaces the deleted `04-code-samples.md` (TraceContext protobuf §4.4.1, PeerImp instrumentation §4.5.1, context serialization §4.4.2), removed by `d6450631bf` |
-| [01-architecture-analysis.md](./01-architecture-analysis.md)                                      | Transaction flow (§1.3), key trace points (§1.6)                                                                                                                                                             |
-| [06-implementation-phases.md](./06-implementation-phases.md)                                      | Phase 3 tasks (§6.4), definition of done (§6.11.3)                                                                                                                                                           |
-| [02-design-decisions.md](./02-design-decisions.md)                                                | Context propagation design (§2.5), attribute schema (§2.4.3)                                                                                                                                                 |
-
----
-
-## Task 3.1: Define TraceContext Protocol Buffer Message
-
-**Objective**: Add trace context fields to the P2P protocol messages so trace IDs can propagate across nodes.
-
-**What to do**:
-
-- Edit `include/xrpl/proto/xrpl.proto` (or `src/xrpld/proto/ripple.proto`, wherever the proto is):
-  - Add `TraceContext` message definition:
-    ```protobuf
-    message TraceContext {
-        bytes trace_id = 1;      // 16-byte trace identifier
-        bytes span_id = 2;       // 8-byte span identifier
-        uint32 trace_flags = 3;  // bit 0 = sampled, bit 1 = random
-        reserved 4;              // trace_state (W3C tracestate), added later
-    }
-    ```
-  - Add `optional TraceContext trace_context = 1001;` to:
-    - `TMTransaction`
-    - `TMProposeSet` (for Phase 4 use)
-    - `TMValidation` (for Phase 4 use)
-  - Use high field numbers (1001+) to avoid conflicts with existing fields
-
-- Regenerate protobuf C++ code
-
-**Key modified files**:
-
-- `include/xrpl/proto/xrpl.proto` (or equivalent)
-
-**Reference**:
-
-- `04-code-samples.md` §4.4.1 (TraceContext message definition) was deleted by
-  `d6450631bf`; the live definition is `include/xrpl/proto/xrpl.proto:101`
-  (`message TraceContext`), attached as field `1001` on the relevant messages
-- [02-design-decisions.md §2.5.2](./02-design-decisions.md) — Protocol buffer context propagation design
-
----
-
-## Task 3.2: Implement Protobuf Context Serialization
-
-**Objective**: Create utilities to serialize/deserialize OTel trace context to/from protobuf `TraceContext` messages.
-
-**What to do**:
-
-- Create `include/xrpl/telemetry/TraceContextPropagator.h` (extend from Phase 2 if exists, or add protobuf methods):
-  - Add protobuf-specific methods:
-    - `static Context extractFromProtobuf(protocol::TraceContext const& proto)` — reconstruct OTel context from protobuf fields
-    - `static void injectToProtobuf(Context const& ctx, protocol::TraceContext& proto)` — serialize current span context into protobuf fields
-  - Both methods guard behind `#ifdef XRPL_ENABLE_TELEMETRY`
-
-- Create/extend `src/libxrpl/telemetry/TraceContextPropagator.cpp`:
-  - Implement extraction: read trace_id (16 bytes), span_id (8 bytes), trace_flags from protobuf, construct `SpanContext`, wrap in `Context`
-  - Implement injection: get current span from context, serialize its TraceId, SpanId, and TraceFlags into protobuf fields
-
-**Key new/modified files**:
-
-- `include/xrpl/telemetry/TraceContextPropagator.h`
-- `src/libxrpl/telemetry/TraceContextPropagator.cpp`
-
-**Reference**:
-
-- `04-code-samples.md` §4.4.2 (full extract/inject implementation) was deleted by
-  `d6450631bf`. As shipped there is **no**
-  `src/libxrpl/telemetry/TraceContextPropagator.cpp`; extract/inject live on
-  `SpanGuard` (`include/xrpl/telemetry/SpanGuard.h:467` extract, `:480-491`
-  `injectCurrentContextToProtobuf`, implemented in
-  `src/libxrpl/telemetry/SpanGuard.cpp`) with the protocol-layer wrappers in
-  `src/xrpld/telemetry/PropagationHelpers.h:52` (`injectSpanContext`)
-
----
-
-## Task 3.3: Instrument PeerImp Transaction Handling
-
-**Objective**: Add trace spans to the peer-level transaction receive and relay path.
-
-**What to do**:
-
-- Edit `src/xrpld/overlay/detail/PeerImp.cpp`:
-  - In `onMessage(TMTransaction)` / `handleTransaction()`:
-    - Extract parent trace context from incoming `TMTransaction::trace_context` field (if present)
-    - Create `tx.receive` span as child of extracted context (or new root if none)
-    - Set attributes: `tx_hash`, `peer_id`, `tx_status`
-    - Create the span only after `HashRouter::shouldProcess()` accepts, so a dropped duplicate produces no span
-    - Wrap validation call with child span `tx.validate`
-    - Wrap relay with `tx.relay` span
-  - When relaying to peers:
-    - Inject current trace context into outgoing `TMTransaction::trace_context`
-    - Set `relay_count` attribute
-
-- Use `SpanGuard::span(TraceCategory::Transactions, "tx", "receive")` factory
-  (Phase 1c replaced macros with the SpanGuard factory pattern)
-
-> **Note**: The `tx.receive` guard is `.detached()` before being moved into the
-> `RcvCheckTx` job so its Scope is popped on the peer thread, not leaked to the
-> worker (else later peer messages would inherit this transaction's trace).
-
-**Key modified files**:
-
-- `src/xrpld/overlay/detail/PeerImp.cpp`
-
-**Reference**:
-
-- [docs/telemetry-runbook.md § Protocol Span Flow](../docs/telemetry-runbook.md#protocol-span-flow)
-  — the authoritative `tx.receive` / relay span-flow reference; replaces
-  `04-code-samples.md` §4.5.1, deleted by `d6450631bf`
-- [01-architecture-analysis.md §1.3](./01-architecture-analysis.md) — Transaction flow diagram
-- [01-architecture-analysis.md §1.6](./01-architecture-analysis.md) — tx.receive trace point
-
----
-
-## Task 3.4: Instrument NetworkOPs Transaction Processing
-
-**Objective**: Trace the transaction processing pipeline in NetworkOPs, covering both sync and async paths.
-
-**What to do**:
-
-- Edit `src/xrpld/app/misc/NetworkOPs.cpp`:
-  - In `processTransaction()`:
-    - Create `tx.process` span
-    - Set attributes: `tx_hash`, `tx_type`, `local` (whether from RPC or peer)
-    - Record whether sync or async path is taken
-    - `.detached()` the guard before storing it in `TransactionStatus::span`,
-      since it is applied on a batch worker thread — this pops the Scope on the
-      origin thread and stops later work inheriting this transaction's trace
-
-  - In `doTransactionAsync()`:
-    - Capture parent context before queuing
-    - Create `tx.queue` span with queue depth attribute
-    - Add event when transaction is dequeued for processing
-
-  - In `doTransactionSync()`:
-    - Create `tx.process_sync` span
-    - Record result (applied, queued, rejected)
-
-**Key modified files**:
-
-- `src/xrpld/app/misc/NetworkOPs.cpp`
-
-**Reference**:
-
-- [01-architecture-analysis.md §1.6](./01-architecture-analysis.md) — tx.validate and tx.process trace points
-- [02-design-decisions.md §2.4.3](./02-design-decisions.md) — Transaction attribute schema
-
----
-
-## Task 3.5: Instrument HashRouter for Dedup Visibility
-
-**Objective**: Make transaction deduplication visible in traces by recording HashRouter decisions as span attributes/events.
-
-**What to do**:
-
-- Edit `src/xrpld/overlay/detail/PeerImp.cpp` (in handleTransaction):
-  - After calling `HashRouter::shouldProcess()` or `addSuppressionPeer()`:
-    - Start the span here, not before, so only transactions this node will process are traced
-    - Record `tx_flags` showing current HashRouter state (SAVED, TRUSTED, etc.)
-    - Add `tx.first_seen` or `tx.duplicate` event
-
-- This is NOT a modification to HashRouter itself — just recording its decisions as span attributes in the existing PeerImp instrumentation from Task 3.3.
-
-**Key modified files**:
-
-- `src/xrpld/overlay/detail/PeerImp.cpp` (same changes as 3.3, logically grouped)
-
----
-
-## Task 3.6: Context Propagation in Transaction Relay
-
-**Status**: COMPLETE (transaction relay). Consensus proposal/validation
-propagation is deferred to Phase 4 — see "Planned (Phase 4)" below.
-
-**Objective**: Ensure trace context flows correctly when transactions are relayed between peers, creating linked spans across nodes.
-
-**What was done**:
-
-- **TX send side**: `NetworkOPs::apply()` now injects the tx.process span's trace
-  context into the outgoing `TMTransaction` protobuf before relay, using
-  `telemetry::injectSpanContext()`. The receiving node's `txReceiveSpan()` (already
-  wired in PeerImp) extracts the parent span_id and creates the tx.receive span
-  as a child of the sender's tx.process span.
-
-- **Edge cases**: Missing trace context (older peers) degrades gracefully to
-  standalone spans. Invalid/corrupted context is treated as absent. Trace
-  flags are propagated and respected.
-
-**New infrastructure**:
-
-- `SpanGuard::getTraceBytes()` — extracts raw trace_id/span_id/trace_flags
-  from a span without exposing OTel types. Safe to call from any thread.
-- `PropagationHelpers.h` — `injectSpanContext(SpanGuard&, proto)` bridge
-  between SpanGuard and protobuf TraceContext.
-- `TraceContextPropagator.h` — `injectToProtobuf(ctx, proto)` for
-  same-thread injection via OTel RuntimeContext.
-
-**Key modified files**:
-
-- `src/xrpld/app/misc/NetworkOPs.cpp` — tx relay injection
-- `include/xrpl/telemetry/SpanGuard.h` — `TraceBytes` struct, `getTraceBytes()`
-- `src/libxrpl/telemetry/SpanGuard.cpp` — `getTraceBytes()` implementation
-- `src/xrpld/telemetry/PropagationHelpers.h` — inject helpers (new file)
-
-**Planned (Phase 4 — not in this PR)**:
-
-The consensus proposal/validation propagation below is Phase 4 scope and is
-not implemented on this branch. It is listed here only to record the intended
-design.
-
-- **Proposal send/receive**: `RCLConsensus::Adaptor::propose()` injects the
-  current thread's active span context into the `TMProposeSet` protobuf via
-  `telemetry::injectToProtobuf()`. PeerImp creates a
-  `consensus.proposal.receive` span that extracts the sender's trace context
-  as parent (via `ConsensusReceiveTracing.h`).
-
-- **Validation send/receive**: `RCLConsensus::Adaptor::validate()` injects
-  the current thread's active span context into the `TMValidation` protobuf.
-  PeerImp creates a `consensus.validation.receive` span that extracts the
-  sender's trace context as parent.
-
-- Planned files: `src/xrpld/app/consensus/RCLConsensus.cpp` (send injection),
-  `src/xrpld/overlay/detail/PeerImp.cpp` (receive spans),
-  `src/xrpld/telemetry/ConsensusReceiveTracing.h` (receive span helpers,
-  new file).
-
-**Reference**:
-
-- [02-design-decisions.md §2.5](./02-design-decisions.md) — Context propagation design
-- Relay context injection pattern: `04-code-samples.md` §4.5.1 was deleted by
-  `d6450631bf`; the live pattern is `injectSpanContext()` in
-  `src/xrpld/telemetry/PropagationHelpers.h:52`, with the flow documented in
-  [docs/telemetry-runbook.md § Protocol Span Flow](../docs/telemetry-runbook.md#protocol-span-flow)
-
----
-
-## Task 3.7: Build Verification and Testing
-
-**Objective**: Verify all Phase 3 changes compile and work correctly.
-
-**What to do**:
-
-1. Build with `telemetry=ON` — verify no compilation errors
-2. Build with `telemetry=OFF` — verify no regressions
-3. Run existing unit tests
-4. Verify protobuf regeneration produces correct C++ code
-5. Document any issues encountered
-
-**Verification Checklist**:
-
-- [ ] Protobuf changes generate valid C++
-- [ ] Build succeeds with telemetry ON
-- [ ] Build succeeds with telemetry OFF
-- [ ] Existing tests pass
-- [ ] No undefined symbols from new telemetry calls
-
----
-
-## Task 3.8: Transaction Span Peer Version Attribute
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — adds peer version context inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
->
-> **Upstream**: Phase 2 (RPC span infrastructure must exist).
-> **Downstream**: Phase 10 (validation checks for this attribute).
-
-**Objective**: Add the relaying peer's xrpld version to `tx.receive` spans so operators can correlate transaction issues with peer version mismatches during network upgrades.
-
-**What to do**:
-
-- Edit `src/xrpld/overlay/detail/PeerImp.cpp`:
-  - In the `tx.receive` span block (after existing `peer_id` setAttribute call):
-    - Add `peer_version` (string) — from `this->getVersion()`
-    - Only set if `getVersion()` returns a non-empty string (avoid empty-string attributes)
-
-**New span attribute**:
-
-| Attribute      | Type   | Source               | Example         |
-| -------------- | ------ | -------------------- | --------------- |
-| `peer_version` | string | `peer->getVersion()` | `"xrpld-2.4.0"` |
-
-**Rationale**: Transaction relay is where version mismatches cause subtle serialization or validation bugs. Tracing "this tx came from a v2.3.0 peer" helps diagnose compatibility issues. The community dashboard tracks peer versions externally; this brings version awareness into the trace itself.
-
-**Key modified files**:
-
-- `src/xrpld/overlay/detail/PeerImp.cpp`
-
-**Exit Criteria**:
-
-- [ ] `tx.receive` spans carry `peer_version` attribute with a non-empty version string
-- [ ] Attribute is omitted (not set to empty string) when `getVersion()` returns empty
-- [ ] Attribute visible in Tempo trace detail view
-
----
-
-## Task 3.9: Deterministic Transaction Trace ID
-
-> **Upstream**: Task 3.2 (protobuf serialization), Task 3.3 (PeerImp span exists).
-> **Downstream**: Phase 10 (workload validation can query by tx hash directly).
-> **Pattern**: Mirrors the consensus deterministic trace ID in Phase 4a
-> (`createDeterministicContext` in `RCLConsensus.cpp`), adapted for transactions.
-
-**Objective**: Derive the trace_id for transaction spans deterministically from the
-transaction hash so that all nodes handling the same transaction independently produce
-spans under the same trace_id — regardless of whether protobuf context propagation
-succeeds.
-
-**Why**: The current approach creates spans with random trace_ids and relies entirely
-on protobuf `TraceContext` propagation to link them. If any hop in the relay chain
-drops the context (older peers, message corruption, mixed-version networks), the trace
-splits and downstream spans become impossible to find. With deterministic trace_ids,
-correlation is guaranteed because every node derives the same trace_id from the same
-`txID`.
-
-**Approach — deterministic trace_id + protobuf span_id propagation**:
-
-1. Derive `trace_id = txHash[0:16]` (first 16 bytes of the 32-byte transaction hash).
-2. Generate a random 8-byte `span_id` per node (each node's span is unique within
-   the shared trace).
-3. Create the span under this deterministic context as parent.
-4. **Additionally**, if protobuf `TraceContext` is present in the incoming
-   `TMTransaction` message, extract the sender's `span_id` and use it as the span's
-   parent — this preserves parent-child ordering in the trace tree.
-5. If protobuf context is absent (older peer, first hop), the span still has the
-   correct deterministic `trace_id` — it appears as a sibling root in the same trace
-   rather than being lost.
-
-This gives the best of both worlds: guaranteed cross-node correlation via deterministic
-`trace_id`, plus parent-child relay ordering via protobuf `span_id` when available.
-
-**What to do**:
-
-- Create `createDeterministicTxContext(uint256 const& txHash)` utility function:
-  - Location: shared header or file-local in `PeerImp.cpp` and `NetworkOPs.cpp`
-    (or a shared telemetry utility if both need it).
-  - Pattern: identical to `createDeterministicContext(uint256 const& ledgerId)` in
-    `RCLConsensus.cpp` — take `txHash[0:16]` as trace_id, random span_id via
-    `default_prng()`, sampled flag set, `remote=false`.
-  - Guard behind `#ifdef XRPL_ENABLE_TELEMETRY`.
-
-  ```cpp
-  opentelemetry::context::Context
-  createDeterministicTxContext(uint256 const& txHash)
-  {
-      namespace trace = opentelemetry::trace;
-
-      // First 16 bytes of the 32-byte tx hash as trace ID.
-      trace::TraceId traceId(
-          opentelemetry::nostd::span<uint8_t const, 16>(txHash.data(), 16));
-
-      // Random span_id so each node's span is unique within the trace.
-      uint8_t spanIdBytes[8];
-      auto const rval = default_prng()();
-      std::memcpy(spanIdBytes, &rval, sizeof(spanIdBytes));
-      trace::SpanId spanId(
-          opentelemetry::nostd::span<uint8_t const, 8>(spanIdBytes, 8));
-
-      trace::SpanContext syntheticCtx(
-          traceId, spanId, trace::TraceFlags(1), /* remote = */ false);
-
-      return opentelemetry::context::Context{}.SetValue(
-          trace::kSpanKey,
-          opentelemetry::nostd::shared_ptr<trace::Span>(
-              new trace::DefaultSpan(syntheticCtx)));
-  }
-  ```
-
-- Edit `src/xrpld/overlay/detail/PeerImp.cpp` — restructure `handleTransaction()`:
-  - **Move span creation after deserialization** (txID must be known first):
-    1. Deserialize `STTx` and get `txID` (existing code at line ~1382).
-    2. Create deterministic parent context: `auto detCtx = createDeterministicTxContext(txID)`.
-    3. If `m->has_trace_context()`: extract protobuf context via `extractFromProtobuf()`,
-       **combine** with deterministic trace_id — use the protobuf span_id as parent
-       to preserve relay ordering, but override trace_id with the deterministic one.
-    4. If no protobuf context: create span under `detCtx` directly.
-    5. Set all existing attributes (`hash`, `peerId`, `peerVersion`, etc.).
-
-  - **Combining deterministic trace_id with protobuf parent span_id**:
-    When both are available, construct a synthetic `SpanContext` with:
-    - `trace_id` = `txHash[0:16]` (deterministic)
-    - `span_id` = extracted from protobuf (sender's span_id → becomes parent)
-    - `trace_flags` = from protobuf
-    - `remote` = true (came from another node)
-
-    ```cpp
-    // Pseudo-code for the combined context:
-    auto detTraceId = trace::TraceId(txHash.data(), 16);
-    auto remoteSpanId = /* from extractFromProtobuf */;
-    auto remoteFlags = /* from extractFromProtobuf */;
-
-    trace::SpanContext combinedCtx(
-        detTraceId, remoteSpanId, remoteFlags, /* remote = */ true);
-    // Use as parent context for the new span.
-    ```
-
-- Edit `src/xrpld/app/misc/NetworkOPs.cpp` — update `processTransaction()`:
-  - `transaction->getID()` is already available at the top of the function.
-  - Create deterministic parent context from `txID`.
-  - Create `tx.process` span under this context.
-  - No protobuf context to extract here (NetworkOPs is intra-node), so
-    deterministic context alone is sufficient.
-
-- Add `trace_strategy` attribute to spans:
-  - Add `inline constexpr auto traceStrategy = "trace_strategy";`
-    to `TxSpanNames.h`.
-  - Set on each tx span: `span.setAttribute(tx_span::attr::traceStrategy, "deterministic")`.
-
-**Key new/modified files**:
-
-- `src/xrpld/overlay/detail/PeerImp.cpp` — restructured span creation
-- `src/xrpld/app/misc/NetworkOPs.cpp` — deterministic context for tx.process
-- `src/xrpld/telemetry/TxSpanNames.h` — new `traceStrategy` attribute constant
-- New or shared utility for `createDeterministicTxContext()` (location TBD: could be
-  a shared header like `include/xrpl/telemetry/DeterministicContext.h`, or file-local
-  if only used in two places)
-
-**Interaction with existing tasks**:
-
-- **Task 3.3 (PeerImp instrumentation)**: The span creation in `handleTransaction()`
-  must be restructured — the span currently starts before `txID` is known. This task
-  moves it after deserialization.
-- **Task 3.6 (Relay context propagation)**: Protobuf injection at the relay site
-  remains the same — `injectToProtobuf()` serializes the current span's `span_id`.
-  The receiver extracts it and combines with the deterministic `trace_id`.
-- **Phase 4a (Consensus deterministic trace ID)**: This task follows the same pattern.
-  Consider extracting a shared utility (e.g., `createDeterministicContext(uint256)`)
-  that both consensus and transaction tracing use.
-
-**Exit Criteria**:
-
-- [ ] `tx.receive` and `tx.process` spans have deterministic trace_id = `txHash[0:16]`
-- [ ] All nodes handling the same transaction produce spans under the same trace_id
-- [x] Protobuf `span_id` propagation still works when available (parent-child ordering)
-- [ ] Missing protobuf context (old peer) degrades gracefully to sibling spans, not lost traces
-- [ ] `trace_strategy` attribute set to `"deterministic"` on all tx spans
-- [ ] Trace queryable by tx hash (truncate hash → trace_id → direct lookup in Tempo)
-
-**Deliverables implemented (not in original plan)**:
-
-- **`SpanGuard::txSpan()` factory method** (`include/xrpl/telemetry/SpanGuard.h`):
-  Two overloads for creating transaction spans with deterministic trace IDs:
-  - `txSpan(category, group, name, txHash)` — standalone span (deterministic
-    trace_id from `txHash[0:16]`, no parent span_id).
-  - `txSpan(category, group, name, txHash, parentCtx)` — child span (deterministic
-    trace_id combined with protobuf-extracted parent span_id for relay ordering).
-
-- **`TxTracing.h` helper functions** (`src/xrpld/telemetry/TxTracing.h`):
-  File-local helpers that wrap `SpanGuard::txSpan()` for the two main PeerImp call
-  sites:
-  - `txReceiveSpan(txHash, parentCtx)` — creates `tx.receive` span with
-    deterministic trace_id and optional protobuf parent context.
-  - `txProcessSpan(txHash)` — creates `tx.process` span with deterministic
-    trace_id only (no protobuf parent, used intra-node).
-  - **Note**: `TxTracing.h` includes `xrpl.pb.h` unconditionally (outside
-    `#ifdef XRPL_ENABLE_TELEMETRY`) because `protocol::TMTransaction` appears in
-    the function signatures regardless of telemetry build mode.
-
----
-
-## Task 3.10: TxQ Instrumentation
-
-**Status**: COMPLETE
-
-**Objective**: Trace the transaction queue lifecycle — enqueue decisions, direct apply, batch clear, ledger-close accept loop, per-tx apply, and cleanup.
-
-**Spans added**:
-
-- `txq.enqueue` — wraps `TxQ::apply()` with tx_hash attribute
-- `txq.apply_direct` — wraps `TxQ::tryDirectApply()` fast-path
-- `txq.batch_clear` — wraps `TxQ::tryClearAccountQueueUpThruTx()`
-- `txq.accept` — wraps `TxQ::accept()` ledger-close dequeue with queue_size attr
-- `txq.accept_tx` — per-tx span inside accept loop with tx_hash, ter_code,
-  retries_remaining attributes
-- `txq.cleanup` — wraps `TxQ::processClosedLedger()` with ledger_seq attribute
-
-**New file**: `src/xrpld/app/misc/detail/TxQSpanNames.h`
-
-**Modified file**: `src/xrpld/app/misc/detail/TxQ.cpp`
-
----
-
-## Task 3.11: TX and TxQ Span Attribute Gap Fill
-
-**Status**: COMPLETE
-
-**Objective**: Add workflow-identifying attributes to transaction spans so operators can filter by transaction type and see outcomes without off-chain correlation.
-
-**Attributes added**:
-
-| Span              | Attribute            | Type   | Source                                                                                                                     |
-| ----------------- | -------------------- | ------ | -------------------------------------------------------------------------------------------------------------------------- |
-| `tx.process`      | `tx_type`            | string | `TxFormats::getInstance().findByType(stx->getTxnType())->getName()`                                                        |
-| `tx.process`      | `fee`                | int64  | `stx->getFieldAmount(sfFee).xrp().drops()`                                                                                 |
-| `tx.process`      | `sequence`           | int64  | `stx->getSeqProxy().value()`                                                                                               |
-| `tx.process`      | `tx_<account field>` | string | one per top-level `STI_ACCOUNT` field, raw r-address (`tx_account`, `tx_destination`, ...); keys in `TxAccountSpanNames.h` |
-| `tx.process`      | `ter_result`         | string | `transToken(e.result)` (set after batch application)                                                                       |
-| `tx.process`      | `applied`            | bool   | `e.applied` (set after batch application)                                                                                  |
-| `tx.receive`      | `tx_type`            | string | `TxFormats::getInstance().findByType(stx->getTxnType())->getName()`                                                        |
-| `txq.enqueue`     | `tx_type`            | string | same pattern as above                                                                                                      |
-| `txq.enqueue`     | `txq_status`         | string | `queued` / `applied_direct` / `applied` / `failed` / `rejected`                                                            |
-| `txq.enqueue`     | `ter_code`           | string | `transToken(directApplied->ter)` (set on the direct-apply path)                                                            |
-| `txq.enqueue`     | `fee_level_paid`     | int64  | `getFeeLevelPaid(view, *tx).value()`                                                                                       |
-| `txq.enqueue`     | `required_fee_level` | int64  | `getRequiredFeeLevel(...).value()`                                                                                         |
-| `txq.batch_clear` | `num_cleared`        | int64  | queued txs cleared ahead of the applying tx                                                                                |
-| `txq.cleanup`     | `expired_count`      | int64  | entries dropped for passed `LastLedgerSequence`                                                                            |
-| `txq.accept_tx`   | `txq_status`         | string | `applied` / `failed` / `retried`                                                                                           |
-| `txq.accept_tx`   | `ter_code`           | string | `transToken(txnResult)` (set before branching on the outcome)                                                              |
-| `txq.accept`      | `ledger_changed`     | bool   | set at end of accept loop                                                                                                  |
-
-**New attr keys**: `TxSpanNames.h` (`txType`, `fee`, `sequence`, `terResult`, `applied`), `TxQSpanNames.h` (`txType`).
-
-**Modified files**:
-
-- `src/xrpld/telemetry/TxSpanNames.h`
-- `src/xrpld/app/misc/detail/TxQSpanNames.h`
-- `src/xrpld/app/misc/NetworkOPs.cpp`
-- `src/xrpld/overlay/detail/PeerImp.cpp`
-- `src/xrpld/app/misc/detail/TxQ.cpp`
-
----
-
-## Summary
-
-| Task | Description                         | New Files | Modified Files | Depends On |
-| ---- | ----------------------------------- | --------- | -------------- | ---------- |
-| 3.1  | TraceContext protobuf message       | 0         | 1              | Phase 2    |
-| 3.2  | Protobuf context serialization      | 1-2       | 0              | 3.1        |
-| 3.3  | PeerImp transaction instrumentation | 0         | 1              | 3.2        |
-| 3.4  | NetworkOPs transaction processing   | 0         | 1              | Phase 2    |
-| 3.5  | HashRouter dedup visibility         | 0         | 1              | 3.3        |
-| 3.6  | Relay context propagation           | 0         | 1-2            | 3.3, 3.5   |
-| 3.7  | Build verification and testing      | 0         | 0              | 3.1-3.6    |
-| 3.8  | TX span peer version attribute      | 0         | 1              | 3.3        |
-| 3.9  | Deterministic transaction trace ID  | 0-1       | 3              | 3.2, 3.3   |
-| 3.10 | TxQ instrumentation (6 spans)       | 1         | 1              | 3.4        |
-| 3.11 | TX/TxQ span attribute gap fill      | 0         | 5              | 3.3, 3.10  |
-
-**Parallel work**: Tasks 3.1 and 3.4 can start in parallel. Task 3.2 depends on 3.1. Tasks 3.3 and 3.5 depend on 3.2. Task 3.6 depends on 3.3 and 3.5. Task 3.8 depends on 3.3 (span must exist). Task 3.9 depends on 3.2 and 3.3. Task 3.10 depends on 3.4 (tx.process span must exist).
-
-**Exit Criteria** (from [06-implementation-phases.md §6.11.3](./06-implementation-phases.md)):
-
-- [x] Transaction traces span across nodes
-- [x] Trace context in Protocol Buffer messages
-- [ ] HashRouter deduplication visible in traces
-- [ ] <5% overhead on transaction throughput
-- [x] Deterministic trace_id: same trace_id for same tx across all nodes
-- [x] Protobuf span_id propagation preserves parent-child ordering when available
-
----
-
-## Known Issues / Future Work
-
-### trace_state is not a proto field yet
-
-`TraceContext` in `xrpl.proto` has no `trace_state` field: number 4 and the name `trace_state` are `reserved` for W3C `tracestate` vendor-specific key-value pairs. When cross-vendor trace propagation is needed, add the field with a size limit and wire it through `TraceContextPropagator`. Until then a peer-sent field 4 is an unknown field, which the parser drops.

OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md

@@ -1,221 +0,0 @@
-# Phase 5: Integration Test Task List
-
-> **Goal**: End-to-end verification of the complete telemetry pipeline using a
-> 6-node consensus network. Proves that RPC, transaction, and consensus spans
-> flow through the observability stack (otel-collector, Tempo, Prometheus,
-> Grafana) under realistic conditions.
->
-> **Scope**: Integration test script, manual testing plan, 6-node local network
-> setup, Tempo/Prometheus/Grafana verification.
->
-> **Branch**: `pratik/otel-phase5-docs-deployment`
-
-### Related Plan Documents
-
-| Document                                                         | Relevance                                  |
-| ---------------------------------------------------------------- | ------------------------------------------ |
-| [07-observability-backends.md](./07-observability-backends.md)   | Tempo, Grafana, Prometheus setup           |
-| [05-configuration-reference.md](./05-configuration-reference.md) | Collector config, Docker Compose           |
-| [06-implementation-phases.md](./06-implementation-phases.md)     | Phase 5 tasks, definition of done          |
-| [Phase5_taskList.md](./Phase5_taskList.md)                       | Phase 5 main task list (5.6 = integration) |
-
----
-
-## Task IT.1: Create Integration Test Script
-
-**Objective**: Automated bash script that stands up a 6-node xrpld network
-with telemetry, exercises all span categories, and verifies data in
-Tempo/Prometheus.
-
-**What to do**:
-
-- Create `docker/telemetry/integration-test.sh`:
-  - Prerequisites check (docker, xrpld binary, curl, jq)
-  - Start observability stack via `docker compose`
-  - Generate 6 validator key pairs via temp standalone xrpld
-  - Generate 6 node configs + shared `validators.txt`
-  - Start 6 xrpld nodes in consensus mode (`--start`, no `-a`)
-  - Wait for all nodes to reach `"proposing"` state (120s timeout)
-
-**Key new file**: `docker/telemetry/integration-test.sh`
-
-**Verification**:
-
-- [ ] Script starts without errors
-- [ ] All 6 nodes reach "proposing" state
-- [ ] Observability stack is healthy (otel-collector, Tempo, Prometheus, Grafana)
-
----
-
-## Task IT.2: RPC Span Verification (Phase 2)
-
-**Objective**: Verify RPC spans flow through the telemetry pipeline.
-
-**What to do**:
-
-- Send `server_info`, `server_state`, `ledger` RPCs to node1 (port 5005)
-- Wait for batch export (5s)
-- Query Tempo API for:
-  - `rpc.request` spans (ServerHandler::onRequest)
-  - `rpc.process` spans (ServerHandler::processRequest)
-  - `rpc.command.server_info` spans (callMethod)
-  - `rpc.command.server_state` spans (callMethod)
-  - `rpc.command.ledger` spans (callMethod)
-- Verify `command` attribute present on `rpc.command.*` spans
-
-**Verification**:
-
-- [ ] Tempo shows `rpc.request` traces
-- [ ] Tempo shows `rpc.process` traces
-- [ ] Tempo shows `rpc.command.*` traces with correct attributes
-
----
-
-## Task IT.3: Transaction Span Verification (Phase 3)
-
-**Objective**: Verify transaction spans flow through the telemetry pipeline.
-
-**What to do**:
-
-- Get genesis account sequence via `account_info` RPC
-- Submit Payment transaction using genesis seed (`snoPBrXtMeMyMHUVTgbuqAfg1SUTb`)
-- Wait for consensus inclusion (10s)
-- Query Tempo API for:
-  - `tx.process` spans (NetworkOPsImp::processTransaction) on submitting node
-  - `tx.receive` spans (PeerImp::handleTransaction) on peer nodes
-- Verify `xrpl.tx.hash` attribute on `tx.process` spans
-- Verify `xrpl.peer.id` attribute on `tx.receive` spans
-
-**Verification**:
-
-- [ ] Tempo shows `tx.process` traces with `xrpl.tx.hash`
-- [ ] Tempo shows `tx.receive` traces with `xrpl.peer.id`
-
----
-
-## Task IT.4: Consensus Span Verification (Phase 4)
-
-**Objective**: Verify consensus spans flow through the telemetry pipeline.
-
-**What to do**:
-
-- Consensus runs automatically in 6-node network
-- Query Tempo API for:
-  - `consensus.proposal.send` (Adaptor::propose)
-  - `consensus.ledger_close` (Adaptor::onClose)
-  - `consensus.accept` (Adaptor::onAccept)
-  - `consensus.validation.send` (Adaptor::validate)
-- Verify attributes:
-  - `xrpl.consensus.mode` on `consensus.ledger_close`
-  - `proposers` on `consensus.accept`
-  - `xrpl.consensus.ledger.seq` on `consensus.validation.send`
-
-**Verification**:
-
-- [ ] Tempo shows `consensus.ledger_close` traces with `xrpl.consensus.mode`
-- [ ] Tempo shows `consensus.accept` traces with `proposers`
-- [ ] Tempo shows `consensus.proposal.send` traces
-- [ ] Tempo shows `consensus.validation.send` traces
-
----
-
-## Task IT.5: Spanmetrics Verification (Phase 5)
-
-**Objective**: Verify spanmetrics connector derives RED metrics from spans.
-
-**What to do**:
-
-- Query Prometheus for `traces_span_metrics_calls_total`
-- Query Prometheus for `traces_span_metrics_duration_milliseconds_count`
-- Verify Grafana loads at `http://localhost:3000`
-
-**Verification**:
-
-- [ ] Prometheus returns non-empty results for `traces_span_metrics_calls_total`
-- [ ] Prometheus returns non-empty results for duration histogram
-- [ ] Grafana UI accessible with dashboards visible
-
----
-
-## Task IT.6: Manual Testing Plan
-
-**Objective**: Document how to run tests manually for future reference.
-
-**What to do**:
-
-- Create `docker/telemetry/TESTING.md` with:
-  - Prerequisites section
-  - Single-node standalone test (quick verification)
-  - 6-node consensus test (full verification)
-  - Expected span catalog (all 11 span names with attributes)
-  - Verification queries (Tempo API, Prometheus API)
-  - Troubleshooting guide
-
-**Key new file**: `docker/telemetry/TESTING.md`
-
-**Verification**:
-
-- [ ] Document covers both single-node and multi-node testing
-- [ ] All 11 span names documented with source file and attributes
-- [ ] Troubleshooting section covers common failure modes
-
----
-
-## Task IT.7: Run and Verify
-
-**Objective**: Execute the integration test and validate results.
-
-**What to do**:
-
-- Run `docker/telemetry/integration-test.sh` locally
-- Debug any failures
-- Leave stack running for manual verification
-- Share URLs:
-  - Tempo: `http://localhost:3200`
-  - Grafana: `http://localhost:3000`
-  - Prometheus: `http://localhost:9090`
-
-**Verification**:
-
-- [ ] Script completes with all checks passing
-- [ ] Tempo UI shows xrpld service with all expected span names
-- [ ] Grafana dashboards load and show data
-
----
-
-## Task IT.8: Commit
-
-**Objective**: Commit all new files to Phase 5 branch.
-
-**What to do**:
-
-- Run `pcc` (pre-commit checks)
-- Commit 3 new files to `pratik/otel-phase5-docs-deployment`
-
-**Verification**:
-
-- [ ] `pcc` passes
-- [ ] Commit created on Phase 5 branch
-
----
-
-## Summary
-
-| Task | Description                   | New Files | Depends On |
-| ---- | ----------------------------- | --------- | ---------- |
-| IT.1 | Integration test script       | 1         | Phase 5    |
-| IT.2 | RPC span verification         | 0         | IT.1       |
-| IT.3 | Transaction span verification | 0         | IT.1       |
-| IT.4 | Consensus span verification   | 0         | IT.1       |
-| IT.5 | Spanmetrics verification      | 0         | IT.1       |
-| IT.6 | Manual testing plan           | 1         | --         |
-| IT.7 | Run and verify                | 0         | IT.1-IT.6  |
-| IT.8 | Commit                        | 0         | IT.7       |
-
-**Exit Criteria**:
-
-- [ ] All 6 xrpld nodes reach "proposing" state
-- [ ] All 11 expected span names visible in Tempo
-- [ ] Spanmetrics available in Prometheus
-- [ ] Grafana dashboards show data
-- [ ] Manual testing plan document complete

OpenTelemetryPlan/Phase5_taskList.md

@@ -1,250 +0,0 @@
-# Phase 5: Documentation & Deployment Task List
-
-> **Goal**: Production readiness — Grafana dashboards, spanmetrics pipeline, operator runbook, alert definitions, and final integration testing. This phase ensures the telemetry system is useful and maintainable in production.
->
-> **Scope**: Grafana dashboard definitions, OTel Collector spanmetrics connector, Prometheus integration, alert rules, operator documentation, and production-ready Docker Compose stack.
->
-> **Branch**: `pratik/otel-phase5-docs-deployment` (from `pratik/otel-phase4-consensus-tracing`)
-
-> **Note on attribute names**: the `xrpl.<domain>.<field>` keys shown below
-> (including the collector spanmetrics dimension examples) are written in the
-> older dotted form for readability — it mirrors how the fully qualified
-> attribute reads in a Tempo trace view. The implemented keys follow the
-> convention in [CONTRIBUTING.md](../CONTRIBUTING.md#telemetry-span-attribute-naming)
-> (underscore form, e.g. `command`, `rpc_status`); the `*SpanNames.h` constants
-> are the single source of truth, and the real collector dimensions must use
-> those exact underscore keys (the CI naming check enforces this).
-
-### Related Plan Documents
-
-| Document                                                         | Relevance                                                                  |
-| ---------------------------------------------------------------- | -------------------------------------------------------------------------- |
-| [07-observability-backends.md](./07-observability-backends.md)   | Tempo setup (§7.1), Grafana dashboards (§7.6), alerts (§7.6.3)             |
-| [05-configuration-reference.md](./05-configuration-reference.md) | Collector config (§5.5), production config (§5.5.2), Docker Compose (§5.6) |
-| [06-implementation-phases.md](./06-implementation-phases.md)     | Phase 5 tasks (§6.6), definition of done (§6.11.5)                         |
-
----
-
-## Task 5.1: Add Spanmetrics Connector to OTel Collector
-
-**Objective**: Derive RED metrics (Rate, Errors, Duration) from trace spans automatically, enabling Grafana time-series dashboards.
-
-**What to do**:
-
-- Edit `docker/telemetry/otel-collector-config.yaml`:
-  - Add `spanmetrics` connector:
-    ```yaml
-    connectors:
-      spanmetrics:
-        histogram:
-          explicit:
-            buckets: [1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 5s]
-        dimensions:
-          - name: command
-          - name: rpc_status
-          - name: consensus_phase
-          - name: tx_type
-    ```
-  - Add `prometheus` exporter:
-    ```yaml
-    exporters:
-      prometheus:
-        endpoint: 0.0.0.0:8889
-    ```
-  - Wire the pipeline:
-    ```yaml
-    service:
-      pipelines:
-        traces:
-          receivers: [otlp]
-          processors: [batch]
-          exporters: [debug, otlp/tempo, spanmetrics]
-        metrics:
-          receivers: [spanmetrics]
-          exporters: [prometheus]
-    ```
-
-- Edit `docker/telemetry/docker-compose.yml`:
-  - Expose port `8889` on the collector for Prometheus scraping
-  - Add Prometheus service
-  - Add Prometheus as Grafana datasource
-
-**Key modified files**:
-
-- `docker/telemetry/otel-collector-config.yaml`
-- `docker/telemetry/docker-compose.yml`
-
-**Key new files**:
-
-- `docker/telemetry/prometheus.yml` (Prometheus scrape config)
-- `docker/telemetry/grafana/provisioning/datasources/prometheus.yaml`
-
-**Reference**:
-
-- [POC_taskList.md §Next Steps](./POC_taskList.md) — Metrics pipeline for Grafana dashboards
-
----
-
-## Task 5.2: Create Grafana Dashboards
-
-**Objective**: Provide pre-built Grafana dashboards for RPC performance, transaction lifecycle, and consensus health.
-
-**What to do**:
-
-- Create `docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml` (provisioning config)
-- Create dashboard JSON files:
-  1. **RPC Performance Dashboard** (`rpc-performance.json`):
-     - RPC request latency (p50/p95/p99) by command — histogram panel
-     - RPC throughput (requests/sec) by command — time series
-     - RPC error rate by command — bar gauge
-     - Top slowest RPC commands — table
-
-  2. **Transaction Overview Dashboard** (`transaction-overview.json`):
-     - Transaction processing rate — time series
-     - Transaction latency distribution — histogram
-     - Suppression rate (duplicates) — stat panel
-     - Transaction processing path (sync vs async) — pie chart
-
-  3. **Consensus Health Dashboard** (`consensus-health.json`):
-     - Consensus round duration — time series
-     - Phase duration breakdown (open/establish/accept) — stacked bar
-     - Proposals sent/received per round — stat panel
-     - Consensus mode distribution (proposing/observing) — pie chart
-
-- Store dashboards in `docker/telemetry/grafana/dashboards/`
-
-**Key new files**:
-
-- `docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml`
-- `docker/telemetry/grafana/dashboards/rpc-performance.json`
-- `docker/telemetry/grafana/dashboards/transaction-overview.json`
-- `docker/telemetry/grafana/dashboards/consensus-health.json`
-
-**Reference**:
-
-- [07-observability-backends.md §7.6](./07-observability-backends.md) — Grafana dashboard specifications
-- [01-architecture-analysis.md §1.8.3](./01-architecture-analysis.md) — Dashboard panel examples
-
----
-
-## Task 5.3: Define Alert Rules
-
-**Objective**: Create alert definitions for key telemetry anomalies.
-
-**What to do**:
-
-- Create `docker/telemetry/grafana/provisioning/alerting/alerts.yaml`:
-  - **RPC Latency Alert**: p99 latency > 1s for any command over 5 minutes
-  - **RPC Error Rate Alert**: Error rate > 5% for any command over 5 minutes
-  - **Consensus Duration Alert**: Round duration > 10s (warn), > 30s (critical)
-  - **Transaction Processing Alert**: Processing rate drops below threshold
-  - **Telemetry Pipeline Health**: No spans received for > 2 minutes
-
-**Key new files**:
-
-- `docker/telemetry/grafana/provisioning/alerting/alerts.yaml`
-
-**Reference**:
-
-- [07-observability-backends.md §7.6.3](./07-observability-backends.md) — Alert rule definitions
-
----
-
-## Task 5.4: Production Collector Configuration
-
-**Objective**: Create a production-ready OTel Collector configuration with tail-based sampling and resource limits.
-
-**What to do**:
-
-- Create `docker/telemetry/otel-collector-config-production.yaml`:
-  - Tail-based sampling policy:
-    - Always sample errors and slow traces
-    - 10% base sampling rate for normal traces
-    - Always sample first trace for each unique RPC command
-  - Resource limits:
-    - Memory limiter processor (80% of available memory)
-    - Queued retry for export failures
-  - TLS configuration for production endpoints
-  - Health check endpoint
-
-**Key new files**:
-
-- `docker/telemetry/otel-collector-config-production.yaml`
-
-**Reference**:
-
-- [05-configuration-reference.md §5.5.2](./05-configuration-reference.md) — Production collector config
-
----
-
-## Task 5.5: Operator Runbook
-
-**Objective**: Create operator documentation for managing the telemetry system in production.
-
-**What to do**:
-
-- Create `docs/telemetry-runbook.md`:
-  - **Setup**: How to enable telemetry in xrpld
-  - **Configuration**: All config options with descriptions
-  - **Collector Deployment**: Docker Compose vs. Kubernetes vs. bare metal
-  - **Troubleshooting**: Common issues and resolutions
-    - No traces appearing
-    - High memory usage from telemetry
-    - Collector connection failures
-    - Sampling configuration tuning
-  - **Performance Tuning**: Batch size, queue size, sampling ratio guidelines
-  - **Upgrading**: How to upgrade OTel SDK and Collector versions
-
-**Key new files**:
-
-- `docs/telemetry-runbook.md`
-
----
-
-## Task 5.6: Final Integration Testing
-
-**Objective**: Validate the complete telemetry stack end-to-end.
-
-**What to do**:
-
-1. Start full Docker stack (Collector, Tempo, Grafana, Prometheus)
-2. Build xrpld with `telemetry=ON`
-3. Run in standalone mode with telemetry enabled
-4. Generate RPC traffic and verify traces in Tempo
-5. Verify dashboards populate in Grafana
-6. Verify alerts trigger correctly
-7. Test telemetry OFF path (no regressions)
-8. Run full test suite
-
-**Verification Checklist**:
-
-- [ ] Docker stack starts without errors
-- [ ] Traces appear in Tempo with correct hierarchy
-- [ ] Grafana dashboards show metrics derived from spans
-- [ ] Prometheus scrapes spanmetrics successfully
-- [ ] Alerts can be triggered by simulated conditions
-- [ ] Build succeeds with telemetry ON and OFF
-- [ ] Full test suite passes
-
----
-
-## Summary
-
-| Task | Description                        | New Files | Modified Files | Depends On |
-| ---- | ---------------------------------- | --------- | -------------- | ---------- |
-| 5.1  | Spanmetrics connector + Prometheus | 2         | 2              | Phase 4    |
-| 5.2  | Grafana dashboards                 | 4         | 0              | 5.1        |
-| 5.3  | Alert definitions                  | 1         | 0              | 5.1        |
-| 5.4  | Production collector config        | 1         | 0              | Phase 4    |
-| 5.5  | Operator runbook                   | 1         | 0              | Phase 4    |
-| 5.6  | Final integration testing          | 0         | 0              | 5.1-5.5    |
-
-**Parallel work**: Tasks 5.1, 5.4, and 5.5 can run in parallel. Tasks 5.2 and 5.3 depend on 5.1. Task 5.6 depends on all others.
-
-**Exit Criteria** (from [06-implementation-phases.md §6.11.5](./06-implementation-phases.md)):
-
-- [ ] Dashboards deployed and showing data
-- [ ] Alerts configured and tested
-- [ ] Operator documentation complete
-- [ ] Production collector config ready
-- [ ] Full test suite passes

OpenTelemetryPlan/Phase7_taskList.md

@@ -1,599 +0,0 @@
-# Phase 7: Native OTel Metrics Migration — Task List
-
-> **Goal**: Replace `StatsDCollector` with a native OpenTelemetry Metrics SDK implementation behind the existing `beast::insight::Collector` interface, eliminating the StatsD UDP dependency.
->
-> **Scope**: New `OTelCollectorImpl` class, `CollectorManager` config change, OTel Collector pipeline update, Grafana dashboard metric name migration, integration tests.
->
-> **Branch**: `pratik/otel-phase7-native-metrics` (from `pratik/otel-phase6-statsd`)
-
-### Related Plan Documents
-
-| Document                                                             | Relevance                                                       |
-| -------------------------------------------------------------------- | --------------------------------------------------------------- |
-| [06-implementation-phases.md](./06-implementation-phases.md)         | Phase 7 plan: motivation, architecture, exit criteria (§6.8)    |
-| [02-design-decisions.md](./02-design-decisions.md)                   | Collector interface design, beast::insight coexistence strategy |
-| [05-configuration-reference.md](./05-configuration-reference.md)     | `[insight]` and `[telemetry]` config sections                   |
-| [09-data-collection-reference.md](./09-data-collection-reference.md) | Complete metric inventory that must be preserved                |
-
----
-
-## Task 7.1: Add OTel Metrics SDK to Build Dependencies
-
-**Objective**: Enable the OTel C++ Metrics SDK components in the build system.
-
-**What to do**:
-
-- Edit `conanfile.py`:
-  - Add OTel metrics SDK components to the dependency list when `telemetry=True`
-  - Components needed: `opentelemetry-cpp::metrics`, `opentelemetry-cpp::otlp_http_metric_exporter`
-
-- Edit `CMakeLists.txt` (telemetry section):
-  - Link `opentelemetry::metrics` and `opentelemetry::otlp_http_metric_exporter` targets
-
-**Key modified files**:
-
-- `conanfile.py`
-- `CMakeLists.txt` (or the relevant telemetry cmake target)
-
-**Reference**: [05-configuration-reference.md §5.3](./05-configuration-reference.md) — CMake integration
-
----
-
-## Task 7.2: Implement OTelCollector Class
-
-**Objective**: Create the core `OTelCollector` implementation that maps beast::insight instruments to OTel Metrics SDK instruments.
-
-**What to do**:
-
-- Create `include/xrpl/beast/insight/OTelCollector.h`:
-  - Public factory: `static std::shared_ptr<OTelCollector> New(std::string const& endpoint, std::string const& prefix, beast::Journal journal)`
-  - Derives from `StatsDCollector` (or directly from `Collector` — TBD based on shared code)
-
-- Create `src/libxrpl/beast/insight/OTelCollector.cpp` (~400-500 lines):
-  - **OTelCounterImpl**: Wraps `opentelemetry::metrics::Counter<int64_t>`. `increment(amount)` calls `counter->Add(amount)`.
-  - **OTelGaugeImpl**: Uses `opentelemetry::metrics::ObservableGauge<uint64_t>` with an async callback. `set(value)` stores value atomically; callback reads it during collection.
-  - **OTelMeterImpl**: Wraps `opentelemetry::metrics::Counter<uint64_t>`. `increment(amount)` calls `counter->Add(amount)`. Semantically identical to Counter but unsigned.
-  - **OTelEventImpl**: Wraps `opentelemetry::metrics::Histogram<double>`. `notify()` calls `histogram->Record(value.count())`. Declares its unit from `beast::insight::Unit`, which is what selects its bucket ladder: the histogram views in `Telemetry.cpp` match on unit, so a `ms` instrument gets the millisecond ladder and a `By` instrument the byte ladder. Bucket edges live in `include/xrpl/telemetry/HistogramBuckets.h` — do not restate them here. The millisecond ladder must contain every representable edge of the collector's spanmetrics ladder and may extend above it (jobs outlive spans); `.github/scripts/telemetry/check_bucket_parity.py` enforces that. An earlier version of this line specified `[1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms` as "matching SpanMetrics" — true when written, then silently false once the collector ladder was extended on its own, which capped every quantile above 5s at a flat 5000.
-  - **OTelHookImpl**: Stores handler function. Called during periodic metric collection (same 1s pattern via PeriodicMetricReader).
-  - **OTelCollectorImp**: Main class.
-    - Creates `MeterProvider` with `PeriodicMetricReader` (1s export interval)
-    - Creates `OtlpHttpMetricExporter` pointing to `[telemetry]` endpoint
-    - Sets resource attributes (service.name, service.instance.id) matching trace exporter
-    - Implements all `make_*()` factory methods
-    - Prefixes metric names with `[insight] prefix=` value
-
-- Guard all OTel SDK includes with `#ifdef XRPL_ENABLE_TELEMETRY` to compile to `NullCollector` equivalents when telemetry disabled.
-
-**Key new files**:
-
-- `include/xrpl/beast/insight/OTelCollector.h`
-- `src/libxrpl/beast/insight/OTelCollector.cpp`
-
-**Key patterns to follow**:
-
-- Match `StatsDCollector.cpp` structure: private impl classes, intrusive list for metrics, strand-based thread safety
-- Match existing telemetry code style from `src/libxrpl/telemetry/Telemetry.cpp`
-- Use RAII for MeterProvider lifecycle (shutdown on destructor)
-
-**Reference**: [03-implementation-strategy.md](./03-implementation-strategy.md) —
-code style and patterns (`04-code-samples.md` was deleted by `d6450631bf`)
-
----
-
-## Task 7.3: Update CollectorManager
-
-**Objective**: Add `server=otel` config option to route metric creation to the new OTel backend.
-
-**What to do**:
-
-- Edit `src/xrpld/app/main/CollectorManager.cpp`:
-  - In the constructor, add a third branch after `server == "statsd"`:
-    ```cpp
-    else if (server == "otel")
-    {
-        // Read endpoint from [telemetry] section
-        auto const endpoint = get(telemetryParams, "endpoint",
-            "http://localhost:4318/v1/metrics");
-        std::string const& prefix(get(params, "prefix"));
-        collector_ = beast::insight::OTelCollector::New(
-            endpoint, prefix, journal);
-    }
-    ```
-  - This requires access to the `[telemetry]` config section — may need to pass it as a parameter or read from Application config.
-
-- Edit `src/xrpld/app/main/CollectorManager.h`:
-  - Add `#include <xrpl/beast/insight/OTelCollector.h>`
-
-**Key modified files**:
-
-- `src/xrpld/app/main/CollectorManager.cpp`
-- `src/xrpld/app/main/CollectorManager.h`
-
----
-
-## Task 7.4: Update OTel Collector Configuration
-
-**Objective**: Add a metrics pipeline to the OTLP receiver and remove the StatsD receiver dependency.
-
-**What to do**:
-
-- Edit `docker/telemetry/otel-collector-config.yaml`:
-  - Remove `statsd` receiver (no longer needed when `server=otel`)
-  - Add metrics pipeline under `service.pipelines`:
-    ```yaml
-    metrics:
-      receivers: [otlp, spanmetrics]
-      processors: [batch]
-      exporters: [prometheus]
-    ```
-  - The OTLP receiver already listens on :4318 — it just needs to be added to the metrics pipeline receivers.
-  - Keep `spanmetrics` connector in the metrics pipeline so span-derived RED metrics continue working.
-
-- Edit `docker/telemetry/docker-compose.yml`:
-  - Remove UDP :8125 port mapping from otel-collector service
-  - Update xrpld service config: change `[insight] server=statsd` to `server=otel`
-
-**Key modified files**:
-
-- `docker/telemetry/otel-collector-config.yaml`
-- `docker/telemetry/docker-compose.yml`
-
-**Note**: Keep a commented-out `statsd` receiver block for operators who need backward compatibility.
-
----
-
-## Task 7.5: Pin Down Metric Names in Prometheus
-
-**Objective**: Fix the exported name of every beast::insight instrument, and point the Grafana dashboards at those names.
-
-**What to do**:
-
-- In `OTelCollector.cpp`, `formatName()` derives the OTel instrument name from the group and name beast::insight was given:
-  - beast::insight `makeGauge("LedgerMaster", "Validated_Ledger_Age")` → OTel instrument name: `ledgermaster_validated_ledger_age`
-  - `Groups` joins group and name with `.`; `formatName()` then lowercases and maps `.` and space to `_`
-  - No prefix is applied on this path. The service is identified by the `service.name` resource attribute, so `[insight] prefix` reaches only the startup log line
-
-- Verify in integration test that key Prometheus queries still return data:
-  - `ledgermaster_validated_ledger_age`
-  - `peer_finder_active_inbound_peers`
-  - `rpc_requests_total`
-
-**Key consideration**: OTel Prometheus exporter may normalize metric names differently than StatsD receiver. Test this early (Task 7.2) and adjust naming strategy if needed. The OTel SDK's Prometheus exporter adds `_total` suffix to counters and converts dots to underscores — match existing conventions.
-
----
-
-## Task 7.6: Update Grafana Dashboards
-
-**Objective**: Update the 3 StatsD dashboards if any metric names change due to OTLP export format differences.
-
-**What to do**:
-
-- If Task 7.5 confirms metric names are preserved exactly, no dashboard changes needed.
-- If OTLP export produces different names (e.g., `_total` suffix on counters), update:
-  - `docker/telemetry/grafana/dashboards/statsd-node-health.json`
-  - `docker/telemetry/grafana/dashboards/statsd-network-traffic.json`
-  - `docker/telemetry/grafana/dashboards/statsd-rpc-pathfinding.json`
-- Rename dashboard titles from "StatsD" to "System Metrics" or similar (since they're no longer StatsD-sourced).
-
-**Key modified files**:
-
-- `docker/telemetry/grafana/dashboards/statsd-*.json` (3 files, conditionally)
-
----
-
-## Task 7.7: Update Integration Tests
-
-**Objective**: Verify the full OTLP metrics pipeline end-to-end.
-
-**What to do**:
-
-- Edit `docker/telemetry/integration-test.sh`:
-  - Update test config to use `[insight] server=otel`
-  - Verify metrics arrive in Prometheus via OTLP (not StatsD)
-  - Add check that StatsD receiver is no longer required
-  - Preserve all existing metric presence checks
-
-**Key modified files**:
-
-- `docker/telemetry/integration-test.sh`
-
----
-
-## Task 7.8: Update Documentation
-
-**Objective**: Update all plan docs, runbook, and reference docs to reflect the migration.
-
-**What to do**:
-
-- Edit `docs/telemetry-runbook.md`:
-  - Update `[insight]` config examples to show `server=otel`
-  - Update troubleshooting section (no more StatsD UDP debugging)
-
-- Edit `OpenTelemetryPlan/09-data-collection-reference.md`:
-  - Update Data Flow Overview diagram (remove StatsD receiver)
-  - Update Section 2 header from "StatsD Metrics" to "System Metrics (OTel native)"
-  - Update config examples
-
-- Edit `OpenTelemetryPlan/05-configuration-reference.md`:
-  - Add `server=otel` option to `[insight]` section docs
-
-- Edit `docker/telemetry/TESTING.md`:
-  - Update setup instructions to use `server=otel`
-
-**Key modified files**:
-
-- `docs/telemetry-runbook.md`
-- `OpenTelemetryPlan/09-data-collection-reference.md`
-- `OpenTelemetryPlan/05-configuration-reference.md`
-- `docker/telemetry/TESTING.md`
-
----
-
-## Task 7.9: ValidationTracker — Validation Agreement Computation
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — the most valuable metric from the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
->
-> **Upstream**: Phase 4 Task 4.8 (validation span attributes provide ledger hash context).
-> **Downstream**: Phase 9 (Validator Health dashboard), Phase 10 (validation checks), Phase 11 (agreement alert rules).
-
-**Objective**: Implement a stateful class that tracks whether our validator's validations agree with network consensus, maintaining rolling 1h and 24h windows with an 8-second grace period and 5-minute late repair window.
-
-**Architecture**:
-
-```
-consensus.validation.send ────> ValidationTracker ────> MetricsRegistry
-(records our validation         (reconciles after       (exports agreement
- for ledger X)                   8s grace period)        gauges every 10s)
-
-ledger.validate ──────────────> ValidationTracker
-(records which ledger           (marks ledger X as
- network validated)              agreed or missed)
-```
-
-**What to do**:
-
-- Create `src/xrpld/telemetry/ValidationTracker.h`:
-  - `recordOurValidation(ledgerHash, ledgerSeq)` — called when we send a validation
-  - `recordNetworkValidation(ledgerHash, seq)` — called when a ledger is fully validated
-  - `reconcile()` — called periodically; reconciles pending ledger events after 8s grace period
-  - Getters: `agreementPct1h()`, `agreementPct24h()`, `agreements1h()`, `missed1h()`, `agreements24h()`, `missed24h()`, `totalAgreements()`, `totalMissed()`, `totalValidationsSent()`, `totalValidationsChecked()`
-  - Thread-safety: atomics for counters, mutex for window deques
-
-- Create `src/xrpld/telemetry/detail/ValidationTracker.cpp`:
-  - Reconciliation logic: after 8s grace period, check if `weValidated && networkValidated && sameHash` → agreement; else missed
-  - Late repair: if a late validation arrives within 5 minutes, correct a false-positive miss
-  - Sliding window: `std::deque<WindowEvent>` evicts entries older than 1h/24h on each reconciliation pass
-  - Ring buffer of 1000 `LedgerEvent` structs for pending reconciliation
-
-- Add recording hooks (modifying Phase 4 code from Phase 7 branch):
-  - `RCLConsensus.cpp` `validate()`: call `tracker.recordOurValidation()`
-  - `LedgerMaster.cpp` fully-validated path: call `tracker.recordNetworkValidation()`
-
-**Key data structures**:
-
-```cpp
-struct LedgerEvent {
-    uint256 ledgerHash;
-    LedgerIndex seq;
-    TimePoint closeTime;
-    bool weValidated = false;
-    bool networkValidated = false;
-    bool reconciled = false;
-    bool agreed = false;
-};
-
-struct WindowEvent {
-    TimePoint time;
-    bool agreed;
-};
-```
-
-**Key new files**:
-
-- `src/xrpld/telemetry/ValidationTracker.h`
-- `src/xrpld/telemetry/detail/ValidationTracker.cpp`
-
-**Key modified files**:
-
-- `src/xrpld/telemetry/MetricsRegistry.h` (add ValidationTracker member)
-- `src/xrpld/telemetry/MetricsRegistry.cpp` (add gauge callback reading from tracker)
-- `src/xrpld/app/consensus/RCLConsensus.cpp` (add recording hooks)
-- `src/xrpld/app/ledger/detail/LedgerMaster.cpp` (add recording hook)
-
-**Exit Criteria**:
-
-- [ ] ValidationTracker correctly tracks agreement with 8s grace period
-- [ ] 5-minute late repair corrects false-positive misses
-- [ ] Thread-safe (atomics + mutex for window deques)
-- [ ] Rolling windows correctly evict stale entries
-- [ ] Unit tests: normal agreement, missed validation, late repair, window eviction
-
----
-
-## Task 7.10: Validator Health Observable Gauges
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
-
-**Objective**: Export amendment blocked, UNL health, and quorum data as a native OTel observable gauge.
-
-**What to do**:
-
-- In `MetricsRegistry.cpp` `registerAsyncGauges()`, add:
-
-```cpp
-validatorHealthGauge_ = meter_->CreateDoubleObservableGauge(
-    "validator_health", "Validator health indicators");
-```
-
-**Gauge label values**:
-
-| Label `metric=`     | Type   | Source                                            |
-| ------------------- | ------ | ------------------------------------------------- |
-| `amendment_blocked` | int64  | `app_.getOPs().isAmendmentBlocked()` → 0/1        |
-| `unl_blocked`       | int64  | `app_.getOPs().isUNLBlocked()` → 0/1              |
-| `unl_expiry_days`   | double | `app_.validators().expires()` → days until expiry |
-| `validation_quorum` | int64  | `app_.validators().quorum()`                      |
-
-### Sub-task 7.10a: Per-Validator Validation Count (Flag Ledger Window)
-
-**Objective**: Track how many ledgers each UNL validator has validated over
-the last 256 consecutive ledgers (one flag ledger window). This is the key
-UNL participation metric — validators consistently below threshold may be
-candidates for removal from the UNL.
-
-**What to do**:
-
-- Add a new observable gauge:
-
-```cpp
-validatorParticipationGauge_ = meter_->CreateInt64ObservableGauge(
-    "validator_participation",
-    "Per-validator validation count over the last 256 ledgers");
-```
-
-- The callback queries `app_.getValidations()` to get the trusted
-  validation set for each of the last 256 ledger hashes (from
-  `LedgerMaster::getValidatedLedger()` walking backwards). For each
-  validator public key in the UNL, count how many of those 256 ledgers
-  have a matching validation.
-
-- **Label dimensions**:
-  - `validator` — base58-encoded validator master public key
-  - `service_instance_id` — this node's identity (standard)
-
-- **Emission**: every flag ledger (256 ledgers, ~15 minutes) or on a
-  10-second async gauge callback with cached results (recompute only
-  at flag ledger boundaries).
-
-- **Data source**: `RCLValidations::getTrustedForLedger(hash, seq)` returns
-  `std::vector<std::shared_ptr<STValidation>>` with `getSignerPublic()`
-  for each. The UNL list is from `app_.getValidators().getTrustedMasterKeys()`.
-
-- **Dashboard panel**: Add a table panel to the Validator Health dashboard
-  showing `validator_participation` grouped by `validator` label,
-  with a threshold color (green >= 240, yellow >= 200, red < 200).
-
-**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
-
-**Exit Criteria**:
-
-- [ ] Gauge emits one time series per UNL validator
-- [ ] Values range 0-256 and update at flag ledger boundaries
-- [ ] Grafana table panel shows per-validator participation
-- [ ] Validators below 75% participation are highlighted in red
-
----
-
-**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
-
-**Exit Criteria**:
-
-- [ ] All 4 base label values emitted every 10s
-- [ ] `unl_expiry_days` is negative when expired, positive when active
-- [ ] Per-validator participation gauge emits at flag ledger boundaries
-- [ ] Values visible in Prometheus
-
----
-
-## Task 7.11: Peer Quality Observable Gauges
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
-
-**Objective**: Export peer health aggregates (latency P90, insane peers, version awareness) as a native OTel observable gauge.
-
-**What to do**:
-
-- In `MetricsRegistry.cpp` `registerAsyncGauges()`, add a callback that iterates `app_.overlay().foreach(...)` to:
-  - Collect per-peer latency values, sort, compute P90
-  - Count peers with `tracking_ == diverged` (insane)
-  - Compare peer `getVersion()` to own version for upgrade awareness
-
-**Gauge label values**:
-
-| Label `metric=`            | Type   | Source                                |
-| -------------------------- | ------ | ------------------------------------- |
-| `peer_latency_p90_ms`      | double | P90 from sorted peer latencies        |
-| `peers_insane_count`       | int64  | Peers with diverged tracking status   |
-| `peers_higher_version_pct` | double | % of peers on newer xrpld version     |
-| `upgrade_recommended`      | int64  | 1 if `peers_higher_version_pct > 60%` |
-
-**Implementation note**: The callback runs every 10s on the metrics reader thread. Iterating ~50-200 peers is acceptable overhead.
-
-**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
-
-**Exit Criteria**:
-
-- [ ] P90 latency computed correctly
-- [ ] Insane count matches `peers` RPC output
-- [ ] Version comparison handles format variations (e.g., "xrpld-2.4.0-rc1")
-
----
-
-## Task 7.12: Ledger Economy Observable Gauges
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
-
-**Objective**: Export fee, reserve, ledger age, and transaction rate as a native OTel observable gauge.
-
-**Gauge label values**:
-
-| Label `metric=`      | Type   | Source                                              |
-| -------------------- | ------ | --------------------------------------------------- |
-| `base_fee_drops`     | double | Base fee from validated ledger fee settings (drops) |
-| `reserve_base_drops` | double | Account reserve from validated ledger (drops)       |
-| `reserve_inc_drops`  | double | Owner reserve increment (drops)                     |
-| `ledger_age_seconds` | double | `now - lastValidatedCloseTime`                      |
-| `transaction_rate`   | double | Derived: tx count delta / time delta (smoothed)     |
-
-**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
-
-**Exit Criteria**:
-
-- [ ] Fee values match `server_info` RPC output
-- [ ] `ledger_age_seconds` increases monotonically between ledger closes
-- [ ] `transaction_rate` is smoothed (rolling average)
-
----
-
-## Task 7.13: State Tracking Observable Gauges
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
-
-**Objective**: Export extended state value (0-6 encoding combining OperatingMode + ConsensusMode) and time-in-current-state.
-
-**Gauge label values**:
-
-| Label `metric=`                 | Type   | Source                                          |
-| ------------------------------- | ------ | ----------------------------------------------- |
-| `state_value`                   | int64  | 0-6 encoding (see spec for mapping)             |
-| `time_in_current_state_seconds` | double | `now - lastModeChangeTime` from StateAccounting |
-
-**State value encoding**: 0=disconnected, 1=connected, 2=syncing, 3=tracking, 4=full, 5=validating (full + validating), 6=proposing (full + proposing).
-
-**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
-
-**Exit Criteria**:
-
-- [ ] `state_value` correctly combines OperatingMode and ConsensusMode
-- [ ] `time_in_current_state_seconds` resets on mode change
-
----
-
-## Task 7.14: Storage Detail and Sync Info Gauges
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
-
-**Objective**: Export the NodeStore stored-bytes total and initial sync duration.
-
-**Gauge label values**:
-
-| Gauge Name       | Label `metric=`                 | Type   | Source                                               |
-| ---------------- | ------------------------------- | ------ | ---------------------------------------------------- |
-| `storage_detail` | `stored_object_bytes`           | int64  | `Database::getStoreSize()` — cumulative object bytes |
-| `sync_info`      | `initial_sync_duration_seconds` | double | Time from start to first FULL                        |
-
-`stored_object_bytes` is not a file size. `getStoreSize()` sums the object payloads
-this process has written, so it excludes NuDB's keys, bucket padding and log, and it
-resets when the process restarts while the files on disk do not. It is the same
-accessor `node_written_bytes` uses, so the two series are equal by construction and
-any write-amplification ratio built from the pair is a constant 1.0. Neither
-`Backend` nor `Database` exposes a file-size accessor, so no metric reports the
-store's on-disk size today.
-
-This label value was `nudb_bytes` when Phase 7 shipped it and was renamed in Phase 9,
-because the value comes from `Database` rather than the NuDB backend.
-
-**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
-
-**Exit Criteria**:
-
-- [ ] Cumulative stored object bytes reported (0 if NuDB not configured)
-- [ ] Sync duration captured once and remains stable after reaching FULL
-
----
-
-## Task 7.15: New Synchronous Counters
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
-
-**Objective**: Add 7 new event counters incremented at their respective instrumentation sites.
-
-| Counter Name                  | Increment Site                   | Source File           |
-| ----------------------------- | -------------------------------- | --------------------- |
-| `ledgers_closed_total`        | `onAccept()` in consensus        | RCLConsensus.cpp      |
-| `validations_sent_total`      | `validate()` in consensus        | RCLConsensus.cpp      |
-| `validations_checked_total`   | Network validation received      | LedgerMaster.cpp      |
-| `validation_agreements_total` | ValidationTracker reconciliation | ValidationTracker.cpp |
-| `validation_missed_total`     | ValidationTracker reconciliation | ValidationTracker.cpp |
-| `state_changes_total`         | `setMode()` in NetworkOPs        | NetworkOPs.cpp        |
-| `jq_trans_overflow_total`     | Job queue overflow path          | JobQueue.cpp          |
-
-**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (declarations), plus recording sites in RCLConsensus.cpp, LedgerMaster.cpp, NetworkOPs.cpp, JobQueue.cpp
-
-**Exit Criteria**:
-
-- [ ] All 7 counters monotonically increase during normal operation
-- [ ] Counter values match expected rates (e.g., ledgers_closed ≈ 1 per 3-5s)
-
----
-
-## Task 7.16: Validation Agreement Observable Gauge
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
-
-**Objective**: Export rolling window agreement stats from `ValidationTracker` (Task 7.9).
-
-**Gauge label values**:
-
-| Gauge Name             | Label `metric=`     | Type   | Source                      |
-| ---------------------- | ------------------- | ------ | --------------------------- |
-| `validation_agreement` | `agreement_pct_1h`  | double | `tracker.agreementPct1h()`  |
-|                        | `agreements_1h`     | int64  | `tracker.agreements1h()`    |
-|                        | `missed_1h`         | int64  | `tracker.missed1h()`        |
-|                        | `agreement_pct_24h` | double | `tracker.agreementPct24h()` |
-|                        | `agreements_24h`    | int64  | `tracker.agreements24h()`   |
-|                        | `missed_24h`        | int64  | `tracker.missed24h()`       |
-
-**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.cpp`
-
-**Exit Criteria**:
-
-- [ ] Agreement percentages in range [0.0, 100.0]
-- [ ] Window stats stabilize after 1h/24h of operation
-
----
-
-## Summary Table
-
-| Task | Description                            | New Files | Modified Files | Depends On |
-| ---- | -------------------------------------- | --------- | -------------- | ---------- |
-| 7.1  | Add OTel Metrics SDK to build deps     | 0         | 2              | —          |
-| 7.2  | Implement OTelCollector class          | 2         | 0              | 7.1        |
-| 7.3  | Update CollectorManager config routing | 0         | 2              | 7.2        |
-| 7.4  | Update OTel Collector YAML and Docker  | 0         | 2              | 7.3        |
-| 7.5  | Preserve metric names in Prometheus    | 0         | 1              | 7.2        |
-| 7.6  | Update Grafana dashboards (if needed)  | 0         | 3              | 7.5        |
-| 7.7  | Update integration tests               | 0         | 1              | 7.4        |
-| 7.8  | Update documentation                   | 0         | 4              | 7.6        |
-| 7.9  | ValidationTracker (agreement tracking) | 2         | 4              | 7.2, P4.8  |
-| 7.10 | Validator health observable gauges     | 0         | 2              | 7.2        |
-| 7.11 | Peer quality observable gauges         | 0         | 2              | 7.2        |
-| 7.12 | Ledger economy observable gauges       | 0         | 2              | 7.2        |
-| 7.13 | State tracking observable gauges       | 0         | 2              | 7.2        |
-| 7.14 | Storage detail and sync info gauges    | 0         | 2              | 7.2        |
-| 7.15 | New synchronous counters               | 0         | 6              | 7.2        |
-| 7.16 | Validation agreement observable gauge  | 0         | 1              | 7.9        |
-
-**Parallel work**: Tasks 7.4 and 7.5 can run in parallel after 7.2/7.3 complete. Task 7.6 depends on 7.5's findings. Tasks 7.7 and 7.8 can run in parallel after 7.6. Tasks 7.10-7.14 can all run in parallel after 7.2. Task 7.15 depends on 7.2. Task 7.16 depends on 7.9. Task 7.9 depends on 7.2 and Phase 4 Task 4.8.
-
-**Exit Criteria** (from [06-implementation-phases.md §6.8](./06-implementation-phases.md)):
-
-- [ ] All 255+ metrics visible in Prometheus via OTLP pipeline (no StatsD receiver)
-- [ ] `server=otel` is the default in development docker-compose
-- [ ] `server=statsd` still works as a fallback
-- [ ] Existing Grafana dashboards display data correctly
-- [ ] Integration test passes with OTLP-only metrics pipeline
-- [ ] No performance regression vs StatsD baseline (< 1% CPU overhead)
-- [ ] Deferred Task 6.1 (`|m` wire format) no longer relevant — Meter mapped to OTel Counter
-- [ ] ValidationTracker agreement % stabilizes after 1h under normal consensus
-- [ ] All new gauges and counters visible in Prometheus with non-zero values

OpenTelemetryPlan/Phase8_taskList.md

@@ -1,239 +0,0 @@
-# Phase 8: Log-Trace Correlation and Centralized Log Ingestion — Task List
-
-> **Goal**: Inject trace context (trace_id, span_id) into xrpld's Journal log output for log-trace correlation, and add OTel Collector filelog receiver to ingest logs into Grafana Loki for unified observability.
->
-> **Scope**: Two independent sub-phases — 8a (code change: trace_id in logs) and 8b (infra only: filelog receiver to Loki). No changes to the `beast::Journal` public API.
->
-> **Branch**: `pratik/otel-phase8-log-correlation` (from `pratik/otel-phase7-native-metrics`)
-
-### Related Plan Documents
-
-| Document                                                         | Relevance                                                      |
-| ---------------------------------------------------------------- | -------------------------------------------------------------- |
-| [06-implementation-phases.md](./06-implementation-phases.md)     | Phase 8 plan: motivation, architecture, exit criteria (§6.8.1) |
-| [07-observability-backends.md](./07-observability-backends.md)   | Loki backend recommendation, Grafana data source provisioning  |
-| [Phase7_taskList.md](./Phase7_taskList.md)                       | Prerequisite — native OTel metrics pipeline must be working    |
-| [05-configuration-reference.md](./05-configuration-reference.md) | `[telemetry]` config (trace_id injection toggle)               |
-
----
-
-## Task 8.1: Inject trace_id into Logs::format()
-
-**Objective**: Add OTel trace context to every log line that is emitted within an active, sampled span. The sampled flag matters because a span dropped by the `ParentBasedSampler` still carries its parent's ids, so emitting them would advertise a trace that was never exported.
-
-**What to do**:
-
-- Edit `src/libxrpl/basics/Log.cpp`:
-  - In `Logs::format()` (around line 346), after severity is appended, check for active OTel span. The implementation checks the context value directly to avoid the heap allocation that `GetSpan()` performs on the no-span path:
-    ```cpp
-    #ifdef XRPL_ENABLE_TELEMETRY
-    {
-        auto context = opentelemetry::context::RuntimeContext::GetCurrent();
-        auto spanValue = context.GetValue(opentelemetry::trace::kSpanKey);
-        if (opentelemetry::nostd::holds_alternative<
-                opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>>(spanValue))
-        {
-            auto span = opentelemetry::nostd::get<
-                opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>>(spanValue);
-            auto spanCtx = span->GetContext();
-            if (spanCtx.IsValid() && spanCtx.IsSampled())
-            {
-                char traceId[32], spanId[16];
-                spanCtx.trace_id().ToLowerBase16(
-                    opentelemetry::nostd::span<char, 32>{traceId});
-                spanCtx.span_id().ToLowerBase16(
-                    opentelemetry::nostd::span<char, 16>{spanId});
-                output += "trace_id=";
-                output.append(traceId, 32);
-                output += " span_id=";
-                output.append(spanId, 16);
-                output += ' ';
-            }
-        }
-    }
-    #endif
-    ```
-  - Add `#include` for OTel context headers, guarded by `#ifdef XRPL_ENABLE_TELEMETRY`
-
-- Edit `include/xrpl/basics/Log.h`:
-  - No changes needed — format() signature unchanged
-
-**Key modified files**:
-
-- `src/libxrpl/basics/Log.cpp`
-
-**Performance note**: The implementation checks the thread-local context value directly (avoiding the heap allocation that `GetSpan()` performs on the no-span path). On threads without an active span (~99% of log lines), the cost is a thread-local read + variant type check (~15-20ns). On the active-span path, an additional shared_ptr copy + `GetContext()` + `IsValid()`/`IsSampled()` adds ~50ns total. Overhead is negligible at typical logging rates.
-
----
-
-## Task 8.2: Add Loki to Docker Compose Stack
-
-**Objective**: Add Grafana Loki as a log storage backend in the development observability stack.
-
-**What to do**:
-
-- Edit `docker/telemetry/docker-compose.yml`:
-  - Add Loki service:
-    ```yaml
-    loki:
-      image: grafana/loki:3.7.6
-      ports:
-        - "3100:3100"
-      command: -config.file=/etc/loki/local-config.yaml
-    ```
-  - Add Loki as a Grafana data source in provisioning
-
-- Create `docker/telemetry/grafana/provisioning/datasources/loki.yaml`:
-  - Configure Loki data source with derived fields linking `trace_id` to Tempo
-
-**Key new files**:
-
-- `docker/telemetry/grafana/provisioning/datasources/loki.yaml`
-
-**Key modified files**:
-
-- `docker/telemetry/docker-compose.yml`
-
----
-
-## Task 8.3: Add Filelog Receiver to OTel Collector
-
-**Objective**: Configure the OTel Collector to tail xrpld's log file and export to Loki.
-
-**What to do**:
-
-- Edit `docker/telemetry/otel-collector-config.yaml`:
-  - Add `filelog` receiver:
-    ```yaml
-    receivers:
-      filelog:
-        include: [/var/log/xrpld/*/debug.log]
-        operators:
-          - type: regex_parser
-            regex: '^(?P<timestamp>\S+)\s+(?P<partition>\S+):(?P<severity>\S+)\s+(?:trace_id=(?P<trace_id>[a-f0-9]+)\s+span_id=(?P<span_id>[a-f0-9]+)\s+)?(?P<message>.*)$'
-            timestamp:
-              parse_from: attributes.timestamp
-              layout: "%Y-%m-%dT%H:%M:%S.%fZ"
-    ```
-  - Add logs pipeline:
-    ```yaml
-    service:
-      pipelines:
-        logs:
-          receivers: [filelog]
-          processors: [batch]
-          exporters: [otlp/loki]
-    ```
-  - Add Loki exporter:
-    ```yaml
-    exporters:
-      otlphttp/loki:
-        endpoint: http://loki:3100/otlp
-    ```
-
-- Mount xrpld's log directory into the collector container via docker-compose volume
-
-**Key modified files**:
-
-- `docker/telemetry/otel-collector-config.yaml`
-- `docker/telemetry/docker-compose.yml`
-
----
-
-## Task 8.4: Configure Grafana Trace-to-Log Correlation
-
-**Objective**: Enable one-click navigation from Tempo traces to Loki logs in Grafana.
-
-**What to do**:
-
-- Edit Grafana Tempo data source provisioning to add `tracesToLogs` configuration:
-
-  ```yaml
-  tracesToLogs:
-    datasourceUid: loki
-    filterByTraceID: true
-    filterBySpanID: false
-    tags: ["partition", "severity"]
-  ```
-
-- Edit Grafana Loki data source provisioning to add `derivedFields` linking trace_id back to Tempo:
-  ```yaml
-  derivedFields:
-    - datasourceUid: tempo
-      matcherRegex: "trace_id=(\\w+)"
-      name: TraceID
-      url: "$${__value.raw}"
-  ```
-
-**Key modified files**:
-
-- `docker/telemetry/grafana/provisioning/datasources/loki.yaml`
-- `docker/telemetry/grafana/provisioning/datasources/` (Tempo data source file)
-
----
-
-## Task 8.5: Update Integration Tests
-
-**Objective**: Verify trace_id appears in logs and Loki correlation works.
-
-**What to do**:
-
-- Edit `docker/telemetry/integration-test.sh`:
-  - After sending RPC requests (which create spans), grep xrpld's log output for `trace_id=`
-  - Verify trace_id matches a trace visible in Tempo
-  - Optionally: query Loki via API to confirm log ingestion
-
-**Key modified files**:
-
-- `docker/telemetry/integration-test.sh`
-
----
-
-## Task 8.6: Update Documentation
-
-**Objective**: Document the log correlation feature in runbook and reference docs.
-
-**What to do**:
-
-- Edit `docs/telemetry-runbook.md`:
-  - Add "Log-Trace Correlation" section explaining how to use Grafana Tempo -> Loki linking
-  - Add LogQL query examples for filtering by trace_id
-
-- Edit `OpenTelemetryPlan/09-data-collection-reference.md`:
-  - Add new section "3. Log Correlation" between SpanMetrics and StatsD sections
-  - Document the log format with trace_id injection
-  - Document Loki as a new backend
-
-- Edit `docker/telemetry/TESTING.md`:
-  - Add log correlation verification steps
-
-**Key modified files**:
-
-- `docs/telemetry-runbook.md`
-- `OpenTelemetryPlan/09-data-collection-reference.md`
-- `docker/telemetry/TESTING.md`
-
----
-
-## Summary Table
-
-| Task | Description                                | Sub-Phase | New Files | Modified Files | Depends On |
-| ---- | ------------------------------------------ | --------- | --------- | -------------- | ---------- |
-| 8.1  | Inject trace_id into Logs::format()        | 8a        | 0         | 1              | Phase 7    |
-| 8.2  | Add Loki to Docker Compose stack           | 8b        | 1         | 1              | --         |
-| 8.3  | Add filelog receiver to OTel Collector     | 8b        | 0         | 2              | 8.1, 8.2   |
-| 8.4  | Configure Grafana trace-to-log correlation | 8b        | 0         | 2              | 8.3        |
-| 8.5  | Update integration tests                   | 8a + 8b   | 0         | 1              | 8.4        |
-| 8.6  | Update documentation                       | 8a + 8b   | 0         | 3              | 8.5        |
-
-**Parallel work**: Task 8.2 (Loki infra) can run in parallel with Task 8.1 (code change). Tasks 8.3-8.6 are sequential.
-
-**Exit Criteria** (from [06-implementation-phases.md §6.8.1](./06-implementation-phases.md)):
-
-- [ ] Log lines within active, sampled spans contain `trace_id=<hex> span_id=<hex>`
-- [ ] Log lines outside spans have no trace context (no empty fields)
-- [ ] Loki ingests xrpld logs via OTel Collector filelog receiver
-- [ ] Grafana Tempo -> Loki one-click correlation works
-- [ ] Grafana Loki -> Tempo reverse lookup works via derived field
-- [ ] Integration test verifies trace_id presence in logs
-- [ ] No performance regression from trace_id injection (< 0.1% overhead)

OpenTelemetryPlan/Phase9_taskList.md

@@ -1,923 +0,0 @@
-<!-- cspell:ignore ISTOGRAM -->
-<!-- The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's
-     compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. -->
-
-# Phase 9: Internal Metric Instrumentation Gap Fill — Task List
-
-> **Status**: Complete for Tasks 9.1-9.13. Tasks 9.14-9.17 remain open by design
-> (see each task for the blocker).
->
-> **Goal**: Instrument xrpld to emit ~50+ metrics that exist in `get_counts`/`server_info`/TxQ/PerfLog but currently lack time-series export via the OTel or beast::insight pipelines.
->
-> **Scope**: Hybrid approach — extend `beast::insight` for metrics near existing registrations, use OTel Metrics SDK `ObservableGauge` callbacks for new categories (TxQ, PerfLog, CountedObjects).
->
-> **Branch**: `pratik/otel-phase9-metric-gap-fill` (from `pratik/otel-phase8-log-correlation`)
->
-> **Depends on**: Phase 7 (native OTel metrics pipeline) and Phase 8 (log-trace correlation)
-
-> **Note on metric names**: there is **no `xrpld_` prefix** on any emitted
-> metric. `77f35c03db` removed it and lowercased names, and
-> `OTelCollectorImp::formatName()`
-> (`src/libxrpl/beast/insight/OTelCollector.cpp:855-874`) adds no prefix at all —
-> it only lowercases the raw name and turns `.` and spaces into `_`. Earlier
-> revisions of this task list spelled every metric `xrpld_<name>`; those spellings
-> have been corrected in place to the emitted names, so the names below can be
-> pasted into Prometheus as written. Instruments created in
-> `src/xrpld/telemetry/MetricsRegistry.cpp` (35 of them) are the single source of
-> truth. `MetricsRegistry.h`'s Doxygen used to disagree on three histogram names;
-> those header comments were repaired in this change set (see Tasks 9.4 and 9.5),
-> so header and `.cpp` now agree.
->
-> **Two shapes do not simply lose the prefix**, so `xrpld_<name>` → `<name>` is
-> not a blanket rule:
->
-> - **Multiplexed observable gauges.** Most of the value names in these task
->   descriptions are a **`metric` label value** on a shared instrument, not a
->   standalone metric name — queue depth is `txq_metrics{metric="txq_count"}`, not
->   `txq_count`. The same applies to `nodestore_state`, `cache_metrics`,
->   `load_factor_metrics`, `server_info`, `db_metrics`, `validator_health`,
->   `peer_quality`, `state_tracking` and `ledger_economy`. Each task below names
->   its owning instrument.
-> - **Unit-suffixed histograms** coming through `beast::insight`.
->   `OTelCollectorImp` appends the unit to the name, so the `ios_latency`
->   histogram is `ios_latency_milliseconds_bucket` in Prometheus — not
->   `ios_latency_bucket`. Instruments created directly on `MetricsRegistry` keep
->   their literal name (`job_queued_us_bucket`, `rpc_method_us_bucket`) because
->   the unit is already in the instrument name.
-
-### Related Plan Documents
-
-| Document                                                             | Relevance                                                      |
-| -------------------------------------------------------------------- | -------------------------------------------------------------- |
-| [06-implementation-phases.md](./06-implementation-phases.md)         | Phase 9 plan: motivation, architecture, exit criteria (§6.8.2) |
-| [09-data-collection-reference.md](./09-data-collection-reference.md) | Current metric inventory + future metrics section              |
-| [Phase7_taskList.md](./Phase7_taskList.md)                           | Prerequisite — OTel Metrics SDK and `OTelCollector` class      |
-| [Phase8_taskList.md](./Phase8_taskList.md)                           | Prerequisite — log-trace correlation                           |
-
-### Third-Party Consumer Context
-
-These metrics serve multiple external consumer categories identified during research:
-
-| Consumer Category         | Key Metrics They Need                                           |
-| ------------------------- | --------------------------------------------------------------- |
-| **Exchanges**             | Fee escalation levels, TxQ depth, settlement latency            |
-| **Payment Processors**    | Load factors, io_latency, transaction throughput                |
-| **Analytics Providers**   | NodeStore I/O, cache hit rates, counted objects                 |
-| **Validators/Operators**  | Per-job execution times, PerfLog RPC counters, consensus timing |
-| **Academic Researchers**  | Consensus performance time-series, fee market dynamics          |
-| **Institutional Custody** | Server health scores, reserve calculations, node availability   |
-
----
-
-## Task 9.1: NodeStore I/O Metrics
-
-**Objective**: Export node store read/write performance as time-series metrics.
-
-**What to do**:
-
-> **As shipped, this did _not_ go through `beast::insight`.** `Database.cpp` has
-> no insight members. The metrics are a single `nodestore_state`
-> `Int64ObservableGauge` on `MetricsRegistry`
-> (`src/xrpld/telemetry/MetricsRegistry.cpp:957-965`) whose callback reads
-> `Database`'s public accessors (`getFetchTotalCount()`, `getFetchHitCount()`,
-> `getStoreCount()`, `getFetchDurationUs()`, `getStoreDurationUs()`, …) and
-> multiplexes every value onto the `metric` label. Write-queue depth comes from
-> the new `include/xrpl/nodestore/WriteStats.h`.
-
-- Export the following as `nodestore_state{metric="…"}` label values:
-  - Gauge: `node_reads_total` (cumulative read operations)
-  - Gauge: `node_reads_hit` (fetches that found an object — not a cache hit; `fetchHitCount_` increments whatever served the fetch)
-  - Gauge: `node_writes` (cumulative write operations)
-  - Gauge: `node_written_bytes` (cumulative bytes written)
-  - Gauge: `node_read_bytes` (cumulative bytes read)
-  - Gauge: `node_reads_duration_us` (cumulative read time in microseconds)
-  - Gauge: `write_load` (current write load score)
-  - Gauge: `read_queue` (items in read queue)
-
-- These values are already computed in `Database::getCountsJson()`. The gauge
-  callback reads the same counters through `Database`'s public accessors.
-
-**Key modified files**:
-
-- `src/xrpld/telemetry/MetricsRegistry.cpp` (the `nodestore_state` gauge)
-- `include/xrpl/nodestore/Database.h` (accessors; **not** `src/libxrpl/nodestore/Database.h`, which does not exist)
-- `include/xrpl/nodestore/WriteStats.h` (new — write-queue depth snapshot)
-
-**Derived Prometheus metrics**: `nodestore_state{metric="node_reads_total"}`,
-`nodestore_state{metric="node_reads_hit"}`, `nodestore_state{metric="write_load"}`,
-etc. There is **no** `xrpld_` prefix — `OTelCollectorImp::formatName()` adds none.
-
-**Grafana dashboard**: Add "NodeStore I/O" panel group to _Node Health_ dashboard.
-
----
-
-## Task 9.2: Cache Hit Rate Metrics
-
-**Objective**: Export SHAMap and ledger cache performance as time-series gauges.
-
-**What to do**:
-
-- Register OTel `ObservableGauge` callbacks (via Phase 7's `OTelCollector`) for:
-  - `SLE_hit_rate` — SLE cache hit rate (0.0–1.0)
-  - `ledger_hit_rate` — Ledger object cache hit rate
-  - `AL_hit_rate` — AcceptedLedger cache hit rate
-  - `treenode_cache_size` — SHAMap TreeNode cache size (entries)
-  - `treenode_track_size` — Tracked tree nodes
-  - `fullbelow_size` — FullBelow cache size
-
-- The callback reads from the same sources as the `GetCounts` handler
-  (`src/xrpld/rpc/handlers/admin/status/GetCounts.cpp` — **not**
-  `src/xrpld/rpc/handlers/GetCounts.cpp`).
-
-- Create a centralized `MetricsRegistry` class that holds all OTel async gauge registrations, polled at 10-second intervals by the `PeriodicMetricReader`.
-
-**Key modified files**:
-
-- New: `src/xrpld/telemetry/MetricsRegistry.h` / `.cpp`
-- New: `src/xrpld/telemetry/MetricMacros.h` (the `XRPL_METRIC_*` call-site macros)
-- `src/xrpld/rpc/handlers/admin/status/GetCounts.cpp` (extract shared access methods)
-- `src/xrpld/app/main/Application.cpp` (register MetricsRegistry at startup)
-
-**Derived Prometheus metrics**: `cache_metrics{metric="SLE_hit_rate"}`,
-`cache_metrics{metric="ledger_hit_rate"}`, `cache_metrics{metric="treenode_cache_size"}`,
-etc. Label values are **case-sensitive** (`SLE_hit_rate`, `AL_size`, `AL_hit_rate`).
-
----
-
-## Task 9.3: Transaction Queue (TxQ) Metrics
-
-**Objective**: Export TxQ depth, capacity, and fee escalation levels as time-series.
-
-**What to do**:
-
-- Register OTel `ObservableGauge` callbacks for TxQ state (from
-  `src/xrpld/app/misc/TxQ.h` — **not** `src/xrpld/app/tx/detail/TxQ.h`):
-  - `txq_count` — Current transactions in queue
-  - `txq_max_size` — Maximum queue capacity
-  - `txq_in_ledger` — Transactions in current open ledger
-  - `txq_per_ledger` — Expected transactions per ledger
-  - `txq_reference_fee_level` — Reference fee level
-  - `txq_min_processing_fee_level` — Minimum fee to get processed
-  - `txq_med_fee_level` — Median fee level in queue
-  - `txq_open_ledger_fee_level` — Open ledger fee escalation level
-
-- Add to the `MetricsRegistry` (Task 9.2).
-
-**Key modified files**:
-
-- `src/xrpld/telemetry/MetricsRegistry.cpp` (add TxQ callbacks)
-- `src/xrpld/app/misc/TxQ.h` (expose metrics accessor if needed)
-
-**Derived Prometheus metrics**: `txq_metrics{metric="txq_count"}`,
-`txq_metrics{metric="txq_max_size"}`, `txq_metrics{metric="txq_open_ledger_fee_level"}`, etc.
-There is one instrument, `txq_metrics` (`MetricsRegistry.cpp:705`); each value above
-is a `metric` label value, not a metric name of its own.
-
-**Grafana dashboard**: New _Fee Market & TxQ_ dashboard (`fee-market`).
-
----
-
-## Task 9.4: PerfLog Per-RPC Method Metrics
-
-**Objective**: Export per-RPC-method call counts and latency as OTel metrics.
-
-**What to do**:
-
-- Register OTel instruments for PerfLog RPC counters (from `PerfLogImp.cpp`):
-  - Counter: `rpc_method_started_total{method="<name>"}` — calls started
-  - Counter: `rpc_method_finished_total{method="<name>"}` — calls completed
-  - Counter: `rpc_method_errored_total{method="<name>"}` — calls errored
-  - Histogram: `rpc_method_us{method="<name>"}` — execution time distribution
-
-- Use OTel `Counter<uint64_t>` and `Histogram<double>` instruments with the
-  `method` attribute label. The RPC instruments carry **only** `method`
-  (`MetricsRegistry.cpp:436-475`) — the `handler` label belongs to the job
-  instruments (Task 9.5), not these.
-
-> **Naming**: the instrument is `rpc_method_us` — declared as
-> `kRpcMethodDurationUs` at `MetricsRegistry.cpp:96` and used both to register the
-> explicit-bucket view and to create the instrument. `MetricsRegistry.h`'s Doxygen
-> comment used to read `rpc_method_duration_us`; **that was fixed in this change**
-> (`MetricsRegistry.h:789`), so header and `.cpp` now agree and there is no
-> caveat left. The prefix `xrpld_` in the original spec is not emitted by anything.
->
-> Same for the job histograms in Task 9.5: `job_queued_us` / `job_running_us`.
-
-- Hook into the existing PerfLog callback mechanism rather than adding new instrumentation points.
-
-**Key modified files**:
-
-- `src/xrpld/perflog/detail/PerfLogImp.cpp` (add OTel instrument updates alongside existing JSON counters)
-- `src/xrpld/telemetry/MetricsRegistry.cpp` (register instruments)
-
-**Derived Prometheus metrics**: `rpc_method_started_total{method="server_info"}`, `rpc_method_us_bucket{method="ledger"}`, etc.
-
-**Grafana dashboard**: Add "Per-Method RPC Breakdown" panel group to _RPC Performance_ dashboard.
-
----
-
-## Task 9.5: PerfLog Per-Job-Type Metrics
-
-**Objective**: Export per-job-type queue and execution metrics.
-
-**What to do**:
-
-- Register OTel instruments for PerfLog job counters. All five carry **two**
-  labels — `job_type` and `handler` — so producers sharing a job type stay
-  distinguishable (`MetricsRegistry.h:794-818`, recorded at
-  `MetricsRegistry.cpp:498,518,527,548,553`). `handler` is the sanitised
-  `addJob` name; `sanitiseHandler()` folds dynamic names into a bounded domain
-  of exactly 44 values, so cardinality stays fixed.
-  - Counter: `job_queued_total{job_type="<name>",handler="<name>"}` — jobs queued
-  - Counter: `job_started_total{job_type="<name>",handler="<name>"}` — jobs started
-  - Counter: `job_finished_total{job_type="<name>",handler="<name>"}` — jobs completed
-  - Histogram: `job_queued_us{job_type="<name>",handler="<name>"}` — time spent waiting in queue
-  - Histogram: `job_running_us{job_type="<name>",handler="<name>"}` — execution time distribution
-
-> **Naming**: the instruments are `job_queued_us` / `job_running_us`
-> (`kJobQueuedDurationUs` / `kJobRunningDurationUs` in `MetricsRegistry.cpp`).
-> `MetricsRegistry.h`'s Doxygen comments used to read
-> `job_queued_duration_us` / `job_running_duration_us`; **both were fixed in this
-> change** (the `jobQueuedDurationHistogram_` and `jobRunningDurationHistogram_`
-> comments in `MetricsRegistry.h`), so there is no header/`.cpp` divergence
-> left to work around.
-
-- Hook into PerfLog's existing job tracking alongside Task 9.4.
-
-**Key modified files**:
-
-- `src/xrpld/perflog/detail/PerfLogImp.cpp`
-- `src/xrpld/telemetry/MetricsRegistry.cpp`
-
-**Derived Prometheus metrics**: `job_queued_total{job_type="ledgerData",handler="ProcessLData"}`, `job_running_us_bucket{job_type="transaction",handler="…"}`, etc.
-
-**Grafana dashboard**: New _Job Queue Analysis_ dashboard (`job-queue`).
-
----
-
-## Task 9.6: Counted Object Instance Metrics
-
-**Objective**: Export live instance counts for key internal object types.
-
-**What to do**:
-
-- Register OTel `ObservableGauge` callbacks for `CountedObject<T>` instance counts:
-  - `object_count{type="xrpl::Transaction"}` — live Transaction objects
-  - `object_count{type="xrpl::Ledger"}` — live Ledger objects
-  - `object_count{type="xrpl::NodeObject"}` — live NodeObject instances
-  - `object_count{type="xrpl::STTx"}` — serialized transaction objects
-  - `object_count{type="xrpl::STLedgerEntry"}` — serialized ledger entries
-  - `object_count{type="xrpl::InboundLedger"}` — ledgers being fetched
-  - `object_count{type="xrpl::Pathfinder"}` — active pathfinding computations
-  - `object_count{type="xrpl::PathRequest"}` — active path requests
-  - `object_count{type="xrpl::HashRouter::Entry"}` — hash router entries (the type is
-    `HashRouter::Entry`; there is no `HashRouterEntry` type)
-
-- The `CountedObject` template already tracks these via atomic counters. The callback just reads the current counts.
-
-**Key modified files**:
-
-- `src/xrpld/telemetry/MetricsRegistry.cpp` (add counted object callbacks)
-- `include/xrpl/basics/CountedObject.h` (may need static accessor for iteration)
-
-**Derived Prometheus metrics**: `object_count{type="xrpl::Transaction"}`, `object_count{type="xrpl::NodeObject"}`, etc.
-The `type` label value is `beast::typeName<Object>()` — the fully-qualified
-demangled C++ type name (`CountedObject.h:109`), not a short word.
-
-**Grafana dashboard**: Add "Object Instance Counts" panel to _Node Health_ dashboard.
-
----
-
-## Task 9.7: Fee Escalation & Load Factor Metrics
-
-**Objective**: Export the full load factor breakdown as time-series.
-
-**What to do**:
-
-- Register OTel `ObservableGauge` callbacks for load factors (from `NetworkOPs.cpp` line ~2694):
-  - `load_factor` — combined transaction cost multiplier
-  - `load_factor_server` — server + cluster + network contribution
-  - `load_factor_local` — local server load only
-  - `load_factor_net` — network-wide load estimate
-  - `load_factor_cluster` — cluster peer load
-  - `load_factor_fee_escalation` — open ledger fee escalation
-  - `load_factor_fee_queue` — queue entry fee level
-
-- These overlap with some existing StatsD metrics but provide finer granularity (individual factor breakdown vs. combined value).
-
-**Key modified files**:
-
-- `src/xrpld/telemetry/MetricsRegistry.cpp`
-- `src/xrpld/app/misc/NetworkOPs.cpp` (expose load factor accessors if needed)
-
-**Derived Prometheus metrics**: `load_factor_metrics{metric="load_factor"}`,
-`load_factor_metrics{metric="load_factor_fee_escalation"}`, etc. There is one
-instrument, `load_factor_metrics` (`MetricsRegistry.cpp:785`); every value listed
-above is a `metric` label value, not a metric name of its own.
-
-**Grafana dashboard**: Add "Load Factor Breakdown" panel to _Fee Market & TxQ_ dashboard.
-
----
-
-## Task 9.7a: push_metrics.py Parity — Missing Observable Gauges
-
-**Objective**: Fill the remaining metric gaps between the external `push_metrics.py` script (in `ripplex-ansible`) and the internal OTel `MetricsRegistry` observable gauges. After this task, all metrics collected by `push_metrics.py` that CAN be collected internally are covered.
-
-**What was done**:
-
-- Extended existing `cacheHitRateGauge_` callback with `AL_size` (AcceptedLedger cache size)
-- Extended existing `nodeStoreGauge_` callback with 4 new metrics from `getCountsJson()`:
-  - `node_reads_duration_us` (JSON string — uses `std::stoll(asString())`)
-  - `read_request_bundle` (native JSON int)
-  - `read_threads_running` (native JSON int)
-  - `read_threads_total` (native JSON int)
-- Added new `server_info` Int64ObservableGauge with 8 metrics:
-  - `server_state` — operating mode as int (0=DISCONNECTED .. 4=FULL)
-  - `uptime` — seconds since server start
-  - `peers` — total peer count
-  - `validated_ledger_seq` — validated ledger sequence (atomic read)
-  - `ledger_current_index` — current open ledger sequence
-  - `peer_disconnects_resources` — cumulative resource-related disconnects
-  - `last_close_proposers` — from `getConsensusInfo()["previous_proposers"]`
-  - `last_close_converge_time_ms` — from `getConsensusInfo()["previous_mseconds"]`
-- Added new `build_info` Int64ObservableGauge (info-style, value=1 with `version` label)
-- Added new `complete_ledgers` Int64ObservableGauge parsing comma-separated ranges into `{bound, index}` pairs
-- Added new `db_metrics` Int64ObservableGauge with 4 metrics:
-  - `db_kb_total`, `db_kb_ledger`, `db_kb_transaction` (SQLite stat queries)
-  - `historical_perminute` (historical ledger fetch rate)
-
-**Key modified files**:
-
-- `src/xrpld/telemetry/MetricsRegistry.h` (4 new gauge members, updated ASCII diagram)
-- `src/xrpld/telemetry/MetricsRegistry.cpp` (4 new callback registrations, 2 callback extensions)
-
-**Not implementable inside xrpld**:
-
-- `connection_count_51233/51234` — OS-level port connection counts from external shell script (`get_connection.sh`)
-
-**Derived Prometheus metrics**: `server_info{metric="server_state"}`, `build_info{version="2.4.0"}`, `complete_ledgers{bound="start",index="0"}`, `db_metrics{metric="db_kb_total"}`, etc.
-
-**Grafana dashboard**: New panels added to _Node Health_ dashboard (`node-health.json`).
-
----
-
-## Task 9.8: New Grafana Dashboards
-
-**Objective**: Create Grafana dashboards for the new metric categories.
-
-**What to do**:
-
-- Create 2 new dashboards:
-  1. **Fee Market & TxQ** (`fee-market`) — TxQ depth/capacity, fee levels, load factor breakdown, fee escalation timeline
-  2. **Job Queue Analysis** (`job-queue`) — Per-job-type rates, queue wait times, execution times, job queue depth
-
-- Update 2 existing dashboards:
-  1. **Node Health** (`node-health`) — Add NodeStore I/O panels, cache hit rate panels, object instance counts
-  2. **RPC Performance** (`rpc-performance`) — Add per-method RPC breakdown panels
-
-> Tasks 9.11-9.13 add two more new dashboards (`validator-health`,
-> `peer-quality`), so Phase 9's total is **4 new + 2 updated**.
-
-**Key modified files** (filenames and uids after the `dashboards/rippled-*` →
-bare rename in `145b1469d6` and `25868f2740` — the
-`dashboards/rippled-*.json` paths no longer exist):
-
-- New: `docker/telemetry/grafana/dashboards/fee-market.json` (uid `fee-market`)
-- New: `docker/telemetry/grafana/dashboards/job-queue.json` (uid `job-queue`)
-- `docker/telemetry/grafana/dashboards/node-health.json` (uid `node-health`)
-- `docker/telemetry/grafana/dashboards/rpc-performance.json` (uid `rpc-performance`)
-
----
-
-## Task 9.9: Update Documentation
-
-**Objective**: Update telemetry reference docs with all new metrics.
-
-**What to do**:
-
-- Update `OpenTelemetryPlan/09-data-collection-reference.md`: ✅ done
-  - Add new section for OTel SDK-exported metrics (NodeStore, cache, TxQ, PerfLog, CountedObjects, load factors) — §5b + "Phase 9: OTel SDK-Exported Metrics (MetricsRegistry)"
-  - Update Grafana dashboard reference table (add 4 new dashboards) — "New Grafana Dashboards (Phase 9)" / "Updated Grafana Dashboards (Phase 9)"
-  - Add Prometheus query examples for new metrics
-
-- Update `docs/telemetry-runbook.md`:
-  - ✅ Alerting section covering the provisioned rules and how to wire a receiver
-  - ✅ Troubleshooting entries for new metric categories
-  - ❌ **Still open**: dashboard guides for **six** dashboards — `fee-market`,
-    `job-queue`, `ledger-data-sync`, `overlay-traffic-detail`, `peer-quality` and
-    `validator-health`. The runbook records the gap at its dashboard reference
-    section: these six are provisioned but have no section there yet. Also still
-    open: the Validation Agreement explainer (8s grace / 5m late repair)
-
-- Provision Grafana alert rules (`docker/telemetry/grafana/provisioning/alerting/`) — **as shipped**:
-  - **14 rules in 5 groups**: `xrpld-consensus` (`LedgerHistoryMismatch`,
-    `LedgerCloseStalled`, `ValidatedLedgerStale`), `xrpld-validator`
-    (`ValidationsMissed`, `ValidationsNotChecked`), `xrpld-jobqueue`
-    (`JobQueueTxOverflow`, `JobQueueLatencyHigh`, `IOEventLoopLatencyHigh`),
-    `xrpld-node-state` (`NodeStateFlapping`, `NodeNotFull`), `xrpld-overlay`
-    (`ManifestJobQueueConvoy`, `ManifestFloodInbound`, `PeerResourceDisconnects`,
-    `UntrustedValidationFlood`)
-  - **2 contact points** — `xrpld-default` (Slack) and `xrpld-critical`
-    (Slack + email) — and a **nested** notification policy: root →
-    `xrpld-default`, child route `severity = critical` → `xrpld-critical`.
-    Auto-loaded via the existing `provisioning/` mount (no docker-compose change)
-  - 3 rules are `severity: critical`, 11 are `severity: warning`
-  - Alerting operator docs (per-alert meaning, tuning, receiver wiring) now live in the Alerting section of `docs/telemetry-runbook.md`
-
-**Key modified files**:
-
-- `OpenTelemetryPlan/09-data-collection-reference.md`
-- `docs/telemetry-runbook.md`
-- `docker/telemetry/grafana/provisioning/alerting/{rules,contactpoints,policies}.yaml` (new)
-- `docs/telemetry-runbook.md` (Alerting section added)
-
----
-
-## Task 9.10: Integration Tests
-
-**Objective**: Verify all new metrics appear in Prometheus after a test workload.
-
-**What to do**:
-
-- ❌ **Not done on this branch**: extend the telemetry integration test to
-  start xrpld with `[telemetry] enabled=1` / `[insight] server=otel`, drive RPC
-  and transaction load, query Prometheus for each new metric family and assert
-  non-zero values. The end-to-end metric assertions live in the **Phase 10**
-  harness (`docker/telemetry/workload/expected_metrics.json`), not here.
-
-- ✅ **Done**: unit tests for the `MetricsRegistry` class —
-  `src/tests/libxrpl/telemetry/MetricsRegistry.cpp` (**18** GTest cases —
-  `grep -cE '\bTEST(_F|_P)?\s*\(' src/tests/libxrpl/telemetry/MetricsRegistry.cpp`
-  = 18, and the four bullets below sum to 4 + 3 + 5 + 6 = 18):
-  - Callback registration / deregistration and shutdown ordering —
-    `async_gauges_start_after_start_is_safe`,
-    `async_gauges_before_start_does_not_break_start`,
-    `async_gauges_respect_the_compile_time_guard`, `destructor_calls_stop`
-  - Graceful behaviour when telemetry is disabled — `disabled_construction`,
-    `disabled_start_stop`, `disabled_recording_methods`
-  - Label sanitisation and mean scaling — `MetricsRegistrySanitiseHandler` (5
-    cases, incl. `output_domain_is_exactly_44_values`) and
-    `MetricsRegistryScaledMean` (6 cases)
-  - ❌ Not covered: asserting metric values match `get_counts` JSON output —
-    that needs a live `Application`, so it is left to the Phase 10 harness
-
-**Key files**:
-
-- `src/tests/libxrpl/telemetry/MetricsRegistry.cpp` (new). The originally
-  planned `src/test/telemetry/MetricsRegistry_test.cpp` was **never created** —
-  Phase 9 tests are GTest under `src/tests/libxrpl/`, per project convention.
-- `src/tests/libxrpl/telemetry/MetricMacros.cpp`, `GetMeter.cpp` (new — cover
-  the `XRPL_METRIC_*` macros and meter lookup)
-
----
-
-## Task 9.11: Validator Health Dashboard (External Dashboard Parity)
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — dashboards for Phase 7 metrics inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
->
-> **Upstream**: Phase 7 Tasks 7.9-7.16 (metrics must be emitting).
-> **Downstream**: Phase 10 (dashboard load checks), Phase 11 (alert rules reference these panels).
-
-**Objective**: Create a Grafana dashboard for validation agreement, amendment/UNL health, and state tracking.
-
-**Dashboard**: `validator-health.json`
-
-| Panel                      | Type       | PromQL                                                   |
-| -------------------------- | ---------- | -------------------------------------------------------- |
-| Agreement % (1h)           | stat       | `validation_agreement{metric="agreement_pct_1h"}`        |
-| Agreement % (24h)          | stat       | `validation_agreement{metric="agreement_pct_24h"}`       |
-| Agreements vs Missed (1h)  | bargauge   | `agreements_1h` and `missed_1h` side by side             |
-| Agreements vs Missed (24h) | bargauge   | `agreements_24h` and `missed_24h` side by side           |
-| Validation Rate            | stat       | `rate(validations_sent_total[5m]) * 60`                  |
-| Validations Checked Rate   | stat       | `rate(validations_checked_total[5m]) * 60`               |
-| Amendment Blocked          | stat       | `validator_health{metric="amendment_blocked"}`           |
-| UNL Expiry (days)          | stat       | `validator_health{metric="unl_expiry_days"}`             |
-| Validation Quorum          | stat       | `validator_health{metric="validation_quorum"}`           |
-| State Value Timeline       | timeseries | `state_tracking{metric="state_value"}`                   |
-| Time in Current State      | stat       | `state_tracking{metric="time_in_current_state_seconds"}` |
-| State Changes Rate         | stat       | `rate(state_changes_total[1h])`                          |
-| Ledgers Closed Rate        | stat       | `rate(ledgers_closed_total[5m]) * 60`                    |
-
-**Dashboard conventions**: `$node` template variable for `service_instance_id` filtering, dark theme, matching existing panel sizes and color schemes.
-
-**Key new files**: `docker/telemetry/grafana/dashboards/validator-health.json`
-(uid `validator-health`). The name reached its current form in **two** renames:
-`dashboards/rippled-validator-health.json` → `xrpld-validator-health.json`
-(`145b1469d6`, the `dashboards/rippled-*` → `xrpld-*` pass), then
-`xrpld-validator-health.json` → `validator-health.json` (`25868f2740`, which
-dropped the `xrpld-` prefix).
-
-**Exit Criteria**:
-
-- [x] Dashboard ships **17** panels (4 more than the 13 planned above) across 3
-      rows — Validation Agreement, Validation Rates, Server State & Consensus
-- [ ] All panels render with non-zero data during normal operation — needs a live
-      stack; the Phase 10 harness asserts the dashboard _loads_, not that panels
-      are non-empty
-- [x] `$node` filter works correctly for multi-node deployments — `node`
-      template variable present (filters on `service_instance_id`), alongside
-      `service_name`, `deployment_environment`, `xrpl_network_type`,
-      `xrpl_work_item`, `xrpl_branch`, `xrpl_node_role`
-- [x] Amendment blocked and UNL expiry panels use color thresholds
-      (red=blocked/expiring) — 11 `thresholds` blocks in the dashboard JSON
-
----
-
-## Task 9.12: Peer Quality Dashboard (External Dashboard Parity)
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
-
-**Objective**: Create a Grafana dashboard for peer health aggregates.
-
-**Dashboard**: `peer-quality.json`
-
-| Panel                  | Type       | PromQL                                                                  |
-| ---------------------- | ---------- | ----------------------------------------------------------------------- |
-| P90 Peer Latency       | timeseries | `peer_quality{metric="peer_latency_p90_ms"}`                            |
-| Insane/Diverged Peers  | stat       | `peer_quality{metric="peers_insane_count"}`                             |
-| Higher Version Peers % | stat       | `peer_quality{metric="peers_higher_version_pct"}`                       |
-| Upgrade Recommended    | stat       | `peer_quality{metric="upgrade_recommended"}`                            |
-| Resource Disconnects   | timeseries | `server_info{metric="peer_disconnects_resources"}`                      |
-| Inbound vs Outbound    | bargauge   | `peer_finder_active_inbound_peers`, `peer_finder_active_outbound_peers` |
-
-> `overlay_peer_disconnects_charges` (the name in the original spec) is **not a
-> real instrument** — nothing registers it. The shipped panel reads
-> `server_info{metric="peer_disconnects_resources"}` instead. Peer-finder gauge
-> names are lowercase: `GroupImp::makeName()` + `OTelCollectorImp::formatName()`
-> turn the `"Peer_Finder"` group into `peer_finder_<name>` with no prefix.
-
-**Key new files**: `docker/telemetry/grafana/dashboards/peer-quality.json`
-(uid `peer-quality`). Two renames, same as Task 9.11:
-`dashboards/rippled-peer-quality.json` → `xrpld-peer-quality.json`
-(`145b1469d6`), then `xrpld-peer-quality.json` → `peer-quality.json`
-(`25868f2740`).
-
-**Exit Criteria**:
-
-- [x] All 6 panels present — P90 Peer Latency, Insane/Diverged Peers, Higher
-      Version Peers %, Upgrade Recommended, Inbound vs Outbound Peers, Resource
-      Disconnects — across 3 rows, with the `$node` template variable
-- [ ] All 6 panels render with data — needs a live stack
-- [x] P90 latency panel is a `timeseries` (shows trend over time)
-- [x] Upgrade recommended panel uses color threshold (red=1, green=0) — 5
-      `thresholds` blocks in the dashboard JSON
-
----
-
-## Task 9.13: Ledger Economy Dashboard Panels (External Dashboard Parity)
-
-> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
-
-**Objective**: Add "Ledger Economy" row to the existing `node-health.json` dashboard.
-
-| Panel                | Type       | PromQL                                        |
-| -------------------- | ---------- | --------------------------------------------- |
-| Base Fee (drops)     | stat       | `ledger_economy{metric="base_fee_drops"}`     |
-| Reserve Base (drops) | stat       | `ledger_economy{metric="reserve_base_drops"}` |
-| Reserve Inc (drops)  | stat       | `ledger_economy{metric="reserve_inc_drops"}`  |
-| Ledger Age           | stat       | `ledger_economy{metric="ledger_age_seconds"}` |
-| Transaction Rate     | timeseries | `ledger_economy{metric="transaction_rate"}`   |
-
-**Key modified files**: `docker/telemetry/grafana/dashboards/node-health.json`
-
-**Exit Criteria**:
-
-- [x] 5 new panels present in the existing dashboard — a "Ledger Economy" row
-      with 5 `ledger_economy` queries is on `node-health.json`
-- [ ] Fee values match `server_info` RPC output — needs a live comparison
-- [ ] Transaction rate shows smooth trend (not spiky) — needs a live run
-
----
-
-## Task 9.14: Overlay Traffic Accounting Defects (Documentation Only)
-
-> **Status**: Defect 1 FIXED in `develop`. Defects 2-4 DOCUMENTED, NOT FIXED. Reference: [09 §6.0-§6.2](./09-data-collection-reference.md#6-known-issues)
-
-**Objective**: Record four pre-existing overlay traffic-accounting defects so
-dashboard readers are not misled. All four originate in `develop`-owned overlay
-files, so **no code fix lands on this branch**.
-
-| #   | Defect                                 | Effect                                                                                            | Fix location (NOT this branch)                   |
-| --- | -------------------------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------ |
-| 1   | `mtCLUSTER` missing from `kTypeLookup` | **Fixed in `develop`.** Cluster messages from members count as cluster; others count as `unknown` | `TrafficCount.cpp`: `kTypeLookup`, `attribute()` |
-| 2   | Stale `Total` header comment           | Claims uncategorized traffic is excluded; it is included                                          | `TrafficCount.h:28-31`                           |
-| 3   | `SquelchIgnored` reported with size 0  | `squelch_ignored_bytes_*` always zero, inconsistent with `SquelchSuppressed`                      | `OverlayImpl.cpp:1460,1489` (+ signature change) |
-| 4   | In/out byte-basis asymmetry            | `_bytes_in` vs `_bytes_out` not comparable under compression                                      | `PeerImp.cpp:1079` vs `:313`                     |
-
-**Why deferred**: Defect 3 requires widening the two
-`OverlayImpl::updateSlotAndSquelch` overloads — a public signature change on
-shared overlay code. Defect 4 needs `PeerImp.cpp`
-edits that are not telemetry-owned. Routing them through the telemetry chain
-would hide overlay changes from overlay reviewers and couple them to a 12-PR
-merge timeline.
-
-> **Constraint narrowed.** The blanket "no telemetry change may touch
-> `TrafficCount.{h,cpp}`" no longer holds for the header: the telemetry chain
-> already edits `TrafficCount.h` — Phase 6's `77f35c03db` fixed the
-> `Category::GetFetchPack` label from `"getobject_Fetch Pack_get"` to
-> `"getobject_Fetch_Pack_get"` in its category-name table, the sole difference from
-> `develop`. Defect 2 (the stale `Total` header comment, `TrafficCount.h:28-31`)
-> is therefore **unblocked** and can land here. Defect 1 is fixed in `develop`.
-> Defects **3 and 4** stay blocked: defect 3 needs the
-> `OverlayImpl` signature change, and defect 4 needs `PeerImp.cpp:1079` vs `:313`
-> to agree on a byte basis (compressed vs uncompressed) — a change to overlay
-> accounting semantics, not telemetry.
-
-**Key modified files**: `OpenTelemetryPlan/09-data-collection-reference.md` only.
-
-**Exit Criteria**:
-
-- [x] Each defect documented with file:line evidence in `09` §6
-- [x] Defect 1 fixed in `develop`'s `TrafficCount` code; `09` §6.0 says what `overhead_cluster_*` counts
-- [ ] Defect 2 (stale `Total` header comment, `TrafficCount.h:28-31`) fixed on
-      this branch — it is **unblocked** (the chain already edits
-      `TrafficCount.h`) but the comment is still uncorrected
-- [ ] Follow-up overlay-owned branch raised for the two still-blocked code
-      fixes (defects 3, 4)
-- [x] No re-baseline needed for defect 1: no alert rule in this repo reads `unknown_*`
-
----
-
-## Task 9.15: Peer Keepalive and Discovery Instrumentation
-
-> **Status**: NOT IMPLEMENTED. The instruments themselves are still to be
-> written; the _permission_ question is settled. Reference:
-> [09 §6.3](./09-data-collection-reference.md#63-peer-keepalive-and-discovery-traffic-gaps-not-implemented)
->
-> **Blocker cleared.** This task used to be held "awaiting a decision on whether
-> `XRPL_METRIC_*` call sites may be added to
-> `src/xrpld/overlay/detail/PeerImp.cpp` from this branch". That decision is
-> de facto **yes** — `PeerImp.cpp` already carries **7** such call sites on this
-> branch (`:2723`, `:2741`, `:2925`, `:2928`, `:2931`, `:2947`, `:2954`, of which
-> three are `XRPL_METRIC_HISTOGRAM_RECORD` — `:2925`, `:2928`, `:2931` — and four
-> are labelled counters — `:2723`, `:2741`, `:2947`, `:2954`). Note that
-> `grep -c XRPL_METRIC src/xrpld/overlay/detail/PeerImp.cpp` returns 8: the eighth
-> hit is the `cspell:ignore` explanation comment at `PeerImp.cpp:2`, not a call
-> site. What remains is the implementation work below, not an approval.
-
-**Objective**: Make peer keepalive and peer-discovery health observable. Today
-`mtPING`, `mtSTATUS_CHANGE` and `mtENDPOINTS` are byte counters only.
-
-| Proposed metric                 | Type      | Labels                           | Record site                                         |
-| ------------------------------- | --------- | -------------------------------- | --------------------------------------------------- |
-| `peer_ping_rtt_ms`              | Histogram | none (see note)                  | `PeerImp.cpp:1150-1163`, where the EWMA is computed |
-| `peer_ping_timeouts_total`      | Counter   | `reason="timeout"\|"bad_cookie"` | `PeerImp.cpp:762` and `:1146`                       |
-| `peer_endpoints_received_total` | Counter   | `result="accepted"\|"malformed"` | `PeerImp.cpp:1265-1270`                             |
-
-**Design notes / open questions**:
-
-- A histogram needs an explicit bucket view: the SDK default tops out at 10000,
-  and these are milliseconds. Follow the µs-ladder precedent in
-  `MetricsRegistry.cpp` (see [09 § GetObject Request Path](./09-data-collection-reference.md#getobject-request-path-synchronous-countershistograms)).
-- `peer_id` as a label is unbounded cardinality — rejected. A bounded
-  `peer_role`-style label is the alternative if per-peer attribution is needed.
-- Splitting `mtPING` out of `Category::Base` changes `TrafficCount.cpp`. It needs its own overlay change, like the `develop` change that fixed Task 9.14 defect 1. (The `.h` half of that constraint no longer applies — see Task 9.14.)
-- Per the runbook's "Adding a New Metric" contract, `_total` is reserved for
-  monotonic counters; a histogram takes no suffix.
-
-**Key files (if approved)**: `src/xrpld/overlay/detail/PeerImp.cpp`,
-`09-data-collection-reference.md`, `docs/telemetry-runbook.md` § Metric Reference,
-`docker/telemetry/grafana/dashboards/peer-quality.json`, and
-`docker/telemetry/workload/expected_metrics.json` (**Phase 10 branch**).
-
-**Exit Criteria**:
-
-- [x] Decision recorded on editing `PeerImp.cpp` from the telemetry chain — yes;
-      7 `XRPL_METRIC_*` call sites already ship in `PeerImp.cpp`
-- [ ] Three instruments emitting, with an explicit histogram bucket view
-- [ ] Rows added to `09` §5b, runbook § Metric Reference, and `expected_metrics.json`
-- [ ] Peer Quality dashboard panels follow the Task 9.12 conventions (`$node`, Title Case, legend dimensions)
-- [ ] `check_otel_naming.py` passes (Rules D and E cover the new labels)
-
----
-
-## Task 9.16: PeerFinder Slot and Cache Metrics
-
-> **Status**: NOT IMPLEMENTED. Reference: [09 §6.5](./09-data-collection-reference.md#65-peerfinder-slot-and-cache-metrics-not-implemented)
-
-**Objective**: Export the PeerFinder slot counts and discovery-cache sizes.
-Only 2 of ~17 available readings are exported today.
-
-**What to do**: Extend the existing `Stats` struct in
-`src/libxrpl/peerfinder/PeerfinderManager.cpp:227-236` with gauges for the
-`Counts` accessors listed in [09 §6.5](./09-data-collection-reference.md#65-peerfinder-slot-and-cache-metrics-not-implemented)
-(slot caps and frees, attempt counts, handshake pipeline depth, fixed-peer state,
-network reachability), plus `Livecache::size()` and `Bootcache::size()`.
-
-**Pipeline constraint**: `PeerfinderManager.cpp` is in `libxrpl`, which **cannot**
-use the `XRPL_METRIC_*` macros. These must go through `beast::insight` —
-arrow **B**, not **C**. Naming follows `GroupImp::makeName()` +
-`OTelCollectorImp::formatName()`, so the `"Peer_Finder"` group yields
-`peer_finder_<name>` lowercased.
-
-**Known obstacle**: `Livecache` and `Bootcache` hold no collector reference, so
-their sizes must either be read through the existing `Manager` hook or have a
-collector plumbed in.
-
-**Exit Criteria**:
-
-- [ ] Slot caps exported so utilization (`active / max`) is computable
-- [ ] Both cache sizes exported
-- [ ] "Inbound vs Outbound" panel on `peer-quality` extended to show utilization %
-- [ ] Rows added to `09` §2.1, runbook § Metric Reference, `expected_metrics.json` (Phase 10)
-
----
-
-## Task 9.17: Peer Span Coverage (Deferred to Phase 11)
-
-> **Status**: NOT IMPLEMENTED — design only, pending approval. Reference:
-> [09 §6.4](./09-data-collection-reference.md#64-peer-span-coverage-gap-not-implemented)
-> and [02 §2.3.2](./02-design-decisions.md#232-complete-span-catalog)
-
-**Objective**: Close the gap between the `02` §2.3.2 span catalog and what
-actually emits. `peer.connect`, `peer.disconnect`, `peer.message.send` and
-`peer.message.receive` were catalogued from the start and never built; 11 of 13
-protocol message families have no spans.
-
-**Scope warning**: This is larger than Tasks 9.14-9.16 combined and changes the
-span-family inventory in `09` §1.1 and the entries in
-`docker/telemetry/workload/expected_spans.json`. `trace_peer` is also
-**on by default** and already flagged as high-volume, so adding per-message
-spans has a volume cost that needs measuring before commitment.
-
-**Exit Criteria**:
-
-- [x] `02` §2.3.2 marked Live / Not built / Renamed against the real inventory
-- [ ] User approval to proceed with span implementation
-- [ ] Volume impact measured under `trace_peer=1` before any span is added
-
----
-
-## Exit Criteria
-
-- [ ] All ~50 new metrics visible in Prometheus via OTLP pipeline — every
-      instrument is registered in `MetricsRegistry.cpp`, but end-to-end
-      visibility is asserted only by the Phase 10 harness
-- [x] `MetricsRegistry` class registers/deregisters cleanly with OTel SDK —
-      `src/tests/libxrpl/telemetry/MetricsRegistry.cpp`
-      (`async_gauges_start_after_start_is_safe`,
-      `async_gauges_before_start_does_not_break_start`,
-      `async_gauges_respect_the_compile_time_guard`, `destructor_calls_stop`)
-- [x] Async gauge callbacks execute at 10s intervals —
-      `MetricsRegistry::initExporterAndProvider`, `readerOpts.export_interval_millis = 10000`.
-      (The "without performance impact" half is unmeasured — see below.)
-- [x] 4 new Grafana dashboards operational (Fee Market, Job Queue, Validator
-      Health, Peer Quality) — all four JSONs are under
-      `docker/telemetry/grafana/dashboards/`
-- [x] 2 existing dashboards updated with new panel groups — `node-health`
-      (NodeStore I/O, Caches, Server Info, Complete Ledgers & DB, Ledger
-      Economy, Job Queue Concurrency Limits rows) and `rpc-performance`
-      (per-method section)
-- [ ] Integration test validates all new metric families are non-zero — not on
-      this branch; lives in the Phase 10 harness (`expected_metrics.json`)
-- [ ] No performance regression (< 0.5% CPU overhead from new callbacks) — not
-      measured; needs the Phase 10 benchmark suite
-- [x] Documentation updated with full new metric inventory —
-      `09-data-collection-reference.md` §5b + "Phase 9: OTel SDK-Exported
-      Metrics (MetricsRegistry)" + "Phase 7+: External Dashboard Parity Metrics"
-- [x] Validator Health dashboard ships (17 panels, 4 more than the 13 planned)
-- [x] Peer Quality dashboard ships (6 panels)
-- [x] Ledger Economy panels added to node-health dashboard (5 panels in a
-      "Ledger Economy" row)
-- [x] Provisioned Grafana alerting: 14 rules / 5 groups, 2 contact points,
-      nested notification policy
-- [ ] Tasks 9.14-9.17 closed — **open by design**: 9.14 partly fixed
-      (defect 1 fixed in `develop`; defects 3 and 4 still blocked; defect 2 unblocked but not yet fixed),
-      9.15 and 9.16 not implemented, 9.17 deferred pending approval and volume
-      measurement
-
----
-
-## Appendix: Alerting Design
-
-> Design for the provisioned Grafana alert rules (Task 9.9a). Previously a standalone spec; merged here so the phase plan is self-contained.
-
-**Date:** 2026-07-06
-**Branch:** `pratik/otel-phase9-metric-gap-fill` (PR #6513)
-**Status:** Approved
-
-### Purpose
-
-Phase 9 exports ~68 internal xrpld metrics and ships Grafana dashboards for
-them. This adds the missing operator-facing piece: **provisioned Grafana alert
-rules** that fire on the health-critical metrics phase 9 introduces. The
-phase-9 task list already lists "alerting rules" as a phase-9 deliverable
-(Task 9.9), so this closes that gap.
-
-Scope is deliberately narrow — the three subsystems whose failure is
-node-fatal: **consensus/ledger health, validator health, job queue**. RPC/API
-health is explicitly out of scope.
-
-### Why phase 9 (not phase 11)
-
-Every metric these alerts fire on is _born_ in phase 9
-(`ledger_hash_mismatch_total`, `ledgers_closed_total`,
-`validation_missed_total`, `validations_checked_total`,
-`jq_trans_overflow_total`, `job_queued_us_bucket` — the histogram instrument is
-`job_queued_us` (`kJobQueuedDurationUs` in `MetricsRegistry.cpp`), so the
-Prometheus bucket series is `job_queued_us_bucket`, not
-`job_queued_duration_us_bucket`). Alerts
-belong with the metrics they watch, and this is where the dependency lives.
-
-### Delivery
-
-Provisioned YAML, version-controlled — matching the existing datasource /
-dashboard provisioning pattern. No docker-compose change: the Grafana service
-already mounts `./grafana/provisioning:/etc/grafana/provisioning:ro`, and
-Grafana auto-loads `provisioning/alerting/*.yaml`.
-
-New files under `docker/telemetry/grafana/provisioning/alerting/`:
-
-| File                 | Purpose                                                                                                                                                                                                                      |
-| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `contactpoints.yaml` | **Two** contact points: `xrpld-default` (Slack) and `xrpld-critical` (Slack + email).                                                                                                                                        |
-| `policies.yaml`      | **Nested** notification policy: root route → `xrpld-default`; child route matching `severity = critical` → `xrpld-critical` (`repeat_interval: 1h` vs the root's `4h`). Both grouped by `alertname` + `service_instance_id`. |
-| `rules.yaml`         | **14** alert rules across **5** groups (below).                                                                                                                                                                              |
-
-Plus the Alerting section of `docs/telemetry-runbook.md` — operator runbook:
-what each alert means, likely causes, and how to point the contact point at a
-real receiver.
-
-### Alert rules
-
-All rules target Prometheus datasource `uid: prometheus`. Each rule has three
-nodes with named refIds: a Prometheus query, a `reduce` that keeps the query's
-last value, and a `threshold` on that value. The threshold node is the rule's
-`condition`. In `LedgerHistoryMismatch`, for example, the three are
-`mismatches_15m`, `mismatches_now` and `is_mismatching`. All
-`rate()`/`histogram_quantile()` expressions aggregate with
-`sum by (service_instance_id)` (or `+ le`) so **each node alerts independently**.
-Alert rules run headless, so they cannot use the dashboards' `$node` template
-variables — they match all series and group by `service_instance_id` instead.
-
-All 5 groups evaluate at `interval: 1m`. Metric names carry **no** `xrpld_`
-prefix — `OTelCollectorImp::formatName()` adds none.
-
-The **Threshold** column is the `threshold` node's evaluator, read straight from
-`rules.yaml` — it is the firing condition, so it is load-bearing, not decoration.
-
-| Group              | Alert                    | Expression (query node)                                                                                                      | Threshold (evaluator)                             | `for` | severity |
-| ------------------ | ------------------------ | ---------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------- | ----- | -------- |
-| `xrpld-consensus`  | LedgerHistoryMismatch    | `sum by (service_instance_id) (increase(ledger_hash_mismatch_total[15m]))`                                                   | `gt [0]`                                          | 2m    | critical |
-| `xrpld-consensus`  | LedgerCloseStalled       | `rate(ledgers_closed_total)` decayed to ≈0                                                                                   | `lt [0.001]`                                      | 3m    | critical |
-| `xrpld-consensus`  | ValidatedLedgerStale     | `max by (service_instance_id) (ledgermaster_validated_ledger_age < 1209600)`                                                 | `gt [60]` (seconds)                               | 5m    | critical |
-| `xrpld-validator`  | ValidationsMissed        | miss **ratio**, nodes up more than 1 h — see the expression below the table                                                  | `gt [0.1]`                                        | 15m   | warning  |
-| `xrpld-validator`  | ValidationsNotChecked    | `rate(validations_checked_total)` ≈0                                                                                         | `lt [0.001]`                                      | 5m    | warning  |
-| `xrpld-jobqueue`   | JobQueueTxOverflow       | `sum by (service_instance_id) (increase(jq_trans_overflow_total[15m]))`                                                      | `gt [0]`                                          | 2m    | warning  |
-| `xrpld-jobqueue`   | JobQueueLatencyHigh      | `histogram_quantile(0.99, sum by (le, service_instance_id) (rate(job_queued_us_bucket[5m])))`                                | `gt [1000000]` (µs = 1s)                          | 5m    | warning  |
-| `xrpld-jobqueue`   | IOEventLoopLatencyHigh   | `histogram_quantile(0.95, sum by (le, service_instance_id) (rate(ios_latency_milliseconds_bucket[10m])))`                    | `gt [1000]` (ms)                                  | 10m   | warning  |
-| `xrpld-node-state` | NodeStateFlapping        | state-transition rate over the node-state series                                                                             | `gt [0]` (transitions)                            | 15m   | warning  |
-| `xrpld-node-state` | NodeNotFull              | operating mode below FULL                                                                                                    | `lt [4]` (FULL = 4)                               | 15m   | warning  |
-| `xrpld-overlay`    | ManifestJobQueueConvoy   | `sum by (service_instance_id) (jobq_manifest_waiting)`                                                                       | `gt [3]` (waiting jobs)                           | 10m   | warning  |
-| `xrpld-overlay`    | ManifestFloodInbound     | inbound manifest byte rate                                                                                                   | `gt [524288]` (B/s = 512 **KiB**/s, not 512 kB/s) | 10m   | warning  |
-| `xrpld-overlay`    | PeerResourceDisconnects  | `sum by (service_instance_id) (increase(server_info{metric="peer_disconnects_resources"}[30m]))`                             | `gt [5]`                                          | 5m    | warning  |
-| `xrpld-overlay`    | UntrustedValidationFlood | untrusted-validation rate over 2m divided by the rate of the hour ending 10m earlier (floor 50/s), nodes up more than 4200 s | `gt [3]` (ratio)                                  | 1m    | warning  |
-
-**`ValidationsMissed` is a ratio, not `rate(...) > 0`.** A few late or missed
-validations do not page; the rule fires only when more than 10% of a 15-minute
-window's ledgers were missed. A node without a validator key publishes neither
-counter, so it has no series and the condition never fires for it. The shipped
-expression is (each selector also carries `service_name="xrpld"`):
-
-- numerator: `sum by (service_instance_id) (rate(validation_missed_total[15m]))`
-- denominator: `clamp_min(` that same numerator `+ sum by (service_instance_id) (rate(validation_agreements_total[15m])), 1e-9)`
-- gate: `and on (service_instance_id) (max by (service_instance_id) (server_info{metric="uptime"}) > 3600)` — skip the first hour after a start, as `NodeStateFlapping` and `NodeNotFull` do
-- evaluator: `gt [0.1]` — more than 10% of the window's ledgers missed
-
-3 rules are `severity: critical`, 11 are `severity: warning`.
-
-Each rule carries labels `severity` and `category`, and annotations `summary`, `description` and `action`. The three flood rules (`ManifestJobQueueConvoy`, `ManifestFloodInbound`, `UntrustedValidationFlood`) also carry the label `page_oncall: "true"`. `summary` and `description` interpolate `{{ $labels.service_instance_id }}`, and `description` reads the reduce node's value as `$values.<reduce refId>.Value` (for example `$values.mismatches_now.Value`).
-
-#### Threshold rationale
-
-- **LedgerCloseStalled `< 0.001` for 3m**: healthy nodes close a ledger every
-  ~3-5s; a 5m rate decaying to ~0 means the node is stuck. The epsilon (not
-  exact `0`) avoids float rate-noise suppressing the alert.
-- **JobQueueLatencyHigh 1s p99**: `gt [1000000]` µs = 1s. A default starting
-  point, easy to tune — jobs queued >1s at p99 indicate the node is saturated.
-- **ValidationsMissed `> 0.1` on a ratio**, not `> 0` on a raw rate: a few late
-  or missed validations should not page. See the note above the rationale list.
-- **ManifestFloodInbound 524288 B/s**: an earlier 50 kB/s threshold produced ~41
-  sustained 5-minute samples on healthy nodes; 512 KiB/s clears normal
-  manifest-exchange peaks.
-- **NodeStateFlapping `> 0`**: one `full → syncing → full` round is one re-entry into `full`, and that one round is the flap to catch.
-- Remaining `gt [0]` rules (`LedgerHistoryMismatch`, `JobQueueTxOverflow`) sit on
-  true error counters where any sustained nonzero rate is actionable.
-
-### Non-goals / YAGNI
-
-- No per-alert silencing schedules, no mute timings.
-- No RPC/API or fee-market alerts (dashboards cover those visually). Overlay
-  alerts _were_ added during implementation — the `xrpld-overlay` group carries
-  four (manifest convoy, manifest flood, peer resource disconnects, untrusted
-  validation flood).
-- Two contact points and a two-level policy tree shipped; deeper routing
-  (Discord, PagerDuty, per-team splits) is left to the operator.
-
-### Verification
-
-1. `yamllint` (or `python -c yaml.safe_load`) on all three YAML files.
-2. `docker compose -f docker/telemetry/docker-compose.yml config -q` still parses.
-3. Optional live check: start stack, `GET /api/v1/provisioning/alert-rules`
-   returns the 14 rules; Grafana logs show no provisioning errors.
-4. Code-review pass (subagent) against phase conventions before commit.

cfg/xrpld-example.cfg

@@ -1267,10 +1267,10 @@
 #
 #     "server"
 #
-#       Choice of server to send metrics to, either "statsd" or "otel".
-#
-#       "statsd" sends UDP packets to a StatsD daemon, which must be running
-#       while xrpld is running. More information on StatsD is available here:
+#       Choice of server to send metrics to. Currently the only choice is
+#       "statsd" which sends UDP packets to a StatsD daemon, which must be
+#       running while xrpld is running. More information on StatsD is
+#       available here:
 #           https://github.com/b/statsd_spec
 #
 #       When server=statsd, these additional keys are used:
@@ -1281,13 +1281,6 @@
 #       "prefix"  A string prepended to each collected metric. This is used
 #                 to distinguish between different running instances of xrpld.
 #
-#       "otel" exports the same metrics over OTLP instead, and needs a build
-#       with telemetry compiled in. The [telemetry] section below owns the
-#       export destination and the resource attributes, so "address" and
-#       "prefix" do not apply. An "endpoint" key is read on this path but
-#       only appears in a startup log line; it does not change where metrics
-#       are sent.
-#
 #     If this section is missing, or the server type is unspecified or unknown,
 #     statistics are not collected or reported.
 #
@@ -1690,213 +1683,3 @@ validators.txt
 # set to ssl_verify to 0.
 [ssl_verify]
 1
-#-------------------------------------------------------------------------------
-#
-# 11. Telemetry (OpenTelemetry Tracing)
-#
-#-------------------------------------------------------------------------------
-#
-# Enables distributed tracing via OpenTelemetry. Takes effect only if tracing
-# was compiled in with the Conan option `-o telemetry=True`; there is no CMake
-# option for it. See docs/build/telemetry.md. Build it out with
-# `-o telemetry=False`, which reduces all tracing code to no-ops. The Conan
-# default is currently True so that CI compiles the telemetry code paths;
-# False is the intended default once this feature is merged, so pass the value
-# you want rather than relying on the default.
-#
-# Note that -DXRPL_ENABLE_TELEMETRY=OFF does NOT work: XRPL_ENABLE_TELEMETRY is
-# a compile definition added by the build, not a CMake option, so it disables
-# nothing. CMake only lists it at the end of configuration under
-# "Manually-specified variables were not used by the project".
-#
-# [telemetry]
-#
-# enabled=0
-#
-#   Enable or disable telemetry at runtime. Default: 0 (disabled).
-#   This key, use_tls and the trace_* keys take 0, 1, true or false, in
-#   any case. Any other value is a startup error that names the key.
-#
-# service_name=xrpld
-#
-#   OTel resource attribute `service.name`. Default: xrpld.
-#   The node's network ID (from [network_id]) is automatically added
-#   as the `xrpl.network.id` and `xrpl.network.type` resource attributes.
-#
-# service_instance_id=<node_public_key>
-#
-#   OTel resource attribute `service.instance.id`, which tells one node's
-#   telemetry apart from another's. Normally left unset: the node identity
-#   is not known when telemetry is constructed, so the server fills this in
-#   with its own Base58 node public key later during startup. Set it only
-#   to pin a stable instance name of your own choosing; doing so suppresses
-#   the node-public-key fallback.
-#   Default: the node's Base58 public key.
-#
-#   SET THIS EXPLICITLY IF YOU USE THE METRICS PIPELINE. The node-public-key
-#   fallback only reaches traces: the metrics resource is built during
-#   startup, before the node key is known, and cannot be changed afterwards.
-#   With this key unset, metrics export with an empty service.instance.id and
-#   the per-node filter on the Grafana dashboards has nothing to split on.
-#
-# traces_endpoint=http://localhost:4318/v1/traces
-#
-#   The OTLP/HTTP endpoint spans are exported to. The server sends trace
-#   data as protobuf-encoded HTTP POST requests to this URL. The full URL
-#   including the signal path is used verbatim; no other endpoint is
-#   derived from it.
-#   Default: http://localhost:4318/v1/traces.
-#
-#   beast::insight metrics ([insight] server=otel) do not follow this
-#   setting; they use metrics_endpoint below.
-#
-#   The scheme of this URL is what decides whether the connection is
-#   encrypted, and it is matched exactly: only a lower-case https:// URL
-#   gives TLS. Because of that, with enabled=1 and use_tls=1 this URL must
-#   start with https://. Any other URL, including the default above, makes
-#   xrpld stop at startup with a message that names traces_endpoint.
-#
-# --- TLS settings for the OTLP exporter connection ---
-#
-# use_tls=0
-#
-#   Whether to hand tls_ca_cert to the exporter as its CA bundle. TLS is
-#   picked by the scheme of each endpoint URL, not by this key. So with
-#   enabled=1, use_tls=1 requires both traces_endpoint and metrics_endpoint
-#   to start with https:// (lower case). Any other URL, including the
-#   defaults, makes xrpld stop at startup with a message that names the
-#   key.
-#   Default: 0 (no CA file is passed, so the exporter keeps its own
-#   default trust store).
-#
-# tls_ca_cert=
-#
-#   Path to a PEM-encoded CA certificate bundle for verifying the
-#   collector's certificate. Only used when use_tls=1, and passed to the
-#   exporter unchanged.
-#   Default: empty (system CA store).
-#
-#   Leaving this empty stays valid and selects the system CA store. A path
-#   that is set is checked like the client paths below: with enabled=1 and
-#   use_tls=1, one that does not exist or cannot be read makes xrpld fail
-#   to start.
-#
-# tls_client_cert=
-#
-#   Path to this node's PEM-encoded client certificate, presented to the
-#   collector for mutual TLS (mTLS). Requires use_tls=1. Leave empty
-#   for one-way (server-only) TLS. Default: empty.
-#
-#   To enable mTLS, both tls_client_cert and tls_client_key must be
-#   specified. If only one is provided, xrpld will fail to start. Providing
-#   them while use_tls=0 also fails to start, rather than being ignored.
-#   traces_endpoint and metrics_endpoint must both be https:// URLs, because
-#   that scheme is what makes each exporter present the certificate at all.
-#   With use_tls=1 each path is opened at startup, so one that does not
-#   exist or cannot be read fails to start too, rather than failing later
-#   as an opaque TLS handshake error. All four checks apply only when
-#   enabled=1; with telemetry disabled these settings are read but never
-#   validated.
-#
-# tls_client_key=
-#
-#   Path to the PEM-encoded private key for tls_client_cert. Required
-#   whenever tls_client_cert is set. Requires use_tls=1, and must be
-#   readable. All three conditions are enforced exactly as described under
-#   tls_client_cert above: when enabled=1, breaking any one of them makes
-#   xrpld fail to start.
-#   Default: empty.
-#
-#   Head sampling is intentionally fixed at 1.0 (sample everything) and is
-#   not configurable. A per-node sampling ratio would let nodes make
-#   divergent keep/drop decisions for the same distributed trace, producing
-#   broken/partial traces. A span with a remote parent is decided by the same
-#   fixed ratio, not by the peer's sampled flag. Reduce volume at the collector
-#   via tail sampling instead; for node-local post-hoc dropping use
-#   SpanGuard::discard() in code.
-#
-# trace_rpc=1
-#
-#   Enable tracing for JSON-RPC and WebSocket API request handling —
-#   command parsing, execution, and response serialization. Default: 1.
-#
-# trace_transactions=1
-#
-#   Enable tracing for the transaction lifecycle — submission, validation,
-#   application to ledgers, and final disposition. Default: 1.
-#
-# trace_consensus=1
-#
-#   Enable tracing for the consensus round lifecycle — proposals,
-#   validations, mode changes, and ledger acceptance. Default: 1.
-#
-# trace_peer=1
-#
-#   Enable tracing for peer-to-peer protocol messages — overlay message
-#   send/receive, peer handshakes, and routing. High volume; enabled
-#   by default. Default: 1.
-#
-# trace_ledger=1
-#
-#   Enable tracing for ledger close and accept operations — ledger
-#   building, state hashing, and write-back to the node store. Default: 1.
-#
-# consensus_trace_strategy=deterministic
-#
-#   How the consensus round span picks its trace id. Two values are
-#   accepted, and anything else makes xrpld fail to start.
-#
-#   deterministic (the default, and the value to use): the trace id comes
-#   from the previous ledger hash, so every validator of a round reports
-#   into one trace and the round can be read end to end across nodes.
-#
-#   random: experimental only, and not used. Each node invents its own
-#   trace id, so a single round arrives as one separate trace per node.
-#   Those traces can only be lined up by hand through the
-#   consensus_ledger_id span attribute.
-#
-# --- Batch processor tuning ---
-#
-# batch_size=512
-#
-#   Maximum number of spans exported in a single batch. Must be at least 1
-#   and must not exceed max_queue_size. Default: 512.
-#
-# batch_delay_ms=5000
-#
-#   Maximum delay (milliseconds) before a partial batch is flushed.
-#   Must be at least 1. Default: 5000 (5 seconds).
-#
-# max_queue_size=2048
-#
-#   Maximum number of spans queued in memory before drops occur. Must be
-#   at least 1 and at least as large as batch_size. Default: 2048.
-#
-# --- Metric export ---
-#
-# metrics_endpoint=http://localhost:4318/v1/metrics
-#
-#   The OTLP/HTTP endpoint metrics are sent to, including the signal path.
-#   Used verbatim and independent of the trace endpoint, so the two signals
-#   can go to different collectors, or to one whose OTLP paths are not the
-#   defaults.
-#   Default: http://localhost:4318/v1/metrics.
-#
-#   The scheme decides encryption here exactly as it does for
-#   traces_endpoint, so with enabled=1 and use_tls=1 this URL must also
-#   start with https://. Any other URL, including the default above, makes
-#   xrpld stop at startup with a message that names metrics_endpoint.
-#
-# metric_export_interval_ms=1000
-#
-#   Gap in milliseconds between two metric exports. Must be greater than 0.
-#   Default: 1000 (1 second), which matches the scrape rate the shipped
-#   dashboards assume.
-#
-# metric_export_timeout_ms=500
-#
-#   How long in milliseconds one metric export may run before it is
-#   cancelled. Must be greater than 0 and below metric_export_interval_ms;
-#   xrpld fails to start otherwise.
-#   Default: 500 (half a second).
-#

cmake/XrplCore.cmake

@@ -80,19 +80,6 @@ include(target_link_modules)
 # Level 01
 add_module(xrpl beast)
 target_link_libraries(xrpl.libxrpl.beast PUBLIC xrpl.imports.main)
-# OTelCollector in beast/insight uses the OTel Metrics SDK when telemetry is
-# enabled. Link the Conan-provided umbrella target rather than individual
-# component targets: the OTel package's per-component dependency graph is
-# under-declared (e.g. the OTLP client references sdk::common symbols without
-# declaring the edge), so naming components directly reorders the static-link
-# line into an unresolvable state. The umbrella carries the full, internally
-# consistent graph the package authors validated.
-if(telemetry)
-    target_link_libraries(
-        xrpl.libxrpl.beast
-        PUBLIC opentelemetry-cpp::opentelemetry-cpp
-    )
-endif()
 
 include(XrplVersion)
 add_module(xrpl git)
@@ -224,51 +211,9 @@ target_link_libraries(
         xrpl.libxrpl.conditions
 )
 
-# Telemetry module — OpenTelemetry distributed tracing support.
-# Sources: include/xrpl/telemetry/ (headers), src/libxrpl/telemetry/ (impl).
-# When telemetry=ON, links the Conan-provided umbrella target
-# opentelemetry-cpp::opentelemetry-cpp (individual component targets like
-# ::api, ::sdk are not available in the Conan package).
-#
-# Declared before its consumers (consensus, tx) because add_module isolates
-# each module's headers: a module can only include xrpl/telemetry/ headers if
-# it links this target, and the target must already exist at that point.
-#
-# Links xrpl.libxrpl.protocol and xrpl.libxrpl.core PUBLICLY: ValidationTracker.h
-# takes LedgerIndex and MetricMacros.h takes ServiceRegistry, both in interfaces.
-# TxAccountSpanNames.cpp also reads the SField table from xrpl.libxrpl.protocol.
-add_module(xrpl telemetry)
-target_link_libraries(
-    xrpl.libxrpl.telemetry
-    PUBLIC
-        xrpl.libxrpl.basics
-        xrpl.libxrpl.beast
-        xrpl.libxrpl.config
-        xrpl.libxrpl.core
-        xrpl.libxrpl.protocol
-)
-if(telemetry)
-    # Telemetry owns both the trace and (as of the direct-metrics API) the
-    # metrics pipeline. Link the umbrella target: it supplies the trace and
-    # metrics SDK components with the correct static-link ordering, which
-    # naming components individually does not (the package under-declares
-    # inter-component dependencies).
-    target_link_libraries(
-        xrpl.libxrpl.telemetry
-        PUBLIC opentelemetry-cpp::opentelemetry-cpp
-    )
-    # PUBLIC, so a parent project that adds this one with add_subdirectory()
-    # and links this module sees the same class layouts it was built with.
-    # CMakeLists.txt also sets the define for every target in this project.
-    # Conan consumers get it from conanfile.py instead.
-    target_compile_definitions(
-        xrpl.libxrpl.telemetry
-        PUBLIC XRPL_ENABLE_TELEMETRY
-    )
-endif()
+add_module(xrpl tx)
+target_link_libraries(xrpl.libxrpl.tx PUBLIC xrpl.libxrpl.ledger)
 
-# Links xrpl.libxrpl.telemetry for the consensus tracing spans declared in
-# include/xrpl/consensus/ConsensusSpanNames.h.
 add_module(xrpl consensus)
 target_link_libraries(
     xrpl.libxrpl.consensus
@@ -277,13 +222,6 @@ target_link_libraries(
         xrpl.libxrpl.json
         xrpl.libxrpl.protocol
         xrpl.libxrpl.ledger
-        xrpl.libxrpl.telemetry
-)
-
-add_module(xrpl tx)
-target_link_libraries(
-    xrpl.libxrpl.tx
-    PUBLIC xrpl.libxrpl.ledger xrpl.libxrpl.telemetry
 )
 
 add_library(xrpl.libxrpl)
@@ -320,7 +258,6 @@ target_link_modules(
     resource
     server
     shamap
-    telemetry
     tx
 )
 

cmake/XrplDocs.cmake

@@ -27,12 +27,8 @@ file(
     src/*.cpp
     src/*.md
     Builds/*.md
+    *.md
 )
-# Add only top-level .md files (README, CONTRIBUTING, etc.) without
-# recursing into dot-directories like .claude/ whose files are not
-# valid Doxygen/CMake sources.
-file(GLOB doxygen_top_md CONFIGURE_DEPENDS "*.md")
-list(APPEND doxygen_input ${doxygen_top_md})
 list(APPEND doxygen_input external/README.md)
 set(dependencies "${doxygen_input}" "${doxyfile}")
 

cmake/xrpl_find_otel_archive.cmake

@@ -1,32 +0,0 @@
-# Find an OpenTelemetry archive, for a test binary to link by its full path.
-# Set the variable named by out to that path, and cache it under that name.
-#
-# A test links an archive this way when the package's targets do not put it
-# on the link line where the test needs it. For example, the Conan package
-# declares the in-memory exporter component with no libraries, so no target
-# links that archive. Link the result before the umbrella target, so that its
-# references into the SDK resolve against the umbrella's archives.
-#
-# CMakeDeps adds the upper-cased build type to its variable names, so the
-# hint's suffix is derived here, not hardcoded as _RELEASE. An empty hint alone
-# is not fatal: the Conan toolchain also puts the package's lib directory on
-# CMAKE_LIBRARY_PATH, which find_library searches.
-#
-# Because the result is cached, every caller gets the same file. A missing
-# archive stops the configure step instead of failing at link time. The check
-# is explicit because find_library's REQUIRED needs CMake 3.18, and this
-# project allows 3.16.
-#
-# xrpl_find_otel_archive(out name)
-function(xrpl_find_otel_archive out name)
-    string(TOUPPER "${CMAKE_BUILD_TYPE}" build_type)
-    find_library(
-        ${out}
-        NAMES ${name}
-        HINTS "${opentelemetry-cpp_LIB_DIRS_${build_type}}"
-    )
-    if(NOT ${out})
-        message(FATAL_ERROR "${name} not found")
-    endif()
-    set(${out} "${${out}}" PARENT_SCOPE)
-endfunction()

conan.lock

@@ -11,14 +11,11 @@
         "rocksdb/10.5.1#4a197eca381a3e5ae8adf8cffa5aacd0%1782392413.075713",
         "re2/20251105#8579cfd0bda4daf0683f9e3898f964b4%1782392402.431897",
         "protobuf/6.33.5#ff253ead763bd8d9904a52979cd21e81%1782392410.233933",
-        "opentelemetry-cpp/1.28.0#2cbf71db4e0e0535df20be305005cb2f%1785939947.292581",
         "openssl/3.6.3#f806de8933e3bf6f01016c6a888cee2e%1783945160.863288",
         "nudb/2.0.9#11149c73f8f2baff9a0198fe25971fc7%1782392402.297166",
-        "nlohmann_json/3.11.3#45828be26eb619a2e04ca517bb7b828d%1701220705.259",
         "mpt-crypto/1.0.2#b313cef0c1a493eb970ad185b2e9bab7%1784285108.866483",
         "lz4/1.10.0#982d9b673900f665a1da109e09c17cab%1782392402.164188",
         "libiconv/1.17#9923bc6dc6f106646d6967e0039a5ada%1782392792.775744",
-        "libcurl/8.21.0#8c26e59c04891ba3373ea3552e18f67f%1783067699.863",
         "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1782392402.420732",
         "libarchive/3.8.7#c446109bd1f1d8ba7936c94189bc50e6%1782392403.066892",
         "jemalloc/5.3.1#1fc58d55316041f10fbc1e8a2eae632a%1776700028.228",
@@ -38,15 +35,9 @@
         "zlib/1.3.2#1cb806da49011867778ffb6ac7190fcb%1782392402.122708",
         "strawberryperl/5.32.1.1#8d114504d172cfea8ea1662d09b6333e%1782395692.540639",
         "protobuf/6.33.5#ff253ead763bd8d9904a52979cd21e81%1782392410.233933",
-        "pkgconf/2.5.1#93c2051284cba1279494a43a4fcfeae2%1757684701.089",
-        "opentelemetry-proto/1.7.0#ed6d5bd761bef0afb0ba09676420b9ea%1749461220.268",
-        "ninja/1.13.2#c8c5dc2a52ed6e4e42a66d75b4717ceb%1764096931.974",
         "nasm/2.16.01#31e26f2ee3c4346ecd347911bd126904%1782395690.33162",
         "msys2/cci.latest#d22fe7b2808f5fd34d0a7923ace9c54f%1770657326.649",
-        "meson/1.10.2#9d2d10681fe7fe61c788c58626c89b25%1775558003.754",
         "m4/1.4.19#1727f439cf74e83826ec96d0b4904eee%1784541921.659",
-        "libtool/2.4.7#14e7739cc128bc1623d2ed318008e47e%1755679003.847",
-        "gnu-config/cci.20210814#466e9d4d7779e1c142443f7ea44b4284%1762363589.329",
         "cmake/4.3.3#840cf00ea09777e05c2050a50a82c722%1782392418.696091",
         "b2/5.4.2#ffd6084a119587e70f11cd45d1a386e2%1782392402.624226",
         "automake/1.16.5#b91b7c384c3deaa9d535be02da14d04f%1755524470.56",
@@ -76,9 +67,6 @@
         ],
         "lz4/[>=1.9.4 <2]": [
             "lz4/1.10.0#982d9b673900f665a1da109e09c17cab"
-        ],
-        "protobuf/[>=4.25.3 <7]": [
-            "protobuf/6.33.5#ff253ead763bd8d9904a52979cd21e81"
         ]
     },
     "config_requires": []

conan/profiles/default

@@ -1,5 +1,3 @@
-{# Read SANITIZERS first: the next line rebinds `os` from the module to the OS name. #}
-{% set sanitizers = os.getenv("SANITIZERS", "") %}
 {% set os = detect_api.detect_os() %}
 {% set arch = detect_api.detect_arch() %}
 {% set compiler, version, compiler_exe = detect_api.detect_default_compiler() %}
@@ -51,17 +49,6 @@ tools.build:compiler_executables={'c':'{{ cc_exe }}','cpp':'{{ cxx_exe }}'}
 user.package:cppstd_version=23
 tools.info.package_id:confs+=["user.package:cppstd_version"]
 
-{# opentelemetry-cpp is an external dependency: sanitizer reports from its own #}
-{# code are not ours to fix, so build it without sanitizer checks. #}
-{# TSan stays on, because it must see the library's atomics or it reports false races. #}
-{% if sanitizers and compiler != "msvc" %}
-{% set otel_sanitizer_flags = ["-fno-sanitize=all"] %}
-{% if "thread" in sanitizers %}
-{% set _ = otel_sanitizer_flags.append("-fsanitize=thread") %}
-{% endif %}
-opentelemetry-cpp/*:tools.build:cxxflags+={{ otel_sanitizer_flags }}
-{% endif %}
-
 {% if os == "Macos" %}
 [buildenv]
 {# os.version adds -mmacosx-version-min to compiler command lines, #}
@@ -71,15 +58,3 @@ opentelemetry-cpp/*:tools.build:cxxflags+={{ otel_sanitizer_flags }}
 {# Scoped to boost/* since it is the only gap. #}
 boost/*:MACOSX_DEPLOYMENT_TARGET={{ min_macos_version }}
 {% endif %}
-{% if os == "Windows" %}
-# opentelemetry-cpp's recipe removes the `shared` option on Windows and never
-# sets BUILD_SHARED_LIBS, so its upstream CMake defaults the protobuf-generated
-# `opentelemetry_proto` target to a DLL (opentelemetry_proto.dll). The rest of
-# the project links statically and nothing deploys that DLL next to the
-# executables, so the telemetry unit test fails to start with
-# STATUS_DLL_NOT_FOUND (0xC0000135). Force the dependency to build fully static
-# so no runtime DLL is produced. The conf is folded into the package id so a
-# fresh static binary is built instead of reusing a previously cached one.
-opentelemetry-cpp/*:tools.cmake.cmaketoolchain:extra_variables={"BUILD_SHARED_LIBS": "OFF"}
-opentelemetry-cpp/*:tools.info.package_id:confs+=["tools.cmake.cmaketoolchain:extra_variables"]
-{% endif %}

conanfile.py

@@ -26,7 +26,6 @@ class Xrpl(ConanFile):
         "rocksdb": [True, False],
         "shared": [True, False],
         "static": [True, False],
-        "telemetry": [True, False],
         "tests": [True, False],
         "unity": [True, False],
         "xrpld": [True, False],
@@ -62,7 +61,6 @@ class Xrpl(ConanFile):
         "rocksdb": True,
         "shared": False,
         "static": True,
-        "telemetry": True,
         "tests": False,
         "unity": False,
         "xrpld": False,
@@ -148,10 +146,6 @@ class Xrpl(ConanFile):
             self.requires("rocksdb/10.5.1")
         self.requires("secp256k1/0.7.1", transitive_headers=True)
         self.requires("sqlite3/3.53.0", force=True)
-        # OpenTelemetry C++ SDK for distributed tracing (optional).
-        # Provides OTLP/HTTP exporter, batch span processor, and trace API.
-        if self.options.telemetry:
-            self.requires("opentelemetry-cpp/1.28.0", transitive_headers=True)
         self.requires("xxhash/0.8.3", transitive_headers=True)
 
     exports_sources = (
@@ -193,7 +187,6 @@ class Xrpl(ConanFile):
         tc.variables["rocksdb"] = self.options.rocksdb
         tc.variables["BUILD_SHARED_LIBS"] = self.options.shared
         tc.variables["static"] = self.options.static
-        tc.variables["telemetry"] = self.options.telemetry
         tc.variables["unity"] = self.options.unity
         tc.variables["xrpld"] = self.options.xrpld
         tc.generate()
@@ -248,9 +241,3 @@ class Xrpl(ConanFile):
         ]
         if self.options.rocksdb:
             libxrpl.requires.append("rocksdb::librocksdb")
-        if self.options.telemetry:
-            libxrpl.requires.append("opentelemetry-cpp::opentelemetry-cpp")
-            # The public telemetry headers pick their class layout on this
-            # define, so a consumer that does not see it compiles a different
-            # SpanGuard than the one inside the library it links.
-            libxrpl.defines.append("XRPL_ENABLE_TELEMETRY")

docker/telemetry/.env.alerting.example

@@ -1,32 +0,0 @@
-# rippled OTel alerting delivery — copy to `.env.alerting` and fill in.
-#
-# `.env.alerting` is gitignored; never commit a real Slack webhook or address.
-# See the Alerting section of docs/telemetry-runbook.md.
-#
-# IMPORTANT — these variables do NOT feed contactpoints.yaml.
-# That file carries literal placeholder values which you edit in place, because
-# Grafana expands ${VAR} but does NOT support ${VAR:-default}: an unset variable
-# expands to empty, fails provisioning validation, and Grafana EXITS 1 — taking
-# the whole telemetry stack down, not just alerting. So a var left blank here
-# does not "disable a delivery path"; referencing a blank one breaks startup.
-#
-# What these are actually for:
-#   GF_SMTP_*       consumed by the Grafana container (compose `env_file`) to
-#                   turn on mail delivery. Without these, an email contact point
-#                   provisions fine and then silently sends nothing.
-
-# --- Slack (local stack: paste the webhook into contactpoints.yaml instead) ---
-# Incoming-webhook URL from the Slack app (Incoming Webhooks feature).
-# Kept here as a convenient place to record it, NOT as a ${VAR} source.
-SLACK_WEBHOOK_URL=
-
-# --- Email ---
-# SMTP relay Grafana sends through. Email only delivers when SMTP is enabled
-# and these point at a real relay (e.g. an internal smart host or an
-# authenticated provider).
-GF_SMTP_ENABLED=false
-GF_SMTP_HOST=smtp.example.com:587
-GF_SMTP_USER=
-GF_SMTP_PASSWORD=
-GF_SMTP_FROM_ADDRESS=alerts@xrpld.local
-GF_SMTP_FROM_NAME=xrpld Alerts

docker/telemetry/.env.grafanacloud-alloy.example

@@ -1,35 +0,0 @@
-# Grafana Alloy -> Grafana Cloud credentials for docker/telemetry/alloy/config.alloy.
-# Copy to `.env.grafanacloud-alloy`, fill in, and source before starting Alloy.
-# `.env.grafanacloud-alloy` is gitignored; never commit real tokens or stack ids.
-#
-# Prometheus values: Grafana Cloud -> Connections -> "Prometheus".
-# OTLP values:       Grafana Cloud -> Connections -> "OpenTelemetry (OTLP)".
-
-# --- Host / systemd metrics (prometheus.remote_write) ---
-# remote_write push URL, e.g. https://prometheus-prod-XX-<region>.grafana.net/api/prom/push
-GRAFANACLOUD_PROM_URL=
-# Numeric instance/stack id (Basic-auth username).
-GRAFANACLOUD_PROM_USER=
-# Access-policy token with metrics:write (Basic-auth password).
-GRAFANACLOUD_PROM_KEY=
-
-# --- xrpld OTLP (otelcol.exporter.otlphttp) ---
-# OTLP/HTTP gateway URL including the /otlp path,
-# e.g. https://otlp-gateway-prod-<region>.grafana.net/otlp
-GRAFANACLOUD_OTLP_URL=
-# Numeric instance/stack id (Basic-auth username).
-GRAFANACLOUD_OTLP_USER=
-# Access-policy token with metrics:write + traces:write (Basic-auth password).
-GRAFANACLOUD_OTLP_KEY=
-
-# --- Per-node label ---
-# host label applied to this node's scraped metrics (e.g. the node's hostname).
-XRPLD_HOST_LABEL=
-
-# --- Not settable here ---
-# deployment.environment and xrpl.network.type are NOT env vars. They are
-# literals inside `otelcol.processor.transform "tier"` in alloy/config.alloy,
-# because OTTL statements do not expand sys.env(). Edit them there to match this
-# node, or every dashboard's environment and network filters will read prod and
-# mainnet. Dashboards filter on service_instance_id, which passes through from
-# the node's own [telemetry] config and needs nothing here.

docker/telemetry/.env.grafanacloud.example

@@ -1,16 +0,0 @@
-# Grafana Cloud OTLP credentials — copy to `.env.grafanacloud` and fill in.
-# Find all three under Grafana Cloud -> Connections -> "OpenTelemetry (OTLP)".
-#
-# `.env.grafanacloud` is gitignored; never commit real tokens.
-
-# OTLP/HTTP gateway URL for your stack, including the /otlp path.
-# Example: https://otlp-gateway-prod-<zone>.grafana.net/otlp
-GRAFANA_CLOUD_OTLP_ENDPOINT=
-
-# Numeric instance/stack id shown on the OTLP connection page
-# (used as the Basic-auth username).
-GRAFANA_CLOUD_INSTANCE_ID=
-
-# A Cloud Access Policy token with metrics:write, traces:write, logs:write
-# (used as the Basic-auth password).
-GRAFANA_CLOUD_API_TOKEN=

docker/telemetry/.gitignore

@@ -1,13 +0,0 @@
-# Runtime data generated by xrpld and telemetry stack
-data/
-
-# Env. file carrying environmental setup data for local or cloud runs.
-.env.*
-
-# Keep examples
-!.env.alerting.example
-!.env.grafanacloud.example
-!.env.grafanacloud-alloy.example
-
-# Do not commit grafana cloud versions with default filters
-grafanacloud/*.json

docker/telemetry/alloy/config.alloy

@@ -1,420 +0,0 @@
-// Grafana Alloy collector config for an xrpld node.
-//
-// WHY THIS FILE EXISTS
-// --------------------
-// Some deployments feed telemetry through Grafana Alloy instead of the
-// reference OpenTelemetry Collector (otel-collector-config.grafanacloud.yaml).
-// xrpld sends OTLP (traces + native beast::insight metrics) to Alloy, which
-// forwards to the Grafana Cloud OTLP gateway. That path carries traces and
-// native metrics, but on its own produces NO span-derived RED metrics
-// (span_calls_total / span_duration_milliseconds_*): those are NOT emitted by
-// xrpld -- they are derived by a spanmetrics connector from the spans. Without
-// the connector a node is missing from every span-based Grafana dashboard.
-//
-// This config therefore mirrors the two stages the reference collector has:
-//   1. resource tagging  (service.name, deployment.environment, xrpl.network.type)
-//   2. spanmetrics       (derives the RED metrics the dashboards query)
-// See docker/telemetry/otel-collector-config.grafanacloud.yaml for the
-// authoritative collector equivalent; keep the dimension list in sync with it.
-//
-// Logs are tailed here too, so all three signals leave a node through one
-// exporter with one resource identity. The reference collector reads the log
-// file with otelcol's own filelog receiver; Alloy's equivalent
-// (otelcol.receiver.filelog) is still public-preview and refuses to load
-// without --stability.level=public-preview on the alloy service, so the Loki
-// source components are used instead and bridged into OTLP by
-// otelcol.receiver.loki. Every component below is generally-available, so the
-// service needs no extra flag.
-//
-// PIPELINE
-//
-//   HOST / SYSTEMD METRICS:
-//     unix exporter + custom scrape --> relabel(host) --> prometheus.remote_write
-//
-//   xrpld OTLP:
-//     receiver.otlp ─▶ processor.transform.tier ─┬─▶ connector.spanmetrics ─┐
-//                                                 │                          │
-//                                                 └─▶ processor.batch ◀───────┘
-//                                                          │ traces + metrics
-//                                                          ▼
-//                                                 exporter.otlphttp (GC OTLP gateway)
-//
-//   xrpld LOGS (debug.log):
-//     file_match ─▶ loki.source.file ─▶ loki.process ─▶ receiver.loki
-//                                       (parse+label)         │ OTLP logs
-//                                                             ▼
-//                                        processor.transform.tier ─▶ processor.batch
-//                                                                          │
-//                                                                          ▼
-//                                                 exporter.otlphttp (GC OTLP gateway)
-//
-// The Grafana Cloud OTLP gateway converts OTLP resource attributes to
-// Prometheus labels server-side, so no otelcol.exporter.prometheus is needed.
-//
-// CONFIGURATION -- NO SECRETS OR TENANT IDS ARE HARD-CODED IN THIS FILE.
-// All credentials, endpoints, usernames and the per-node host label are read
-// from the environment (same pattern as .env.grafanacloud.example). Copy
-// .env.grafanacloud-alloy.example to .env.grafanacloud-alloy, fill it in, and
-// source it before starting Alloy. Required variables:
-//   GRAFANACLOUD_PROM_URL   Prometheus remote_write push URL
-//   GRAFANACLOUD_PROM_USER   remote_write basic-auth username (numeric stack id)
-//   GRAFANACLOUD_PROM_KEY    remote_write basic-auth password (access token)
-//   GRAFANACLOUD_OTLP_URL    OTLP/HTTP gateway URL, including the /otlp path
-//   GRAFANACLOUD_OTLP_USER   OTLP basic-auth username (numeric stack id)
-//   GRAFANACLOUD_OTLP_KEY    OTLP basic-auth password (access token)
-//   XRPLD_HOST_LABEL         host label for this node's scraped metrics, and
-//                            the service.instance.id stamped on its logs. It
-//                            must equal the node's [telemetry]
-//                            service_instance_id, or logs carry a node name
-//                            that no trace or metric shares and nothing joins.
-// Optional:
-//   XRPLD_LOG_GLOB           debug.log path to tail. Defaults to
-//                            /space/xrpld/log/debug.log, the devnet layout.
-//
-// PER-DEPLOYMENT EDITS: the deployment.environment and xrpl.network.type tier
-// values in otelcol.processor.transform are literals (OTTL cannot read env
-// vars) -- edit them to match this node's tier and network. The log path is the
-// exception: it is built in River, where sys.env does expand, and concatenated
-// into the OTTL statement.
-
-logging {
-  level = "info"
-}
-
-// ===========================================================================
-// HOST / SYSTEMD METRICS  (Prometheus remote_write path)
-// ===========================================================================
-
-prometheus.remote_write "grafanacloud" {
-  endpoint {
-    url = sys.env("GRAFANACLOUD_PROM_URL")
-
-    basic_auth {
-      username = sys.env("GRAFANACLOUD_PROM_USER")
-      password = sys.env("GRAFANACLOUD_PROM_KEY")
-    }
-  }
-}
-
-prometheus.exporter.unix "default" {
-  enable_collectors = ["systemd", "textfile"]
-  systemd {
-    unit_include = "(rippled|xrpld).*"
-  }
-  textfile {
-    directory = "/usr/local/bin/"
-  }
-}
-
-prometheus.scrape "system_metrics" {
-  targets         = prometheus.exporter.unix.default.targets
-  scrape_interval = "15s"
-  forward_to      = [prometheus.relabel.hostname.receiver]
-}
-
-prometheus.scrape "custom_rippled_metrics" {
-  targets = [{
-    __address__ = "localhost:9256",
-  }]
-  scrape_interval = "15s"
-  forward_to      = [prometheus.relabel.hostname.receiver]
-}
-
-prometheus.relabel "hostname" {
-  forward_to = [prometheus.remote_write.grafanacloud.receiver]
-
-  rule {
-    action        = "replace"
-    source_labels = ["instance"]
-    replacement   = sys.env("XRPLD_HOST_LABEL")
-    target_label  = "host"
-  }
-}
-
-// ===========================================================================
-// xrpld OTLP -> Grafana Cloud  (traces + native metrics + derived spanmetrics)
-// ===========================================================================
-
-// Receiver: xrpld sends OTLP/HTTP on 4318 and gRPC on 4317 (localhost only).
-otelcol.receiver.otlp "xrpld" {
-  grpc {
-    endpoint = "127.0.0.1:4317"
-  }
-  http {
-    endpoint = "127.0.0.1:4318"
-  }
-
-  output {
-    // Both signals pass through resource tagging first so they leave with
-    // identical resource identity.
-    traces  = [otelcol.processor.transform.tier.input]
-    metrics = [otelcol.processor.transform.tier.input]
-  }
-}
-
-// Resource tagging (reference: resource/tier + resource/stripsdk).
-//   * service.name -> "xrpld". Also corrects the service_name="true"
-//     bool-binding bug on stale binaries: even an un-rebuilt node exports a
-//     correct service.name once it passes through here.
-//   * deployment.environment -> set from XRPLD_DEPLOYMENT_ENV (the collector
-//     IS the environment, so it is authoritative -> upsert).
-//   * xrpl.network.type -> set only when absent (don't overwrite the node's
-//     own value). OTTL `where ... == nil` gives insert (not upsert) semantics.
-//   * telemetry.sdk.* -> deleted (SDK noise).
-// ===========================================================================
-// xrpld LOGS  (debug.log --> OTLP gateway)
-// ===========================================================================
-
-// Tail the node's debug.log. devnet writes a single flat file
-// (/space/xrpld/log/debug.log) rather than the per-node subdirectory the docker
-// stack uses, so identity cannot be read off the path here -- it comes from
-// XRPLD_HOST_LABEL below. Override XRPLD_LOG_GLOB for a different layout.
-local.file_match "xrpld_logs" {
-  path_targets = [{
-    __path__ = coalesce(sys.env("XRPLD_LOG_GLOB"), "/space/xrpld/log/debug.log"),
-  }]
-}
-
-loki.source.file "xrpld_logs" {
-  targets    = local.file_match.xrpld_logs.targets
-  forward_to = [loki.process.xrpld_logs.receiver]
-}
-
-// Parse the line Logs::format() emits:
-//   YYYY-Mmm-DD HH:MM:SS.fffffffff UTC <partition>:<severity> [trace_id=... span_id=...] <message>
-// The `partition:` prefix is omitted when the partition is empty, so that group
-// is optional, and trace_id/span_id are present only for a sampled span.
-loki.process "xrpld_logs" {
-  forward_to = [otelcol.receiver.loki.xrpld.receiver]
-
-  stage.regex {
-    expression = "^(?P<log_time>\\S+\\s+\\S+)\\s+\\S+\\s+(?:(?P<partition>\\S+):)?(?P<severity>\\S+)\\s+(?:trace_id=(?P<trace_id>[a-f0-9]+)\\s+span_id=(?P<span_id>[a-f0-9]+)\\s+)?"
-  }
-
-  // Use the node's own timestamp, not ingest time, or a log line cannot be
-  // lined up with the span it belongs to. The node emits nanosecond precision;
-  // the fractional part of this layout accepts any number of digits.
-  stage.timestamp {
-    source   = "log_time"
-    format   = "2006-Jan-02 15:04:05.999999999"
-    location = "UTC"
-  }
-
-  // A regex capture stays in the pipeline's extracted map and never reaches the
-  // entry unless a stage attaches it. Structured metadata keeps these queryable
-  // without making any of them an indexed label.
-  stage.structured_metadata {
-    values = {
-      partition = "",
-      severity  = "",
-      trace_id  = "",
-      span_id   = "",
-    }
-  }
-}
-
-// Bridge the Loki entries into OTLP so logs use the same exporter, and get the
-// same resource identity, as traces and metrics. Note this receiver produces an
-// EMPTY resource and puts everything on the log record, so the resource
-// attributes are set in processor.transform below.
-otelcol.receiver.loki "xrpld" {
-  output {
-    logs = [otelcol.processor.transform.tier.input]
-  }
-}
-
-otelcol.processor.transform "tier" {
-  error_mode = "ignore"
-
-  // NOTE: statements are OTTL (raw strings) -- River sys.env() does NOT expand
-  // inside them, and OTTL has no env() converter. The tier values below are
-  // plain deployment labels (not secrets), so they are literals here.
-  // PER-DEPLOYMENT EDIT: set deployment.environment and xrpl.network.type to
-  // match this node's tier and network.
-  trace_statements {
-    context = "resource"
-    statements = [
-      `set(attributes["service.name"], "xrpld")`,
-      `set(attributes["deployment.environment"], "prod")`,
-      `set(attributes["xrpl.network.type"], "mainnet") where attributes["xrpl.network.type"] == nil`,
-      `delete_key(attributes, "telemetry.sdk.language")`,
-      `delete_key(attributes, "telemetry.sdk.name")`,
-      `delete_key(attributes, "telemetry.sdk.version")`,
-    ]
-  }
-
-  metric_statements {
-    context = "resource"
-    statements = [
-      `set(attributes["service.name"], "xrpld")`,
-      `set(attributes["deployment.environment"], "prod")`,
-      `set(attributes["xrpl.network.type"], "mainnet") where attributes["xrpl.network.type"] == nil`,
-      `delete_key(attributes, "telemetry.sdk.language")`,
-      `delete_key(attributes, "telemetry.sdk.name")`,
-      `delete_key(attributes, "telemetry.sdk.version")`,
-    ]
-  }
-
-  // Logs use `log` context, not `resource`, for two reasons: receiver.loki
-  // hands over an empty resource, and the fields to promote (severity, trace
-  // ids) live on the record. Each Alloy instance serves one node, so writing a
-  // resource attribute from a record is unambiguous here.
-  //
-  // service.instance.id is concatenated in from XRPLD_HOST_LABEL because OTTL
-  // has no env() converter. Loki promotes only an allow-listed set of RESOURCE
-  // attributes to indexed stream labels, and service.instance.id is on that
-  // list, so it arrives as the label service_instance_id that the dashboards
-  // filter on. Setting it as a record attribute instead would make it
-  // structured metadata, which cannot be used in a {...} stream selector.
-  log_statements {
-    context = "log"
-    statements = [
-      "set(resource.attributes[\"service.instance.id\"], \"" + coalesce(sys.env("XRPLD_HOST_LABEL"), "unknown") + "\")",
-      `set(resource.attributes["service.name"], "xrpld")`,
-      `set(resource.attributes["deployment.environment"], "prod")`,
-      `set(resource.attributes["xrpl.network.type"], "mainnet") where resource.attributes["xrpl.network.type"] == nil`,
-      // Promote the parsed fields onto the first-class OTLP record fields, so
-      // Grafana links a log line to its trace natively instead of re-parsing
-      // the body. The guards matter: most lines are emitted outside a sampled
-      // span and must keep an empty trace id rather than an invalid one.
-      `set(severity_text, attributes["severity"]) where attributes["severity"] != nil`,
-      `set(trace_id.string, attributes["trace_id"]) where attributes["trace_id"] != nil`,
-      `set(span_id.string, attributes["span_id"]) where attributes["span_id"] != nil`,
-      // Drop the bridge's own bookkeeping so it does not become structured
-      // metadata on every line.
-      `delete_key(attributes, "filename")`,
-      `delete_key(attributes, "loki.attribute.labels")`,
-      `delete_key(attributes, "log.file.path")`,
-      `delete_key(attributes, "log.file.name")`,
-    ]
-  }
-
-  output {
-    // Traces fan out: to the batch/gateway path AND into the spanmetrics
-    // connector so the RED metrics are derived from the same tagged spans.
-    traces = [
-      otelcol.processor.batch.xrpld.input,
-      otelcol.connector.spanmetrics.xrpld.input,
-    ]
-    // Native metrics go straight to the batch/gateway path.
-    metrics = [otelcol.processor.batch.xrpld.input]
-    logs    = [otelcol.processor.batch.xrpld.input]
-  }
-}
-
-// Spanmetrics connector (reference: connectors.spanmetrics, namespace "span").
-// Derives span_calls_total and span_duration_milliseconds_* that every span
-// dashboard queries. The dimension list and buckets are copied from the
-// reference otel-collector-config.grafanacloud.yaml -- keep them in sync (a
-// panel that groups by a dimension missing here renders empty).
-otelcol.connector.spanmetrics "xrpld" {
-  namespace = "span"
-
-  // Flush every 15s, matching otel-collector-config.grafanacloud.yaml. Alloy
-  // defaults to 60s, which is longer than [$__rate_interval] at short dashboard
-  // ranges, so a rate() panel gets under two samples per window and renders
-  // "No data". Costs ~4x span-metric datapoints.
-  metrics_flush_interval = "15s"
-
-  histogram {
-    unit = "ms"
-    explicit {
-      // Buckets MUST stay strictly ascending (the connector binary-searches
-      // them and silently misbuckets otherwise) and MUST match the two
-      // otel-collector-config*.yaml lists. Sub-millisecond boundaries exist
-      // because most xrpld spans sit far below 1ms: a 1ms floor puts nearly
-      // every sample in bucket one and pins p95/p99 to that bucket's edge.
-      // Sub-second boundaries cover RPC, tx and ledger spans. The 2s-4s edges
-      // resolve second-scale consensus spans, and 10s/30s give the
-      // ledger.acquire catch-up tail a home so its p95/p99 stop reading +Inf.
-      // Each edge was chosen from a measured duration distribution for the
-      // spans it covers.
-      buckets = ["0.01ms", "0.05ms", "0.1ms", "0.25ms", "0.5ms", "1ms", "5ms", "10ms", "25ms", "50ms", "100ms", "250ms", "500ms", "1s", "2s", "3s", "4s", "5s", "10s", "30s"]
-    }
-  }
-
-  // RPC / transaction dimensions.
-  dimension { name = "command" }
-  dimension { name = "rpc_status" }
-  dimension { name = "tx_type" }
-  dimension { name = "ter_result" }
-  dimension { name = "stage" }
-  dimension { name = "txq_status" }
-  dimension { name = "load_type" }
-  dimension { name = "is_batch" }
-
-  // Consensus dimensions.
-  dimension { name = "consensus_mode" }
-  dimension { name = "close_time_correct" }
-  dimension { name = "consensus_state" }
-  dimension { name = "mode_new" }
-  dimension { name = "consensus_stalled" }
-  dimension { name = "consensus_phase" }
-  dimension { name = "consensus_result" }
-
-  // Overlay / peer dimensions.
-  dimension { name = "local" }
-  dimension { name = "proposal_trusted" }
-  dimension { name = "validation_trusted" }
-
-  // gRPC surface dimensions.
-  dimension { name = "method" }
-  dimension { name = "grpc_role" }
-  dimension { name = "grpc_status" }
-
-  // ledger.acquire dimensions.
-  dimension { name = "outcome" }
-  dimension { name = "acquire_reason" }
-
-  // Sync-diagnostic span dimensions. `timed_out` separates "slow but arriving"
-  // from "the retry budget ran out" on the acquire phase spans; `object_type`
-  // splits ledger.serve by what was asked for. Both are small closed sets.
-  dimension { name = "timed_out" }
-  dimension { name = "object_type" }
-
-  // Trace-causality dimensions on consensus.validation.accept. The ledger_hash
-  // join key stays span-only -- one metric series per ledger would be unbounded
-  // -- and is indexed in Tempo instead.
-  dimension { name = "validation_status" }
-  dimension { name = "accept_gated" }
-
-  output {
-    // Derived span metrics rejoin the metric stream at the batch processor.
-    metrics = [otelcol.processor.batch.xrpld.input]
-  }
-}
-
-// Batch traces + metrics (native and span-derived) before export.
-otelcol.processor.batch "xrpld" {
-  timeout         = "1s"
-  send_batch_size = 1024
-
-  output {
-    traces  = [otelcol.exporter.otlphttp.grafanacloud.input]
-    metrics = [otelcol.exporter.otlphttp.grafanacloud.input]
-    logs    = [otelcol.exporter.otlphttp.grafanacloud.input]
-  }
-}
-
-// Grafana Cloud OTLP gateway auth + exporter. The gateway converts OTLP
-// resource attributes to Prometheus labels server-side.
-otelcol.auth.basic "grafanacloud" {
-  username = sys.env("GRAFANACLOUD_OTLP_USER")
-  password = sys.env("GRAFANACLOUD_OTLP_KEY")
-}
-
-otelcol.exporter.otlphttp "grafanacloud" {
-  client {
-    endpoint = sys.env("GRAFANACLOUD_OTLP_URL")
-    auth     = otelcol.auth.basic.grafanacloud.handler
-  }
-  retry_on_failure {
-    enabled          = true
-    max_elapsed_time = "5m"
-  }
-  sending_queue {
-    enabled       = true
-    num_consumers = 4
-    queue_size    = 1000
-  }
-}

docker/telemetry/docker-compose.grafanacloud.yaml

@@ -1,33 +0,0 @@
-# Compose override — enable Grafana Cloud export at runtime.
-#
-# Layer this on top of the base stack to swap the collector onto the
-# Grafana Cloud dual-export config and inject the OTLP credentials. The
-# base docker-compose.yml is unchanged, so plain `up` stays local-only.
-#
-# Usage:
-#   1. Put your three Grafana Cloud values in docker/telemetry/.env.grafanacloud
-#      (template: .env.grafanacloud.example). That file is gitignored.
-#   2. Bring the stack up with BOTH compose files:
-#        docker compose -f docker/telemetry/docker-compose.yml \
-#                       -f docker/telemetry/docker-compose.grafanacloud.yaml up -d
-#   3. To go back to local-only, bring the stack up with just the base file.
-#
-# Log mount: inherited from the base compose (./data/logs -> /var/log/xrpld).
-# Both nodes' xrpld cfgs write to docker/telemetry/data/logs/<net>/debug.log,
-# so no per-box remap is needed here.
-
-services:
-  otel-collector:
-    volumes:
-      # Mount the Grafana Cloud collector config over the default path
-      # (replaces local-only config for this run; file on disk untouched).
-      - ./otel-collector-config.grafanacloud.yaml:/etc/otel-collector-config.yaml:ro
-      # Mount the cloud variant of the file_log offset overlay over the base
-      # one. The collector replaces service.extensions on merge, and the base
-      # variant omits basicauth/grafanacloud, which stops the cloud exporters
-      # starting.
-      - ./otel-collector-filestorage.grafanacloud.yaml:/etc/otel-collector-filestorage.yaml:ro
-    # Secrets from the env file, injected as container env and
-    # resolved by the collector via ${env:...}.
-    env_file:
-      - .env.grafanacloud

docker/telemetry/docker-compose.workload.yaml

@@ -1,123 +0,0 @@
-# Docker Compose workload harness for telemetry validation.
-#
-# Runs the OTel telemetry backend only. There are no validator services here:
-#   - OTel Collector (traces + native OTLP metrics)
-#   - Tempo (trace backend + search API)
-#   - Prometheus (metrics)
-#   - Loki (log aggregation for log-trace correlation)
-#   - Grafana (dashboards + trace/log exploration)
-#
-# The validator cluster runs as host processes, not containers.
-# run-full-validation.sh starts NUM_NODES (default 5) xrpld instances on
-# 127.0.0.1, each with a cfg it generates inline, peered to each other via
-# [ips_fixed]. They reach the collector through the published ports below and
-# write their logs into the bind-mounted workdir for the file_log receiver.
-#
-# Usage:
-#   # Start the telemetry backend on its own:
-#   docker compose -f docker/telemetry/docker-compose.workload.yaml up -d
-#
-#   # Or let the orchestrator start this stack and the node cluster together:
-#   docker/telemetry/workload/run-full-validation.sh
-#
-# Prerequisites (for the orchestrator, not for this stack):
-#   - xrpld binary built with -DXRPL_ENABLE_TELEMETRY=ON
-#   - Validator keys generated via generate-validator-keys.sh
-#
-# Image tags are pinned to the same versions as docker-compose.yml, which
-# mounts these same collector, Tempo and Prometheus config files. Floating
-# tags would let an upstream release change the harness result.
-#
-# Note: No Docker healthchecks are defined here. The orchestrator script
-# (run-full-validation.sh) polls each service endpoint directly from the
-# host, which avoids issues with missing curl/wget in container images.
-
-services:
-  # ---------------------------------------------------------------------------
-  # Telemetry Backend Stack
-  # ---------------------------------------------------------------------------
-
-  otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.158.0
-    command: ["--config=/etc/otel-collector-config.yaml"]
-    # Published on the host loopback only. The receivers have no auth and no
-    # TLS, so only processes on this host may reach them. Note this 127.0.0.1
-    # is the HOST interface docker listens on; the container-side bind lives in
-    # the collector config and is a separate choice. Upstream asks for a
-    # specific interface rather than 0.0.0.0 on either side (CWE-1327):
-    # https://opentelemetry.io/docs/security/config-best-practices/
-    ports:
-      - "127.0.0.1:4317:4317" # OTLP gRPC
-      - "127.0.0.1:4318:4318" # OTLP HTTP (traces + beast::insight metrics)
-      - "127.0.0.1:8889:8889" # Prometheus metrics endpoint
-      - "127.0.0.1:13133:13133" # Health check
-    volumes:
-      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
-      # Mount the validation workdir so the file_log receiver can tail node
-      # logs. run-full-validation.sh sets XRPLD_LOG_DIR to its workdir; the
-      # default matches that workdir so a bare `docker compose up` also works.
-      - ${XRPLD_LOG_DIR:-/tmp/xrpld-validation}:/var/log/xrpld:ro
-    depends_on:
-      - tempo
-    networks:
-      - workload-net
-
-  tempo:
-    image: grafana/tempo:2.9.4
-    command: ["-config.file=/etc/tempo.yaml"]
-    ports:
-      - "127.0.0.1:3200:3200" # Tempo HTTP API
-    volumes:
-      - ./tempo.yaml:/etc/tempo.yaml:ro
-      - tempo-data:/var/tempo
-    networks:
-      - workload-net
-
-  prometheus:
-    image: prom/prometheus:v3.13.2
-    ports:
-      - "127.0.0.1:9090:9090"
-    volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
-    depends_on:
-      - otel-collector
-    networks:
-      - workload-net
-
-  loki:
-    image: grafana/loki:3.7.6
-    ports:
-      - "127.0.0.1:3100:3100" # Loki HTTP API
-    command: ["-config.file=/etc/loki/local-config.yaml"]
-    networks:
-      - workload-net
-
-  grafana:
-    image: grafana/grafana:13.1.2
-    # Anonymous Admin is deliberate, and matches the sibling stack in
-    # docker-compose.yml. This stack is an ephemeral local/CI backend that
-    # run-full-validation.sh brings up and tears down around a single run; it
-    # holds no durable data and is never exposed beyond the host. Admin rather
-    # than Viewer because the harness drives the Grafana API against it, and
-    # the dashboards and datasources are provisioned from the mounts below.
-    environment:
-      - GF_AUTH_ANONYMOUS_ENABLED=true # No login required for local dev
-      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin # Full access without auth
-    ports:
-      - "127.0.0.1:3000:3000" # Grafana web UI
-    volumes:
-      - ./grafana/provisioning:/etc/grafana/provisioning:ro
-      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
-    depends_on:
-      - tempo
-      - prometheus
-      - loki
-    networks:
-      - workload-net
-
-volumes:
-  tempo-data:
-
-networks:
-  workload-net:
-    driver: bridge

docker/telemetry/docker-compose.yml

@@ -1,241 +0,0 @@
-# Docker Compose stack for xrpld OpenTelemetry observability.
-#
-# Provides services for local development:
-#   - otel-collector: receives OTLP traces from xrpld, batches and
-#     forwards them to Tempo. Also tails xrpld log files
-#     via file_log receiver and exports to Loki. Listens on ports
-#     4317 (gRPC) and 4318 (HTTP).
-#   - tempo: Grafana Tempo tracing backend, queryable via Grafana Explore
-#     on port 3000. Recommended for production (S3/GCS storage, TraceQL).
-#   - loki: Grafana Loki log aggregation backend for centralized log
-#     ingestion and log-trace correlation.
-#   - grafana: dashboards on port 3000, pre-configured with Tempo,
-#     Prometheus, and Loki datasources.
-#
-# Requires Docker Compose >= 2.24.0. The grafana service uses the long-form
-# `env_file` mapping (`path:` / `required:`), which older Compose cannot parse —
-# and it fails for the whole file, not just that service. The long form is
-# needed: `.env.alerting` is gitignored and absent in a fresh clone, and the
-# short form treats a missing env file as an error.
-#
-# Usage:
-#   docker compose -f docker/telemetry/docker-compose.yml up -d
-#
-# Configure xrpld to export traces by adding to xrpld.cfg:
-#   [telemetry]
-#   enabled=1
-#   traces_endpoint=http://localhost:4318/v1/traces
-
-services:
-  # One-shot init for the collector's offset store. Docker creates a fresh
-  # named volume owned by root, but the collector image runs as 10001:10001
-  # and ships no writable directory, so the file_storage extension could not
-  # create its database and the collector would fail to start. Chown the
-  # volume once, then exit; the collector waits for this to complete.
-  #
-  # Reuses the Prometheus image purely because the stack already pulls it and
-  # it has a shell — this adds no new image dependency. The entrypoint is
-  # overridden since that image normally starts the Prometheus server.
-  otelcol-storage-init:
-    image: prom/prometheus:v3.13.2
-    user: "0:0"
-    entrypoint: ["sh", "-c"]
-    command: ["mkdir -p /data/file_storage && chown -R 10001:10001 /data"]
-    volumes:
-      - otelcol-storage:/data
-    networks:
-      - xrpld-telemetry
-
-  # One-shot init for the xrpld log root. Docker creates a missing bind-mount
-  # source as root, and xrpld then cannot create the <network> subdirectory
-  # inside it. Config::getDebugLogFile() only warns on that failure and carries
-  # on, so the node looks healthy while writing no debug.log at all and the
-  # whole log pipeline stays empty with no error at any layer. Create the
-  # directory here and hand it to the host user instead.
-  #
-  # XRPLD_UID/XRPLD_GID default to 1000, the first non-root user on a typical
-  # Linux host. Set them if `id -u` differs, or xrpld still cannot write.
-  # Reuses the Prometheus image for the same reason otelcol-storage-init does.
-  xrpld-logdir-init:
-    image: prom/prometheus:v3.13.2
-    user: "0:0"
-    entrypoint: ["sh", "-c"]
-    command:
-      [
-        "mkdir -p /data/logs && chown ${XRPLD_UID:-1000}:${XRPLD_GID:-1000} /data /data/logs",
-      ]
-    volumes:
-      - ./data:/data
-    networks:
-      - xrpld-telemetry
-
-  # OpenTelemetry Collector: receives spans from xrpld via OTLP protocol,
-  # batches them for efficiency, and forwards to Tempo for storage.
-  otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.158.0
-    # Second --config layers file_log offset persistence on top of the shared
-    # base config; the collector deep-merges them. Only this stack keeps its
-    # logs across restarts, so only this stack needs it.
-    command:
-      [
-        "--config=/etc/otel-collector-config.yaml",
-        "--config=/etc/otel-collector-filestorage.yaml",
-      ]
-    # Published on the host loopback only. The receivers have no auth and no
-    # TLS, so only processes on this host may reach them. Note this 127.0.0.1
-    # is the HOST interface docker listens on; the container-side bind lives in
-    # the collector config and is a separate choice. Upstream asks for a
-    # specific interface rather than 0.0.0.0 on either side (CWE-1327):
-    # https://opentelemetry.io/docs/security/config-best-practices/
-    ports:
-      - "127.0.0.1:4317:4317" # OTLP gRPC
-      - "127.0.0.1:4318:4318" # OTLP HTTP (traces + native OTel metrics)
-      - "127.0.0.1:8889:8889" # Prometheus metrics (span_metrics + OTLP)
-      # StatsD UDP port removed — beast::insight now uses native OTLP.
-      # Uncomment if using server=statsd fallback:
-      # - "127.0.0.1:8125:8125/udp"
-      - "127.0.0.1:13133:13133" # Health check
-    volumes:
-      # Mount collector pipeline config (receivers → processors → exporters)
-      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
-      # Dev-only overlay: persist file_log read offsets across restarts
-      - ./otel-collector-filestorage.yaml:/etc/otel-collector-filestorage.yaml:ro
-      # Mount the xrpld log root for the file_log receiver. The telemetry
-      # configs write to docker/telemetry/data/logs/<network>/debug.log, so
-      # the default source is the repo-relative ./data/logs, which
-      # xrpld-logdir-init has already created and handed to the host user.
-      # Override XRPLD_LOG_DIR to point at another root (e.g. the integration
-      # test sets it to its own workdir; that root is created by the test, so
-      # the init service is a no-op there). Mounted read-only so the collector
-      # only tails.
-      - ${XRPLD_LOG_DIR:-./data/logs}:/var/log/xrpld:ro
-      # Persisted file_log read offsets, so a collector restart resumes
-      # instead of re-reading every debug.log from the top.
-      - otelcol-storage:/var/lib/otelcol
-    depends_on:
-      tempo:
-        condition: service_started
-      loki:
-        condition: service_started
-      otelcol-storage-init:
-        condition: service_completed_successfully
-      xrpld-logdir-init:
-        condition: service_completed_successfully
-    networks:
-      - xrpld-telemetry
-
-  # Grafana Tempo: distributed tracing backend that stores and indexes
-  # spans. Queryable via TraceQL in Grafana Explore.
-  tempo:
-    image: grafana/tempo:2.9.4
-    command: ["-config.file=/etc/tempo.yaml"]
-    ports:
-      - "127.0.0.1:3200:3200" # Tempo HTTP API (health check, query)
-    volumes:
-      # Mount Tempo storage and ingestion config
-      - ./tempo.yaml:/etc/tempo.yaml:ro
-      # Persistent volume for trace data (WAL + blocks)
-      - tempo-data:/var/tempo
-    networks:
-      - xrpld-telemetry
-
-  # Grafana Loki for centralized log ingestion and log-trace
-  # correlation. Loki 3.x supports native OTLP ingestion, so the OTel
-  # Collector exports via otlp_http to Loki's /otlp endpoint.
-  # Query logs via Grafana Explore -> Loki at http://localhost:3000.
-  loki:
-    image: grafana/loki:3.7.6
-    ports:
-      - "127.0.0.1:3100:3100"
-    command: -config.file=/etc/loki/local-config.yaml
-    volumes:
-      - loki-data:/loki
-    networks:
-      - xrpld-telemetry
-
-  prometheus:
-    # Pinned to an exact patch release for reproducible, config-stable runs.
-    image: prom/prometheus:v3.13.2
-    ports:
-      - "127.0.0.1:9090:9090"
-    volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - prometheus-data:/prometheus
-    depends_on:
-      - otel-collector
-    networks:
-      - xrpld-telemetry
-
-  # Grafana: visualization UI with Tempo pre-configured as a datasource.
-  # Anonymous admin access enabled for local development convenience.
-  grafana:
-    image: grafana/grafana:13.1.2
-    # The SMTP relay settings for email alerts come from this gitignored
-    # file. Contact-point destinations do not: edit them in place in
-    # contactpoints.yaml. An absent file leaves SMTP off, which is fine for a
-    # local stack. See .env.alerting.example.
-    env_file:
-      - path: .env.alerting
-        required: false
-    environment:
-      - GF_AUTH_ANONYMOUS_ENABLED=true # No login required for local dev
-      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin # Full access without auth
-      # Remote image rendering: point Grafana at the renderer container.
-      # These belong on the grafana service (the server delegates renders);
-      # the callback URL is how the renderer fetches the panel from grafana.
-      - GF_RENDERING_SERVER_URL=http://renderer:8081/render
-      - GF_RENDERING_CALLBACK_URL=http://grafana:3000/
-      # Grafana 13 turns the renderAuthJWT feature toggle on by default, which
-      # makes the renderer reject requests unless both sides share a token that
-      # is not the "-" default. Must match AUTH_TOKEN on the renderer service.
-      - GF_RENDERING_RENDERER_TOKEN=${GF_RENDERING_RENDERER_TOKEN:-xrpld-local-render}
-    # The SMTP settings for the critical-tier email receiver come from the
-    # env_file above, which puts them straight into the container
-    # environment. They are deliberately NOT duplicated here: a compose-level
-    # `environment:` entry is interpolated from the host shell and would
-    # override (and blank out) the env_file values.
-    ports:
-      - "127.0.0.1:3000:3000" # Grafana web UI
-    volumes:
-      # Auto-provision Tempo datasource and search filters on startup
-      - ./grafana/provisioning:/etc/grafana/provisioning:ro
-      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
-    depends_on:
-      - tempo
-      - prometheus
-      - loki
-      - renderer
-    networks:
-      - xrpld-telemetry
-
-  # Grafana image renderer: a sidecar that renders panels/dashboards to PNG
-  # for image export and alerting. Grafana calls it at http://renderer:8081.
-  renderer:
-    # Pinned like every other image in this stack so a stack that worked
-    # yesterday still works today. Upstream maintains only the newest release,
-    # which is expected to serve all supported Grafana versions.
-    image: grafana/grafana-image-renderer:v5.12.0
-    environment:
-      # Shared secret for the JWT-authenticated render requests Grafana 13
-      # sends. Must match GF_RENDERING_RENDERER_TOKEN on the grafana service.
-      - AUTH_TOKEN=${GF_RENDERING_RENDERER_TOKEN:-xrpld-local-render}
-    # No `ports:` on purpose. Grafana reaches this over the compose network at
-    # http://renderer:8081, so publishing 8081 on the host adds nothing the
-    # stack needs. AUTH_TOKEN above is the only guard on the endpoint, and its
-    # default is a fixed string in this file, so keep the service off the host.
-    networks:
-      - xrpld-telemetry
-# Named volume for Tempo trace storage (WAL and compacted blocks).
-# Data persists across container restarts. Remove with:
-#   docker compose -f docker/telemetry/docker-compose.yml down -v
-volumes:
-  tempo-data:
-  prometheus-data:
-  loki-data:
-  otelcol-storage:
-
-# Isolated bridge network so services communicate by container name
-# (e.g., the collector reaches Tempo at http://tempo:4317).
-networks:
-  xrpld-telemetry:
-    driver: bridge

docker/telemetry/grafana/dashboards/fee-market.json

@@ -1,714 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      },
-      {
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": false,
-        "iconColor": "rgb(70, 70, 70)",
-        "name": "Perf Runs (JMeter)",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": ["perf-iac", "jmeter"],
-          "type": "tags"
-        },
-        "type": "tags"
-      },
-      {
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": false,
-        "iconColor": "rgb(15, 122, 102)",
-        "name": "Perf Runs (Locust)",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": ["perf-iac", "locust"],
-          "type": "tags"
-        },
-        "type": "tags"
-      }
-    ]
-  },
-  "description": "What this shows: Transaction queue depth and capacity, fee-escalation levels, and load-factor breakdown that together set the minimum fee to enter the open ledger.  \u2014  Use it to: Understand why the minimum transaction fee is rising and how full the open ledger and queue are.",
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 1,
-  "id": null,
-  "links": [],
-  "panels": [
-    {
-      "title": "Queue Depth & Throughput",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 1
-    },
-    {
-      "title": "Transaction Queue Depth",
-      "description": "###### What this is:\n*Transactions currently waiting in the transaction queue versus the queue's maximum capacity.*\n\n###### How it's computed:\n*Instantaneous gauge readings of current queue count and configured max size.*\n\n###### Reading it:\n*Queue depth well below capacity is normal; depth approaching capacity means the node is saturating.*\n\n###### Healthy range:\n*Depth near 0 in quiet periods; workload-dependent under load.*\n\n###### Watch for:\n*Depth pinned at capacity for sustained periods, which signals demand exceeding throughput or a fee-spam burst.*\n\n###### Keywords:\n- **Transaction queue (TxQ)** *(per node)* \u2014 holds transactions that meet local cost but not the open-ledger cost, to include in a later ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (AppMetricGauges, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[AppMetricGauges.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/AppMetricGauges.cpp)\n\n###### Function:\n`registerTxqGauge`\n\n###### References:\n[Transaction queue (TxQ)](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-queue-txq)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 1
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_count\"}, \"series\", \"Queue Depth\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_max_size\"}, \"series\", \"Max Capacity\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "short",
-          "custom": {
-            "axisLabel": "Transactions",
-            "drawStyle": "line",
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "spanNulls": 1800000,
-            "pointSize": 5,
-            "gradientMode": "none"
-          },
-          "color": {
-            "mode": "palette-classic"
-          },
-          "decimals": 0
-        },
-        "overrides": []
-      },
-      "id": 2
-    },
-    {
-      "title": "Transactions Per Ledger",
-      "description": "###### What this is:\n*Transactions already placed in the current open ledger versus the expected per-ledger target.*\n\n###### How it's computed:\n*Instantaneous gauge readings of in-ledger count and the target count that governs fee escalation.*\n\n###### Reading it:\n*Staying at or below the expected target is normal; exceeding it triggers open-ledger fee escalation.*\n\n###### Healthy range:\n*At or under the expected per-ledger target.*\n\n###### Watch for:\n*In-ledger count persistently above target, indicating sustained congestion pushing fees up.*\n\n###### Keywords:\n- **Open ledger** *(per node)* \u2014 the temporary workspace ledger where incoming transactions are provisionally applied before a close.\n- **In-ledger vs target count** *(per node)* \u2014 transactions already in the open ledger versus the soft target that triggers fee escalation.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (AppMetricGauges, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[AppMetricGauges.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/AppMetricGauges.cpp)\n\n###### Function:\n`registerTxqGauge`\n\n###### References:\n[Open ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [In-ledger vs target count](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#open-ledger)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 1
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_in_ledger\"}, \"series\", \"In Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_per_ledger\"}, \"series\", \"Expected Per Ledger\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "short",
-          "custom": {
-            "axisLabel": "Transactions",
-            "drawStyle": "line",
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "spanNulls": 1800000,
-            "pointSize": 5,
-            "gradientMode": "none"
-          },
-          "color": {
-            "mode": "palette-classic"
-          }
-        },
-        "overrides": []
-      },
-      "id": 3
-    },
-    {
-      "title": "Fee Escalation",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 11
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 4
-    },
-    {
-      "title": "Fee Escalation Levels",
-      "description": "###### What this is:\n*The fee levels that govern queue admission: reference (baseline), minimum processing, median, and open-ledger levels.*\n\n###### How it's computed:\n*Instantaneous gauge readings of each fee level, shown on a log scale.*\n\n###### Reading it:\n*Open-ledger level near the reference level means cheap entry; a large gap above reference means escalation is active.*\n\n###### Healthy range:\n*Open-ledger level at or near reference during normal traffic.*\n\n###### Watch for:\n*Open-ledger level spiking far above reference, the hallmark of congestion or a fee-bidding war.*\n\n###### Keywords:\n- **Queue admission rejection** *(per node)* \u2014 a transaction refused entry to the queue, e.g. queue_full when the queue is at capacity.\n- **Fee levels** *(per node)* \u2014 cost thresholds governing queue admission: reference (baseline), minimum, median, and open-ledger.\n- **Open ledger** *(per node)* \u2014 the temporary workspace ledger where incoming transactions are provisionally applied before a close.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (AppMetricGauges, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[AppMetricGauges.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/AppMetricGauges.cpp)\n\n###### Function:\n`registerTxqGauge`\n\n###### References:\n[Queue admission rejection](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Fee levels](https://xrpl.org/docs/concepts/transactions/transaction-cost#fee-levels) \u00b7 [Open ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Fee escalation](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#queue-admission-rejection)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 12
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_reference_fee_level\"}, \"series\", \"Reference Fee Level\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_min_processing_fee_level\"}, \"series\", \"Min Processing Fee Level\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_med_fee_level\"}, \"series\", \"Median Fee Level\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"txq_open_ledger_fee_level\"}, \"series\", \"Open Ledger Fee Level\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "short",
-          "custom": {
-            "axisLabel": "Fee Level",
-            "drawStyle": "line",
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "scaleDistribution": {
-              "type": "log",
-              "log": 2
-            },
-            "spanNulls": 1800000,
-            "pointSize": 5,
-            "gradientMode": "none"
-          },
-          "color": {
-            "mode": "palette-classic"
-          }
-        },
-        "overrides": []
-      },
-      "id": 5
-    },
-    {
-      "title": "Load Factor Breakdown",
-      "description": "###### What this is:\n*The combined load factor and its server, fee-escalation, and fee-queue contributors as unitless fee multipliers (1.0 = no load).*\n\n###### How it's computed:\n*Instantaneous gauge readings of each load-factor component.*\n\n###### Reading it:\n*Values at 1.0 mean base fees; higher values raise the fee to transact.*\n\n###### Healthy range:\n*Around 1.0 under normal conditions.*\n\n###### Watch for:\n*Combined factor climbing well above 1.0, showing the node is charging premium fees due to congestion or overload.*\n\n###### Keywords:\n- **Load factor** *(per node)* \u2014 a unitless multiplier (1.0 = no load) that scales the base transaction cost as the node comes under load.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n- **Transaction cost** *(network-wide)* \u2014 the XRP a transaction destroys to be processed; scales up with load to deter spam.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (AppMetricGauges, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[AppMetricGauges.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/AppMetricGauges.cpp)\n\n###### Function:\n`registerLoadFactorGauge`\n\n###### References:\n[Load factor](https://xrpl.org/docs/concepts/transactions/transaction-cost#local-load-cost) \u00b7 [Fee escalation](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Transaction cost](https://xrpl.org/docs/concepts/transactions/transaction-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#load-factor)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 12
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor\"}, \"series\", \"Combined Load Factor\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_server\"}, \"series\", \"Server\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_fee_escalation\"}, \"series\", \"Fee Escalation\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_fee_queue\"}, \"series\", \"Fee Queue\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "short",
-          "custom": {
-            "axisLabel": "Multiplier",
-            "drawStyle": "line",
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "spanNulls": 1800000,
-            "pointSize": 5,
-            "gradientMode": "none"
-          },
-          "color": {
-            "mode": "palette-classic"
-          },
-          "thresholds": {
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 2
-              },
-              {
-                "color": "red",
-                "value": 10
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "id": 6
-    },
-    {
-      "title": "Load Factor Components",
-      "description": "###### What this is:\n*The individual load-factor inputs, local server load, network load, and cluster load, as unitless multipliers.*\n\n###### How it's computed:\n*Instantaneous gauge readings of each component.*\n\n###### Reading it:\n*All at 1.0 means no load pressure from any source; a raised component identifies where load originates.*\n\n###### Healthy range:\n*Around 1.0 for each component.*\n\n###### Watch for:\n*A single component rising sharply, which pinpoints whether the pressure is local, network-wide, or cluster-driven.*\n\n###### Keywords:\n- **Load factor** *(per node)* \u2014 a unitless multiplier (1.0 = no load) that scales the base transaction cost as the node comes under load.\n- **Cluster** *(cluster-wide)* \u2014 a group of trusted co-operated nodes that share load information and skip some verification.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (AppMetricGauges, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[AppMetricGauges.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/AppMetricGauges.cpp)\n\n###### Function:\n`registerLoadFactorGauge`\n\n###### References:\n[Load factor](https://xrpl.org/docs/concepts/transactions/transaction-cost#local-load-cost) \u00b7 [Cluster](https://xrpl.org/docs/concepts/networks-and-servers/clustering) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#load-factor)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 22
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_local\"}, \"series\", \"Local\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_net\"}, \"series\", \"Network\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"load_factor_cluster\"}, \"series\", \"Cluster\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "short",
-          "custom": {
-            "axisLabel": "Multiplier",
-            "drawStyle": "line",
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "spanNulls": 1800000,
-            "pointSize": 5,
-            "gradientMode": "none"
-          },
-          "color": {
-            "mode": "palette-classic"
-          }
-        },
-        "overrides": []
-      },
-      "id": 7
-    },
-    {
-      "title": "Queue Rejections",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 32
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 8
-    },
-    {
-      "title": "Queue Abandonment Rate (Expired)",
-      "description": "###### What this is:\n*Transactions dropped from the queue because their last-ledger deadline passed before they could be included.*\n\n###### How it's computed:\n*Per-second rate of the cumulative expired-transaction counter over the dashboard's rate interval.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means submitters under-bid the escalating fee and their transactions timed out.*\n\n###### Healthy range:\n*Near 0 expirations per second.*\n\n###### Watch for:\n*Sustained expiry rate, a demand-frustration signal often coinciding with fee spikes or spam that crowds out honest traffic.*\n\n###### Keywords:\n- **Queue admission rejection** *(per node)* \u2014 a transaction refused entry to the queue, e.g. queue_full when the queue is at capacity.\n- **Queue expiry / abandonment** *(per node)* \u2014 removing a queued transaction whose LastLedgerSequence deadline passed before inclusion.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementTxqExpired (caller TxQ.cpp)`\n\n###### References:\n[Queue admission rejection](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Queue expiry / abandonment](https://xrpl.org/docs/concepts/transactions/reliable-transaction-submission) \u00b7 [Fee escalation](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#queue-admission-rejection)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 33
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(txq_expired_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Expired / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "si: transactions/s",
-          "decimals": 0,
-          "custom": {
-            "axisLabel": "Transactions / Sec",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 9
-    },
-    {
-      "title": "Queue Admission Rejections (Dropped)",
-      "description": "###### What this is:\n*Transactions refused entry to the queue, broken down by reason such as queue_full.*\n\n###### How it's computed:\n*Per-second rate of the cumulative dropped-transaction counter over the dashboard's rate interval, split by reason.*\n\n###### Reading it:\n*Near zero is healthy; queue_full rejections mean the queue is at capacity and applying backpressure.*\n\n###### Healthy range:\n*Near 0 rejections per second.*\n\n###### Watch for:\n*A burst of queue_full drops, distinct from expiry, indicating the node is being flooded faster than it can drain.*\n\n###### Keywords:\n- **Queue admission rejection** *(per node)* \u2014 a transaction refused entry to the queue, e.g. queue_full when the queue is at capacity.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementTxqDropped (caller TxQ.cpp)`\n\n###### References:\n[Queue admission rejection](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#queue-admission-rejection)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 33
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(txq_dropped_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"$1\", \"reason\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "si: transactions/s",
-          "decimals": 0,
-          "custom": {
-            "axisLabel": "Transactions / Sec",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 10
-    }
-  ],
-  "schemaVersion": 39,
-  "tags": ["transactions", "fees"],
-  "templating": {
-    "list": [
-      {
-        "name": "DS_PROMETHEUS",
-        "type": "datasource",
-        "label": "Prometheus",
-        "query": "prometheus",
-        "regex": "",
-        "current": {},
-        "hide": 0,
-        "refresh": 1,
-        "includeAll": false,
-        "multi": false,
-        "options": []
-      },
-      {
-        "name": "service_name",
-        "label": "Service Name",
-        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
-        "type": "query",
-        "query": "label_values(service_name)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "xrpld",
-          "value": "xrpld"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "deployment_environment",
-        "label": "Environment",
-        "description": "Filter by deployment tier [local / ci / test / prod]",
-        "type": "query",
-        "query": "label_values(deployment_environment)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_network_type",
-        "label": "Network",
-        "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]",
-        "type": "query",
-        "query": "label_values(xrpl_network_type)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_work_item",
-        "label": "Work Item",
-        "description": "Filter by perf-iac work item / ticket",
-        "type": "query",
-        "query": "label_values(xrpl_work_item)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_branch",
-        "label": "Branch",
-        "description": "Filter by comparison side (baseline:<ref>:<commit> / test:<ref>:<commit>)",
-        "type": "query",
-        "query": "label_values(xrpl_branch)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_node_role",
-        "label": "Node Role",
-        "description": "Filter by node role (validator / peer)",
-        "type": "query",
-        "query": "label_values(xrpl_node_role)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "node",
-        "label": "Node",
-        "description": "Filter by rippled node (service.instance.id)",
-        "type": "query",
-        "query": "label_values(target_info, service_instance_id)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      }
-    ]
-  },
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "browser",
-  "title": "Fee Market & TxQ",
-  "uid": "fee-market",
-  "version": 1,
-  "refresh": "30s"
-}

docker/telemetry/grafana/dashboards/job-queue.json

@@ -1,888 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      },
-      {
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": false,
-        "iconColor": "rgb(70, 70, 70)",
-        "name": "Perf Runs (JMeter)",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": ["perf-iac", "jmeter"],
-          "type": "tags"
-        },
-        "type": "tags"
-      },
-      {
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": false,
-        "iconColor": "rgb(15, 122, 102)",
-        "name": "Perf Runs (Locust)",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": ["perf-iac", "locust"],
-          "type": "tags"
-        },
-        "type": "tags"
-      }
-    ]
-  },
-  "description": "What this shows: Per-job-type throughput, queue wait times, and execution times for the node's internal worker job queue.\nUse it to: Find job types that are backing up or running slowly and causing internal processing delays.",
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 1,
-  "id": null,
-  "links": [],
-  "panels": [
-    {
-      "title": "Current State",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 1
-    },
-    {
-      "title": "Slowest Job Queue Wait by Job Type (p99) [$xrpl_network_type]",
-      "description": "###### What this is:\n*At-a-glance p99 of how long jobs sit in the queue before a worker picks them up, by job type.*\n\n###### How it's computed:\n*99th percentile of the job wait-time histogram over the dashboard's selected time range.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Wait p99 under 100ms.*\n\n###### Watch for:\n*Wait p99 climbing into the red, meaning worker threads are saturated and jobs are backing up.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
-      "type": "bargauge",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 1
-      },
-      "options": {
-        "displayMode": "gradient",
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": false
-        },
-        "maxVizHeight": 300,
-        "minVizHeight": 16,
-        "minVizWidth": 8,
-        "namePlacement": "auto",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
-        },
-        "showUnfilled": true,
-        "sizing": "manual",
-        "valueMode": "color"
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "instant": true,
-          "range": false,
-          "expr": "histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__range])))"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.job_type} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "\u00b5s",
-          "min": 0,
-          "max": 1000000,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 100000
-              },
-              {
-                "color": "red",
-                "value": 1000000
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "id": 2,
-      "repeat": "xrpl_network_type",
-      "repeatDirection": "v"
-    },
-    {
-      "title": "Slowest Job Execution by Job Type (p99) [$xrpl_network_type]",
-      "description": "###### What this is:\n*At-a-glance p99 of how long jobs take to run once a worker starts them, by job type.*\n\n###### How it's computed:\n*99th percentile of the job run-time histogram over the dashboard's selected time range.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Exec p99 under 100ms.*\n\n###### Watch for:\n*One job type in the red, meaning a single handler is holding a worker thread for a long time.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
-      "type": "bargauge",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 1
-      },
-      "options": {
-        "displayMode": "gradient",
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": false
-        },
-        "maxVizHeight": 300,
-        "minVizHeight": 16,
-        "minVizWidth": 8,
-        "namePlacement": "auto",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
-        },
-        "showUnfilled": true,
-        "sizing": "manual",
-        "valueMode": "color"
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "instant": true,
-          "range": false,
-          "expr": "histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__range])))"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.job_type} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "\u00b5s",
-          "min": 0,
-          "max": 1000000,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 100000
-              },
-              {
-                "color": "red",
-                "value": 1000000
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "id": 13,
-      "repeat": "xrpl_network_type",
-      "repeatDirection": "v"
-    },
-    {
-      "title": "Throughput",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 11
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 3
-    },
-    {
-      "title": "Job Throughput Rate (Per Second)",
-      "description": "###### What this is:\n*Rate of jobs queued, started, and finished across all job types.*\n\n###### How it's computed:\n*Per-second rate of each cumulative job counter over the dashboard's rate interval.*\n\n###### Reading it:\n*Queued, started, and finished tracking together means the queue keeps up.*\n\n###### Healthy range:\n*Workload-dependent; the three rates should stay roughly equal.*\n\n###### Watch for:\n*Queued rate persistently above finished rate, which indicates a growing backlog.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobQueued / recordJobStarted / recordJobFinished`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 12
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval])), \"series\", \"Queued/s\", \"\", \"\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval])), \"series\", \"Started/s\", \"\", \"\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval])), \"series\", \"Finished/s\", \"\", \"\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "si: jobs/s",
-          "decimals": 0,
-          "custom": {
-            "drawStyle": "line",
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "axisLabel": "Jobs / Sec",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "gradientMode": "none"
-          },
-          "color": {
-            "mode": "palette-classic"
-          }
-        },
-        "overrides": []
-      },
-      "id": 4
-    },
-    {
-      "title": "Per-Job-Type Queued Rate",
-      "description": "###### What this is:\n*Rate of jobs entering the queue, broken down by job type (top 10).*\n\n###### How it's computed:\n*Per-second rate of the queued-job counter per job_type over the dashboard's rate interval.*\n\n###### Reading it:\n*Identifies which job types generate the most queue activity.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A single job type surging unexpectedly, which can point to a flood of a particular request or peer message.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobQueued`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 24,
-        "x": 0,
-        "y": 22
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        },
-        "legend": {
-          "displayMode": "table",
-          "placement": "right",
-          "calcs": ["mean", "max"]
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "expr": "label_replace(topk(10, sum by (job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval]))), \"series\", \"$1\", \"job_type\", \"(.*)\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "si: jobs/s",
-          "decimals": 0,
-          "custom": {
-            "drawStyle": "line",
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "axisLabel": "Jobs / Sec",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "gradientMode": "none"
-          },
-          "color": {
-            "mode": "palette-classic"
-          }
-        },
-        "overrides": []
-      },
-      "id": 5
-    },
-    {
-      "title": "Per-Job-Type Finish Rate",
-      "description": "###### What this is:\n*Rate of jobs completing, broken down by job type (top 10).*\n\n###### How it's computed:\n*Per-second rate of the finished-job counter per job_type over the dashboard's rate interval.*\n\n###### Reading it:\n*Compare against the queued rate per type to spot which types are falling behind.*\n\n###### Healthy range:\n*Workload-dependent; should match the per-type queued rate.*\n\n###### Watch for:\n*A type whose finish rate lags its queued rate, revealing where the backlog concentrates.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 24,
-        "x": 0,
-        "y": 32
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        },
-        "legend": {
-          "displayMode": "table",
-          "placement": "right",
-          "calcs": ["mean", "max"]
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "expr": "label_replace(topk(10, sum by (job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval]))), \"series\", \"$1\", \"job_type\", \"(.*)\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "si: jobs/s",
-          "decimals": 0,
-          "custom": {
-            "drawStyle": "line",
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "axisLabel": "Jobs / Sec",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "gradientMode": "none"
-          },
-          "color": {
-            "mode": "palette-classic"
-          }
-        },
-        "overrides": []
-      },
-      "id": 6
-    },
-    {
-      "title": "Latency",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 42
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 7
-    },
-    {
-      "title": "Job Queue Wait Time",
-      "description": "###### What this is:\n*How long jobs sit in the queue before a worker picks them up, split by job type (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job wait-time histogram over a 5-minute window, kept per job type. Limited to the ten types with the highest wait so the legend stays readable.*\n\n###### Reading it:\n*Lower is better. The legend names the job type, so a single slow queue is identifiable rather than hidden in an all-types average. A widening p75-to-p99 gap on one type signals occasional stalls there.*\n\n###### Healthy range:\n*Sub-millisecond to low-millisecond waits on an unloaded node.*\n\n###### Watch for:\n*Rising p99 wait on a capped type -- ledgerRequest, ledgerData and makeFetchPack have small concurrency limits, so they queue first. Cross-check the deferred gauge for that type.*\n\n###### Keywords:\n- **Job queue** *(per node)* \u2014 the worker pool that runs xrpld's background work; each unit of work is a job with a type.\n- **Dequeue wait** *(per node)* \u2014 time a job sits enqueued before a worker starts it, as distinct from how long it then runs.\n- **Concurrency limit** *(per node)* \u2014 the maximum number of jobs of one type allowed to run at once; work beyond it is deferred, not rejected.\n\n###### Computation boundary:\n*Result: Per node and job type \u2014 each series is one server's own value for one job type.*\n*Recorded in code as an OTel SDK histogram, then aggregated to percentiles by the Grafana query.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted`\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type) \u00b7 [Concurrency limit](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#concurrency-limit)\n",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 43
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "expr": "label_replace(topk(10, histogram_quantile(0.75, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"p75 Wait $1\", \"job_type\", \"(.*)\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "expr": "label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"p99 Wait $1\", \"job_type\", \"(.*)\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "\u00b5s",
-          "custom": {
-            "drawStyle": "line",
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "axisLabel": "Duration (\u00b5s)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "gradientMode": "none"
-          },
-          "color": {
-            "mode": "palette-classic"
-          }
-        },
-        "overrides": []
-      },
-      "id": 8
-    },
-    {
-      "title": "Job Execution Time",
-      "description": "###### What this is:\n*How long jobs run once started, split by job type (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job run-time histogram over a 5-minute window, kept per job type. Limited to the ten slowest types so the legend stays readable.*\n\n###### Reading it:\n*Lower is better. The legend names the job type, so an expensive job type is identifiable rather than averaged away. Stable p75 with a controlled p99 is healthy.*\n\n###### Healthy range:\n*Workload-dependent, but stable over time.*\n\n###### Watch for:\n*Growing execution times, which point to CPU pressure or expensive individual jobs.*\n\n###### Keywords:\n- **Job queue** *(per node)* \u2014 the worker pool that runs xrpld's background work; each unit of work is a job with a type.\n- **Execution time** *(per node)* \u2014 time a job spends running after a worker picks it up, excluding its queue wait.\n\n###### Computation boundary:\n*Result: Per node and job type \u2014 each series is one server's own value for one job type.*\n*Recorded in code as an OTel SDK histogram, then aggregated to percentiles by the Grafana query.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)\n",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 43
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "expr": "label_replace(topk(10, histogram_quantile(0.75, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"p75 Exec $1\", \"job_type\", \"(.*)\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "expr": "label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"p99 Exec $1\", \"job_type\", \"(.*)\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "\u00b5s",
-          "custom": {
-            "drawStyle": "line",
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "axisLabel": "Duration (\u00b5s)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "gradientMode": "none"
-          },
-          "color": {
-            "mode": "palette-classic"
-          }
-        },
-        "overrides": []
-      },
-      "id": 9
-    },
-    {
-      "title": "Per-Job-Type Execution Time (p99)",
-      "description": "###### What this is:\n*The 10 slowest job types ranked by p99 execution time.*\n\n###### How it's computed:\n*p99 derived from the run-time histogram per job_type over a 5-minute window, top 10 selected.*\n\n###### Reading it:\n*Highlights which job types cost the most CPU time at the tail.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A job type whose p99 grows over time, indicating a slow or degrading operation.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 24,
-        "x": 0,
-        "y": 53
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        },
-        "legend": {
-          "displayMode": "table",
-          "placement": "right",
-          "calcs": ["mean", "max"]
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "expr": "label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"$1\", \"job_type\", \"(.*)\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "\u00b5s",
-          "custom": {
-            "drawStyle": "line",
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "axisLabel": "Duration (\u00b5s)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "gradientMode": "none"
-          },
-          "color": {
-            "mode": "palette-classic"
-          }
-        },
-        "overrides": []
-      },
-      "id": 10
-    },
-    {
-      "title": "Overflow",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 63
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 11
-    },
-    {
-      "title": "Transaction Overflow Rate",
-      "description": "###### What this is:\n*Rate at which transaction jobs are shed when the queue's transaction limit is exceeded.*\n\n###### How it's computed:\n*Per-second rate of the overflow counter over the dashboard's rate interval, scaled to per minute. The counter is observed from the overlay's cumulative overflow tally.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means the job queue is shedding transaction work under load.*\n\n###### Healthy range:\n*0 overflows per minute.*\n\n###### Watch for:\n*Any sustained non-zero rate \u2014 the node is dropping transaction jobs because the queue is saturated.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (AppMetricGauges, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[AppMetricGauges.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/AppMetricGauges.cpp)\n\n###### Function:\n`AppMetricGauges::registerJqTransOverflowCounter (observed from Overlay::getJqTransOverflow)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 64
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "expr": "label_replace(rate(jq_trans_overflow_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Overflows/min\", \"\", \"\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "si: overflows/min",
-          "decimals": 0,
-          "thresholds": {
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 1
-              },
-              {
-                "color": "red",
-                "value": 10
-              }
-            ]
-          },
-          "custom": {
-            "axisLabel": "Overflows / Min",
-            "drawStyle": "line",
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "gradientMode": "none"
-          },
-          "color": {
-            "mode": "palette-classic"
-          }
-        },
-        "overrides": []
-      },
-      "id": 12
-    }
-  ],
-  "schemaVersion": 39,
-  "tags": ["node", "jobqueue"],
-  "templating": {
-    "list": [
-      {
-        "name": "DS_PROMETHEUS",
-        "type": "datasource",
-        "label": "Prometheus",
-        "query": "prometheus",
-        "regex": "",
-        "current": {},
-        "hide": 0,
-        "refresh": 1,
-        "includeAll": false,
-        "multi": false,
-        "options": []
-      },
-      {
-        "name": "service_name",
-        "label": "Service Name",
-        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
-        "type": "query",
-        "query": "label_values(service_name)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "xrpld",
-          "value": "xrpld"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "deployment_environment",
-        "label": "Environment",
-        "description": "Filter by deployment tier [local / ci / test / prod]",
-        "type": "query",
-        "query": "label_values(deployment_environment)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_network_type",
-        "label": "Network",
-        "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]",
-        "type": "query",
-        "query": "label_values(xrpl_network_type)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_work_item",
-        "label": "Work Item",
-        "description": "Filter by perf-iac work item / ticket",
-        "type": "query",
-        "query": "label_values(xrpl_work_item)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_branch",
-        "label": "Branch",
-        "description": "Filter by comparison side (baseline:<ref>:<commit> / test:<ref>:<commit>)",
-        "type": "query",
-        "query": "label_values(xrpl_branch)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_node_role",
-        "label": "Node Role",
-        "description": "Filter by node role (validator / peer)",
-        "type": "query",
-        "query": "label_values(xrpl_node_role)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "node",
-        "label": "Node",
-        "description": "Filter by rippled node (service.instance.id)",
-        "type": "query",
-        "query": "label_values(job_queued_us_count, service_instance_id)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "job_type",
-        "label": "Job Type",
-        "description": "Filter by job type",
-        "type": "query",
-        "query": "label_values(job_queued_total, job_type)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "handler",
-        "label": "Handler",
-        "description": "Filter by enqueueing handler (sanitised addJob name, e.g. RcvGetLedger)",
-        "type": "query",
-        "query": "label_values(job_queued_total, handler)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      }
-    ]
-  },
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "browser",
-  "title": "Job Queue Analysis",
-  "uid": "job-queue",
-  "version": 1,
-  "refresh": "30s"
-}

docker/telemetry/grafana/dashboards/ledger-operations.json

@@ -1,728 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      },
-      {
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": false,
-        "iconColor": "rgb(70, 70, 70)",
-        "name": "Perf Runs (JMeter)",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": ["perf-iac", "jmeter"],
-          "type": "tags"
-        },
-        "type": "tags"
-      },
-      {
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": false,
-        "iconColor": "rgb(15, 122, 102)",
-        "name": "Perf Runs (Locust)",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": ["perf-iac", "locust"],
-          "type": "tags"
-        },
-        "type": "tags"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 1,
-  "id": null,
-  "links": [],
-  "panels": [
-    {
-      "title": "Ledger Build & Validate",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 1
-    },
-    {
-      "title": "Ledger Build Rate [$xrpl_network_type]",
-      "description": "###### What this is:\n*How many new ledgers this node finishes building per second.*\n\n###### How it's computed:\n*Per-second rate of completed ledger-build operations, over the dashboard's rate interval, split by node.*\n\n###### Reading it:\n*A steady flat line; the value should track the network close cadence.*\n\n###### Healthy range:\n*About 0.2-0.3 ledgers/sec on mainnet (roughly one every 3-5s); workload-dependent on test networks.*\n\n###### Watch for:\n*A drop toward zero (node fell out of sync or stalled) or a value well above the network rate (rebuilding history).*\n\n###### Keywords:\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`\n\n###### References:\n[Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-build)",
-      "type": "stat",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 1
-      },
-      "options": {
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.build\"}[$__rate_interval])), \"series\", \"Builds / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "si: ledgers/s",
-          "decimals": 0
-        },
-        "overrides": []
-      },
-      "repeat": "xrpl_network_type",
-      "repeatDirection": "h",
-      "maxPerRow": 2,
-      "id": 2
-    },
-    {
-      "title": "Ledger Validation Rate [$xrpl_network_type]",
-      "description": "###### What this is:\n*How often ledgers reach full validation (accepted by the trusted validator quorum) per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-validation events over the dashboard's rate interval, per node.*\n\n###### Reading it:\n*Should closely match the build rate under normal, in-sync operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; workload-dependent elsewhere.*\n\n###### Watch for:\n*A validation rate that lags the build rate, signalling the node is building ahead of the network consensus it trusts.*\n\n###### Keywords:\n- **Ledger validation** *(network event)* \u2014 the second consensus stage where the node confirms a built ledger matches the trusted validator quorum and marks it final.\n- **Validation quorum** *(network-wide)* \u2014 the minimum number of agreeing trusted validations needed to declare a ledger fully validated.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`\n\n###### References:\n[Ledger validation](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Validation quorum](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-validation)",
-      "type": "stat",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "options": {
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.validate\"}[$__rate_interval])), \"series\", \"Validations / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "si: ledgers/s",
-          "decimals": 0
-        },
-        "overrides": []
-      },
-      "repeat": "xrpl_network_type",
-      "repeatDirection": "h",
-      "maxPerRow": 2,
-      "id": 3
-    },
-    {
-      "title": "Ledger Build Duration",
-      "description": "###### What this is:\n*The time taken to build a single ledger, at the 95th percentile.*\n\n###### How it's computed:\n*95th-percentile of ledger-build durations over a 5-minute window, per node.*\n\n###### Reading it:\n*Lower is better; the line should stay well under the ledger interval.*\n\n###### Healthy range:\n*Typically tens to low hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained rises approaching the close interval, which indicate heavy transaction sets or disk/I/O pressure.*\n\n###### Keywords:\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`\n\n###### References:\n[Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-build)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 21
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.build\"}[5m]))), \"series\", \"P95 Build Duration\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "ms",
-          "custom": {
-            "axisLabel": "Duration (ms)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 4
-    },
-    {
-      "title": "Ledger Build Duration Heatmap",
-      "description": "###### What this is:\n*The full distribution of ledger-build times over the window, not just a single percentile.*\n\n###### How it's computed:\n*Counts of ledger builds falling in each duration band per 5-minute window, shown as color density.*\n\n###### Reading it:\n*A tight low band is healthy; scattered high cells mean occasional slow builds.*\n\n###### Healthy range:\n*Most mass concentrated in the low-millisecond bands; workload-dependent.*\n\n###### Watch for:\n*A second cluster of hot cells at high durations (bimodal build times) hidden by percentile charts.*\n\n###### Keywords:\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`\n\n###### References:\n[Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-build)",
-      "type": "heatmap",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 21
-      },
-      "options": {
-        "calculate": false,
-        "cellGap": 1,
-        "color": {
-          "mode": "scheme",
-          "scheme": "Turbo",
-          "steps": 64
-        },
-        "legend": {
-          "show": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "desc",
-          "maxHeight": 600
-        },
-        "yAxis": {
-          "axisLabel": "Duration (ms)"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "sum(increase(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.build\"}[5m])) by (le)",
-          "legendFormat": "{{le}}",
-          "format": "heatmap"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "unit": "ms"
-        },
-        "overrides": []
-      },
-      "id": 5
-    },
-    {
-      "title": "Transaction Apply",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 31
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 6
-    },
-    {
-      "title": "Transaction Apply Duration",
-      "description": "###### What this is:\n*The time spent applying the agreed transaction set into the new ledger, at the 95th percentile.*\n\n###### How it's computed:\n*95th-percentile of transaction-apply durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; this is a large share of total build time.*\n\n###### Healthy range:\n*A few to tens of milliseconds; scales with transaction volume per ledger.*\n\n###### Watch for:\n*Spikes during large or expensive transaction sets, which push out overall ledger build time.*\n\n###### Keywords:\n- **Transaction apply phase** *(per node)* \u2014 the step that executes the agreed transaction set into the new ledger during a close.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`applyTransactions`\n\n###### References:\n[Transaction apply phase](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-apply-phase)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 32
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.apply\"}[5m]))), \"series\", \"P95 tx.apply\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "ms",
-          "custom": {
-            "axisLabel": "Duration (ms)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 7
-    },
-    {
-      "title": "Transaction Apply Rate",
-      "description": "###### What this is:\n*How often the transaction-apply phase runs per second (once per ledger build).*\n\n###### How it's computed:\n*Per-second rate of transaction-apply operations over the dashboard's rate interval, per node.*\n\n###### Reading it:\n*Should track the ledger build rate almost exactly.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; workload-dependent.*\n\n###### Watch for:\n*Divergence from the build rate, which would indicate a metric or pipeline anomaly.*\n\n###### Keywords:\n- **Transaction apply phase** *(per node)* \u2014 the step that executes the agreed transaction set into the new ledger during a close.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`applyTransactions`\n\n###### References:\n[Transaction apply phase](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-apply-phase)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 32
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"tx.apply\"}[$__rate_interval])), \"series\", \"tx.apply / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "si: transactions/s",
-          "decimals": 0,
-          "custom": {
-            "axisLabel": "Transactions / Sec",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 8
-    },
-    {
-      "title": "Store & Close Timing",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 42
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 9
-    },
-    {
-      "title": "Ledger Store Rate [$xrpl_network_type]",
-      "description": "###### What this is:\n*How often completed ledgers are written into ledger history per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-store operations over the dashboard's rate interval, per node.*\n\n###### Reading it:\n*Should match the build rate during normal operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; can burst higher while backfilling history.*\n\n###### Watch for:\n*A store rate below the build rate (storage falling behind) or a stall at zero.*\n\n###### Keywords:\n- **Ledger store** *(per node)* \u2014 writing a completed ledger into the node's ledger history on disk.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Consensus stall** *(per node)* \u2014 a health check reporting that consensus is not making forward progress.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::storeLedger`\n\n###### References:\n[Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus stall](https://xrpl.org/docs/concepts/consensus-protocol/consensus-principles-and-rules) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-store)",
-      "type": "stat",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 43
-      },
-      "options": {
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.store\"}[$__rate_interval])), \"series\", \"Stores / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "si: ledgers/s",
-          "decimals": 0
-        },
-        "overrides": []
-      },
-      "repeat": "xrpl_network_type",
-      "repeatDirection": "h",
-      "maxPerRow": 2,
-      "id": 10
-    },
-    {
-      "title": "Build vs Close Duration",
-      "description": "###### What this is:\n*Ledger build time compared with the full consensus round duration, both at the 95th percentile.*\n\n###### How it's computed:\n*Two 95th-percentile duration series over 5 minutes: ledger construction (ledger.build span) and the whole consensus round from open to accept (consensus.round span), per node.*\n\n###### Reading it:\n*Build should sit well below the round; the gap is consensus wait time (proposing, converging, validating) outside construction.*\n\n###### Healthy range:\n*Round tracks the network close interval (~3-5s on mainnet); build is a fraction of it (tens to hundreds of ms).*\n\n###### Watch for:\n*Build time approaching the round duration \u2014 construction is dominating the close and leaving little slack.*\n\n###### Note:\n*The close series uses consensus.round, not consensus.ledger_close: the latter span only wraps the onClose() prologue (sub-millisecond) and is not the ledger close time.*\n\n###### Keywords:\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Consensus mode** *(per node)* \u2014 the node's role/health in the current round: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Ledger close** *(network event)* \u2014 the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp) \u00b7 [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`buildLedgerImpl ; RCLConsensus::Adaptor::onClose (round span)`\n\n###### References:\n[Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-round)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 53
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.build\"}[5m]))), \"series\", \"P95 ledger.build\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.round\"}[5m]))), \"series\", \"P95 Close (consensus.round)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "ms",
-          "custom": {
-            "axisLabel": "Duration (ms)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 11
-    },
-    {
-      "title": "Ledger Close Interval & Age",
-      "description": "###### What this is:\n*The wall-clock time between consecutive ledger closes \u2014 the network close cadence.*\n\n###### How it's computed:\n*Close Interval: 1 / rate(ledgers_closed_total), the average seconds between closes from the monotonic close counter (scrape-independent, unlike a gauge delta which would alias to the scrape period). Last-Close Age: time() minus the last-close network time (server_info last_close_time gauge + Ripple-epoch offset), i.e. seconds since the last ledger closed.*\n\n###### Reading it:\n*A steady line near the network's target close interval.*\n\n###### Healthy range:\n*About 3-5s on mainnet; workload-dependent on test networks.*\n\n###### Watch for:\n*A rising interval (consensus slowing or the node lagging) or a flat line at zero (ledgers no longer closing).*\n\n###### Keywords:\n- **Ledger close** *(network event)* \u2014 the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Last-close time: computed in xrpld code (AppMetricGauges, OpenTelemetry SDK). Ledgers closed: computed in xrpld code (call-site metric macro, OpenTelemetry SDK); MetricsRegistry creates it at 0. The collector only forwards them; the Grafana queries turn them into the age and the interval.*\n\n###### Source:\n[AppMetricGauges.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/AppMetricGauges.cpp) \u00b7 [MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp) \u00b7 [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`AppMetricGauges::registerServerInfoGauge` (last_close_time) \u00b7 `RCLConsensus::Adaptor::doAccept` (ledgers_closed_total) \u00b7 `MetricsRegistry::initSyncInstruments` (creates it at 0)\n\n###### References:\n[Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-close)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 53
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(time() - (server_info{metric=\"last_close_time\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} + 946684800), \"series\", \"Last-Close Age (s)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(1 / sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(ledgers_closed_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Close Interval\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "s",
-          "custom": {
-            "axisLabel": "Seconds",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byRegexp",
-              "options": "/^Last-Close Age/"
-            },
-            "properties": [
-              {
-                "id": "unit",
-                "value": "s"
-              },
-              {
-                "id": "custom.axisLabel",
-                "value": "Age Since Last Close (Sec)"
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byRegexp",
-              "options": "/^Close Interval/"
-            },
-            "properties": [
-              {
-                "id": "unit",
-                "value": "s"
-              },
-              {
-                "id": "custom.axisPlacement",
-                "value": "right"
-              },
-              {
-                "id": "custom.axisLabel",
-                "value": "Close Interval (Sec)"
-              }
-            ]
-          }
-        ]
-      },
-      "id": 12
-    }
-  ],
-  "schemaVersion": 39,
-  "tags": ["ledger"],
-  "templating": {
-    "list": [
-      {
-        "name": "DS_PROMETHEUS",
-        "type": "datasource",
-        "label": "Prometheus",
-        "query": "prometheus",
-        "regex": "",
-        "current": {},
-        "hide": 0,
-        "refresh": 1,
-        "includeAll": false,
-        "multi": false,
-        "options": []
-      },
-      {
-        "name": "service_name",
-        "label": "Service Name",
-        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
-        "type": "query",
-        "query": "label_values(service_name)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "xrpld",
-          "value": "xrpld"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "deployment_environment",
-        "label": "Environment",
-        "description": "Filter by deployment tier [local / ci / test / prod]",
-        "type": "query",
-        "query": "label_values(deployment_environment)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_network_type",
-        "label": "Network",
-        "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]",
-        "type": "query",
-        "query": "label_values(xrpl_network_type)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_work_item",
-        "label": "Work Item",
-        "description": "Filter by perf-iac work item / ticket",
-        "type": "query",
-        "query": "label_values(xrpl_work_item)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_branch",
-        "label": "Branch",
-        "description": "Filter by comparison side (baseline:<ref>:<commit> / test:<ref>:<commit>)",
-        "type": "query",
-        "query": "label_values(xrpl_branch)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_node_role",
-        "label": "Node Role",
-        "description": "Filter by node role (validator / peer)",
-        "type": "query",
-        "query": "label_values(xrpl_node_role)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "node",
-        "label": "Node",
-        "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)",
-        "type": "query",
-        "query": "label_values(target_info, service_instance_id)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      }
-    ]
-  },
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "title": "Ledger Operations",
-  "uid": "ledger-operations",
-  "refresh": "30s",
-  "description": "What this shows: Ledger construction, validation, and storage activity and timing for this node.  \u2014  Use it to: Confirm ledgers are being built, validated, and stored on schedule and find the slow stage when they are not."
-}

docker/telemetry/grafana/dashboards/peer-network.json

@@ -1,588 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      },
-      {
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": false,
-        "iconColor": "rgb(70, 70, 70)",
-        "name": "Perf Runs (JMeter)",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": ["perf-iac", "jmeter"],
-          "type": "tags"
-        },
-        "type": "tags"
-      },
-      {
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": false,
-        "iconColor": "rgb(15, 122, 102)",
-        "name": "Perf Runs (Locust)",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": ["perf-iac", "locust"],
-          "type": "tags"
-        },
-        "type": "tags"
-      }
-    ]
-  },
-  "description": "What this shows: Consensus proposals and validations received from peers, and transaction relay efficiency for this node.  \u2014  Use it to: Check that peers are feeding this node proposals, validations, and transactions, and gauge relay overhead.",
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 1,
-  "id": null,
-  "links": [],
-  "panels": [
-    {
-      "title": "Proposal & Validation Receive",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 1
-    },
-    {
-      "title": "Peer Proposal Receive Rate",
-      "description": "###### What this is:\n*How many consensus proposals this node receives from peers per second.*\n\n###### How it's computed:\n*Per-second rate of received proposals over the dashboard's rate interval, per node.*\n\n###### Reading it:\n*A steady rate roughly proportional to the number of proposing validators.*\n\n###### Healthy range:\n*Workload-dependent; scales with validator count and connectivity.*\n\n###### Watch for:\n*A drop toward zero (isolation from the network) or a sudden flood far above baseline (proposal spam).*\n\n###### Keywords:\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Proposers** *(network event)* \u2014 the count of validators whose proposals this node heard in the last closed round.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMProposeSet)`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 1
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"peer.proposal.receive\"}[$__rate_interval])), \"series\", \"Proposals Received / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "si: proposals/s",
-          "decimals": 0,
-          "custom": {
-            "axisLabel": "Proposals / Sec",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 2
-    },
-    {
-      "title": "Peer Validation Receive Rate",
-      "description": "###### What this is:\n*How many ledger validations this node receives from peers per second.*\n\n###### How it's computed:\n*Per-second rate of received validations over the dashboard's rate interval, per node.*\n\n###### Reading it:\n*Steady and proportional to the number of validators the node hears from.*\n\n###### Healthy range:\n*Workload-dependent; roughly one burst per validator per closed ledger.*\n\n###### Watch for:\n*A fall toward zero (loss of validator connectivity) or an abnormal surge from untrusted sources.*\n\n###### Keywords:\n- **Trusted / untrusted / duplicate** *(per node)* \u2014 message classification: trusted (from UNL validators), untrusted (others), or duplicate (already seen).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMValidation)`\n\n###### References:\n[Trusted / untrusted / duplicate](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#trusted-untrusted-duplicate)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 1
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"peer.validation.receive\"}[$__rate_interval])), \"series\", \"Validations Received / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "si: validations/s",
-          "decimals": 0,
-          "custom": {
-            "axisLabel": "Validations / Sec",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 3
-    },
-    {
-      "title": "Trust Split",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 11
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 4
-    },
-    {
-      "title": "Proposals Trusted vs Untrusted",
-      "description": "###### What this is:\n*The share of received proposals that come from trusted (UNL) validators versus everyone else.*\n\n###### How it's computed:\n*Received-proposal counts over the dashboard's rate interval, split by trust status and shown as proportions.*\n\n###### Reading it:\n*A healthy node with a good UNL shows a solid trusted slice.*\n\n###### Healthy range:\n*Workload-dependent; a meaningful trusted fraction is expected.*\n\n###### Watch for:\n*A collapsing trusted share or a large untrusted volume, consistent with proposal flooding from non-UNL peers.*\n\n###### Keywords:\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Trusted / untrusted / duplicate** *(per node)* \u2014 message classification: trusted (from UNL validators), untrusted (others), or duplicate (already seen).\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMProposeSet)`\n\n###### References:\n[Proposal](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Trusted / untrusted / duplicate](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#proposal)",
-      "type": "piechart",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 12
-      },
-      "options": {
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (proposal_trusted, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"proposal_trusted\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "id": 5
-    },
-    {
-      "title": "Validations Trusted vs Untrusted",
-      "description": "###### What this is:\n*The share of received validations from trusted (UNL) validators versus untrusted sources.*\n\n###### How it's computed:\n*Received-validation counts over the dashboard's rate interval, split by trust status and shown as proportions.*\n\n###### Reading it:\n*The trusted slice should dominate for a well-configured node.*\n\n###### Healthy range:\n*Workload-dependent; trusted validations expected to be the majority.*\n\n###### Watch for:\n*A shrinking trusted share or a spike of untrusted validations, which can indicate misconfiguration or abuse.*\n\n###### Keywords:\n- **Trusted / untrusted / duplicate** *(per node)* \u2014 message classification: trusted (from UNL validators), untrusted (others), or duplicate (already seen).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMValidation)`\n\n###### References:\n[Trusted / untrusted / duplicate](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#trusted-untrusted-duplicate)",
-      "type": "piechart",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 12
-      },
-      "options": {
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (validation_trusted, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"validation_trusted\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "id": 6
-    },
-    {
-      "title": "Reduce-Relay",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 22
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 7
-    },
-    {
-      "title": "Reduce-Relay Peer Selection",
-      "description": "###### What this is:\n*How transaction relay picks peers: chosen relay sources, suppressed peers, and peers with the feature off.*\n\n###### How it's computed:\n*Current peer counts in each category (selected, suppressed, not-enabled), per node.*\n\n###### Reading it:\n*A high suppressed-to-selected ratio means relay is saving bandwidth as intended.*\n\n###### Healthy range:\n*Workload-dependent; suppressed should exceed selected in a well-connected mesh.*\n\n###### Watch for:\n*A large not-enabled count (older peers forcing full relay) or selected climbing while suppressed falls.*\n\n###### Keywords:\n- **Reduce-relay** *(per node)* \u2014 an optimization that relays messages through selected peers only, suppressing redundant forwarding.\n- **Transaction suppression** *(per node)* \u2014 dropping a transaction already seen from another peer, so it is not reprocessed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (AppMetricGauges, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[AppMetricGauges.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/AppMetricGauges.cpp)\n\n###### Function:\n`registerReduceRelayGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#reduce-relay)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 23
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(reduce_relay_metrics{metric=\"selected_peers\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Selected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(reduce_relay_metrics{metric=\"suppressed_peers\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Suppressed\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(reduce_relay_metrics{metric=\"not_enabled_peers\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Not Enabled\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "short",
-          "custom": {
-            "axisLabel": "Peer Count",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          },
-          "decimals": 0
-        },
-        "overrides": []
-      },
-      "id": 8
-    },
-    {
-      "title": "Reduce-Relay Missing-Tx Frequency",
-      "description": "###### What this is:\n*How often a peer has to fetch a transaction it missed because relay suppressed it.*\n\n###### How it's computed:\n*The reported frequency of on-demand missing-transaction fetches, per node.*\n\n###### Reading it:\n*Lower is better; near-flat means suppression is well tuned.*\n\n###### Healthy range:\n*Workload-dependent; a low, stable value is expected.*\n\n###### Watch for:\n*A rising trend, meaning suppression is too aggressive and the on-demand fetch path is growing.*\n\n###### Keywords:\n- **Reduce-relay** *(per node)* \u2014 an optimization that relays messages through selected peers only, suppressing redundant forwarding.\n- **Transaction suppression** *(per node)* \u2014 dropping a transaction already seen from another peer, so it is not reprocessed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (AppMetricGauges, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[AppMetricGauges.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/AppMetricGauges.cpp)\n\n###### Function:\n`registerReduceRelayGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#reduce-relay)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 23
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(reduce_relay_metrics{metric=\"missing_tx_freq\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Missing Tx Freq\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "short",
-          "custom": {
-            "axisLabel": "Frequency",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 9
-    }
-  ],
-  "schemaVersion": 39,
-  "tags": ["network", "peer"],
-  "templating": {
-    "list": [
-      {
-        "name": "DS_PROMETHEUS",
-        "type": "datasource",
-        "label": "Prometheus",
-        "query": "prometheus",
-        "regex": "",
-        "current": {},
-        "hide": 0,
-        "refresh": 1,
-        "includeAll": false,
-        "multi": false,
-        "options": []
-      },
-      {
-        "name": "service_name",
-        "label": "Service Name",
-        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
-        "type": "query",
-        "query": "label_values(service_name)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "xrpld",
-          "value": "xrpld"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "deployment_environment",
-        "label": "Environment",
-        "description": "Filter by deployment tier [local / ci / test / prod]",
-        "type": "query",
-        "query": "label_values(deployment_environment)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_network_type",
-        "label": "Network",
-        "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]",
-        "type": "query",
-        "query": "label_values(xrpl_network_type)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_work_item",
-        "label": "Work Item",
-        "description": "Filter by perf-iac work item / ticket",
-        "type": "query",
-        "query": "label_values(xrpl_work_item)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_branch",
-        "label": "Branch",
-        "description": "Filter by comparison side (baseline:<ref>:<commit> / test:<ref>:<commit>)",
-        "type": "query",
-        "query": "label_values(xrpl_branch)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_node_role",
-        "label": "Node Role",
-        "description": "Filter by node role (validator / peer)",
-        "type": "query",
-        "query": "label_values(xrpl_node_role)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "node",
-        "label": "Node",
-        "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)",
-        "type": "query",
-        "query": "label_values(target_info, service_instance_id)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "proposal_trusted",
-        "label": "Proposal Trusted",
-        "description": "Filter by proposal trust status (true = from trusted validator)",
-        "type": "query",
-        "query": "label_values(span_calls_total{span_name=\"peer.proposal.receive\"}, proposal_trusted)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "validation_trusted",
-        "label": "Validation Trusted",
-        "description": "Filter by validation trust status (true = from trusted validator)",
-        "type": "query",
-        "query": "label_values(span_calls_total{span_name=\"peer.validation.receive\"}, validation_trusted)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      }
-    ]
-  },
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "title": "Peer Network",
-  "uid": "peer-network",
-  "refresh": "30s"
-}

docker/telemetry/grafana/dashboards/rpc-pathfinding.json

@@ -1,947 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      },
-      {
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": false,
-        "iconColor": "rgb(70, 70, 70)",
-        "name": "Perf Runs (JMeter)",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": ["perf-iac", "jmeter"],
-          "type": "tags"
-        },
-        "type": "tags"
-      },
-      {
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": false,
-        "iconColor": "rgb(15, 122, 102)",
-        "name": "Perf Runs (Locust)",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": ["perf-iac", "locust"],
-          "type": "tags"
-        },
-        "type": "tags"
-      }
-    ]
-  },
-  "description": "What this shows: RPC request throughput and response timing, pathfinding cost, gRPC read traffic, and resource-manager enforcement for this node.  \u2014  Use it to: Watch client request load and cost, and confirm resource limits are protecting the node.",
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 1,
-  "id": null,
-  "links": [],
-  "panels": [
-    {
-      "title": "RPC Request Rate [$xrpl_network_type]",
-      "description": "###### What this is:\n*How many RPC requests the server counts per second.*\n\n###### How it's computed:\n*Per-second rate of the RPC request counter over the dashboard's rate interval, per node.*\n\n###### Reading it:\n*A steady line proportional to client demand; cross-checks the trace-based RPC dashboard.*\n\n###### Healthy range:\n*Workload-dependent; tracks client activity.*\n\n###### Watch for:\n*A sudden surge above baseline, consistent with a client flooding the RPC endpoint.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)",
-      "type": "stat",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 0
-      },
-      "options": {
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(rate(rpc_requests_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Requests / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "reqps"
-        },
-        "overrides": []
-      },
-      "repeat": "xrpl_network_type",
-      "repeatDirection": "h",
-      "maxPerRow": 2,
-      "id": 1
-    },
-    {
-      "title": "RPC Response",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 10
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 2
-    },
-    {
-      "title": "RPC Response Time",
-      "description": "###### What this is:\n*The 95th-percentile end-to-end RPC response time, including HTTP handling.*\n\n###### How it's computed:\n*95th-percentile of measured response times over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; this is broader than command-only latency.*\n\n###### Healthy range:\n*A few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A sustained climb, indicating handler overload or slow downstream work.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Response Time\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "ms",
-          "custom": {
-            "axisLabel": "Latency (ms)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 3
-    },
-    {
-      "title": "RPC Response Size",
-      "description": "###### What this is:\n*The 95th-percentile size of RPC response payloads in bytes.*\n\n###### How it's computed:\n*95th-percentile of response payload sizes over 5 minutes, per node. The instrument declares a byte unit, so it exports as `rpc_size_bytes_bucket` and is recorded on the byte bucket ladder (512 B to 1 MiB); the panel unit `decbytes` therefore shows a true size.*\n\n###### Reading it:\n*Smaller is cheaper; large responses cost bandwidth and memory.*\n\n###### Healthy range:\n*Workload-dependent; small for status queries, large for bulk data queries.*\n\n###### Watch for:\n*Growth in large responses, consistent with expensive queries or API misuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 11
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(rpc_size_bytes_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Response Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "decbytes",
-          "custom": {
-            "axisLabel": "Size (Bytes)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 4
-    },
-    {
-      "title": "RPC Response Time Distribution",
-      "description": "###### What this is:\n*RPC response time at the 90th, 95th, and 99th percentiles together.*\n\n###### How it's computed:\n*Three response-time quantiles over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; the gap between P90 and P99 shows the long tail.*\n\n###### Healthy range:\n*Percentiles clustered low; workload-dependent.*\n\n###### Watch for:\n*A widening P99, revealing long-tail or bimodal slow requests.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 21
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.9, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P90\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P99\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "ms",
-          "custom": {
-            "axisLabel": "Latency (ms)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 5
-    },
-    {
-      "title": "Pathfinding",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 31
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 6
-    },
-    {
-      "title": "Resource Warnings Rate [$xrpl_network_type]",
-      "description": "###### What this is:\n*How often the resource manager warns a peer or client for excessive usage, per second.*\n\n###### How it's computed:\n*Per-second rate of resource-warning events over the dashboard's rate interval, per node.*\n\n###### Reading it:\n*Green near zero, yellow above 0.1/sec, red above 1/sec.*\n\n###### Healthy range:\n*Near zero in normal operation.*\n\n###### Watch for:\n*A rising rate, consistent with aggressive clients that may need throttling.*\n\n###### Keywords:\n- **Resource drops / warnings** *(per node)* \u2014 the resource manager warning (then dropping/blocking) a peer or client for excessive usage.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/resource/detail/Logic.h)\n\n###### Function:\n`Logic::Stats ctor`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-drops-warnings)",
-      "type": "stat",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 32
-      },
-      "options": {
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(rate(warn_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.xrpl_ident}",
-          "unit": "si: warnings/s",
-          "decimals": 0,
-          "thresholds": {
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 0.1
-              },
-              {
-                "color": "red",
-                "value": 1
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "repeat": "xrpl_network_type",
-      "repeatDirection": "h",
-      "maxPerRow": 2,
-      "id": 7
-    },
-    {
-      "title": "Resource Drops Rate [$xrpl_network_type]",
-      "description": "###### What this is:\n*How often the resource manager drops or blocks a peer or client for abuse, per second.*\n\n###### How it's computed:\n*Per-second rate of resource-drop events over the dashboard's rate interval, per node.*\n\n###### Reading it:\n*Green near zero, yellow above 0.01/sec, red above 0.1/sec.*\n\n###### Healthy range:\n*Zero when no abusive consumers are present.*\n\n###### Watch for:\n*Non-zero values, meaning the node is actively rejecting abusive connections.*\n\n###### Keywords:\n- **Resource drops / warnings** *(per node)* \u2014 the resource manager warning (then dropping/blocking) a peer or client for excessive usage.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/resource/detail/Logic.h)\n\n###### Function:\n`Logic::Stats ctor`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-drops-warnings)",
-      "type": "stat",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 42
-      },
-      "options": {
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(rate(drop_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.xrpl_ident}",
-          "unit": "si: drops/s",
-          "decimals": 0,
-          "thresholds": {
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 0.01
-              },
-              {
-                "color": "red",
-                "value": 0.1
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "repeat": "xrpl_network_type",
-      "repeatDirection": "h",
-      "maxPerRow": 2,
-      "id": 8
-    },
-    {
-      "title": "Pathfinding Fast Duration",
-      "description": "###### What this is:\n*The 95th-percentile time of the fast pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of fast pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; fast mode trades accuracy for speed.*\n\n###### Healthy range:\n*Typically a few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A rising trend, indicating heavy pathfinding demand or an expensive order book.*\n\n###### Keywords:\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n- **Order book** *(network-wide)* \u2014 the ledger's list of standing offers to trade a currency pair on the decentralized exchange.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Order book](https://xrpl.org/docs/concepts/tokens/decentralized-exchange) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#pathfinding-fast-full)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 52
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Fast Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "ms",
-          "custom": {
-            "axisLabel": "Duration (ms)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 9
-    },
-    {
-      "title": "Pathfinding Full Duration",
-      "description": "###### What this is:\n*The 95th-percentile time of the full, exhaustive pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of full pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; full mode is much more expensive than fast mode.*\n\n###### Healthy range:\n*Tens to hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained high durations, consistent with pathfinding-heavy clients straining the node.*\n\n###### Keywords:\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#pathfinding-fast-full)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 52
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Full Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "ms",
-          "custom": {
-            "axisLabel": "Duration (ms)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 10
-    },
-    {
-      "title": "gRPC (Spans)",
-      "type": "row",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 62
-      },
-      "collapsed": false,
-      "panels": [],
-      "id": 11
-    },
-    {
-      "title": "gRPC Request Rate by Method (Spans)",
-      "description": "###### What this is:\n*How many gRPC calls of each method the node serves per second.*\n\n###### How it's computed:\n*Per-second rate of gRPC calls over the dashboard's rate interval, grouped by method and node.*\n\n###### Reading it:\n*Non-zero only when reporting/Clio-style clients use the gRPC API.*\n\n###### Healthy range:\n*Workload-dependent; zero without gRPC traffic.*\n\n###### Watch for:\n*A single method spiking, consistent with a heavy or misbehaving gRPC consumer.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n- **Clio / reporting client** *(per node)* \u2014 an external read-scaling service (Clio) that serves history/API queries, often via the gRPC interface.\n- **gRPC** *(per node)* \u2014 a binary RPC interface used mainly by reporting/Clio-style clients, separate from HTTP/WebSocket.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[GRPCServer.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/GRPCServer.cpp)\n\n###### Function:\n`GRPCServerImpl::CallData::process`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 63
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (method, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$grpc_method\", span_name=~\"grpc\\\\..*\"}[$__rate_interval])), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "si: calls/s",
-          "decimals": 0,
-          "custom": {
-            "axisLabel": "Calls / Sec",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 12
-    },
-    {
-      "title": "gRPC Latency P95 by Method (Spans)",
-      "description": "###### What this is:\n*The 95th-percentile latency of each gRPC method.*\n\n###### How it's computed:\n*95th-percentile of gRPC call durations over 5 minutes, grouped by method and node.*\n\n###### Reading it:\n*Lower is better; identifies slow gRPC read paths.*\n\n###### Healthy range:\n*Workload-dependent; scales with ledger data volume served.*\n\n###### Watch for:\n*A method whose latency climbs, indicating an expensive read path.*\n\n###### Keywords:\n- **gRPC** *(per node)* \u2014 a binary RPC interface used mainly by reporting/Clio-style clients, separate from HTTP/WebSocket.\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[GRPCServer.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/GRPCServer.cpp)\n\n###### Function:\n`GRPCServerImpl::CallData::process`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#grpc)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 63
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, method, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$grpc_method\", span_name=~\"grpc\\\\..*\"}[5m]))), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "ms",
-          "custom": {
-            "axisLabel": "Duration (ms)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 13
-    },
-    {
-      "title": "gRPC Error Rate by Status (Spans)",
-      "description": "###### What this is:\n*gRPC call rate broken down by outcome status (success or error).*\n\n###### How it's computed:\n*Per-second rate of gRPC calls over the dashboard's rate interval, grouped by status and node.*\n\n###### Reading it:\n*Success should dominate; the error rate should stay low.*\n\n###### Healthy range:\n*Workload-dependent; errors near the floor.*\n\n###### Watch for:\n*A rising error rate, indicating gRPC clients hitting failures.*\n\n###### Keywords:\n- **gRPC** *(per node)* \u2014 a binary RPC interface used mainly by reporting/Clio-style clients, separate from HTTP/WebSocket.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[GRPCServer.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/GRPCServer.cpp)\n\n###### Function:\n`GRPCServerImpl::CallData::process`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#grpc)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 73
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (grpc_status, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=~\"grpc\\\\..*\", grpc_status!=\"\"}[$__rate_interval])), \"series\", \"$1\", \"grpc_status\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "si: calls/s",
-          "decimals": 0,
-          "custom": {
-            "axisLabel": "Calls / Sec",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 14
-    },
-    {
-      "title": "Pathfinding Compute Duration (Spans)",
-      "description": "###### What this is:\n*The 95th-percentile time to compute paths for a single request.*\n\n###### How it's computed:\n*95th-percentile of path-computation durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; complements the fast/full pathfinding timers with span-level detail.*\n\n###### Healthy range:\n*Workload-dependent; grows with order-book depth and request complexity.*\n\n###### Watch for:\n*A rising trend under pathfinding load, indicating expensive path computation.*\n\n###### Keywords:\n- **Batch vs single RPC** *(per node)* \u2014 whether an RPC request carried several commands (batch) or one (single); batch is used by bulk clients.\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n- **Order book** *(network-wide)* \u2014 the ledger's list of standing offers to trade a currency pair on the decentralized exchange.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PathRequest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequest.cpp)\n\n###### Function:\n`PathRequest::doUpdate`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Order book](https://xrpl.org/docs/concepts/tokens/decentralized-exchange) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#batch-vs-single-rpc)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 73
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"pathfind.compute\"}[5m]))), \"series\", \"P95 Compute\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "ms",
-          "custom": {
-            "axisLabel": "Duration (ms)",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 15
-    },
-    {
-      "title": "Pathfinding Request & Discovery Rate (Spans)",
-      "description": "###### What this is:\n*The rate of client path requests versus path-discovery passes.*\n\n###### How it's computed:\n*Per-second rates of path requests and discovery passes over the dashboard's rate interval, per node.*\n\n###### Reading it:\n*Discovery cost tracks request demand; useful for subscription-heavy nodes.*\n\n###### Healthy range:\n*Workload-dependent; both zero without pathfinding clients.*\n\n###### Watch for:\n*Discovery rate climbing out of proportion to requests, a cost driver for subscription-heavy nodes.*\n\n###### Keywords:\n- **Path request / discovery** *(per node)* \u2014 a client's ongoing pathfinding subscription (request) and the periodic path-refresh passes (discovery).\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PathFind.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/handlers/orderbook/PathFind.cpp) \u00b7 [PathRequest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequest.cpp)\n\n###### Function:\n`doPathFind ; PathRequest::findPaths`\n\n###### References:\n[Path request / discovery](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods/path-and-order-book-methods/path_find) \u00b7 [Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#path-request-discovery)",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 0,
-        "y": 83
-      },
-      "options": {
-        "annotations": {
-          "clustering": -1,
-          "multiLane": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"pathfind.request\"}[$__rate_interval])), \"series\", \"Requests / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"pathfind.discover\"}[$__rate_interval])), \"series\", \"Discoveries / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "si: operations/s",
-          "decimals": 0,
-          "custom": {
-            "axisLabel": "Operations / Sec",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 5,
-            "lineWidth": 1,
-            "fillOpacity": 0,
-            "gradientMode": "none"
-          }
-        },
-        "overrides": []
-      },
-      "id": 16
-    }
-  ],
-  "schemaVersion": 39,
-  "tags": ["rpc", "pathfinding"],
-  "templating": {
-    "list": [
-      {
-        "name": "DS_PROMETHEUS",
-        "type": "datasource",
-        "label": "Prometheus",
-        "query": "prometheus",
-        "regex": "",
-        "current": {},
-        "hide": 0,
-        "refresh": 1,
-        "includeAll": false,
-        "multi": false,
-        "options": []
-      },
-      {
-        "name": "service_name",
-        "label": "Service Name",
-        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
-        "type": "query",
-        "query": "label_values(service_name)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "xrpld",
-          "value": "xrpld"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "deployment_environment",
-        "label": "Environment",
-        "description": "Filter by deployment tier [local / ci / test / prod]",
-        "type": "query",
-        "query": "label_values(deployment_environment)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_network_type",
-        "label": "Network",
-        "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]",
-        "type": "query",
-        "query": "label_values(xrpl_network_type)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_work_item",
-        "label": "Work Item",
-        "description": "Filter by perf-iac work item / ticket",
-        "type": "query",
-        "query": "label_values(xrpl_work_item)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_branch",
-        "label": "Branch",
-        "description": "Filter by comparison side (baseline:<ref>:<commit> / test:<ref>:<commit>)",
-        "type": "query",
-        "query": "label_values(xrpl_branch)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "xrpl_node_role",
-        "label": "Node Role",
-        "description": "Filter by node role (validator / peer)",
-        "type": "query",
-        "query": "label_values(xrpl_node_role)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "node",
-        "label": "Node",
-        "description": "Filter by rippled node (service.instance.id)",
-        "type": "query",
-        "query": "label_values(target_info, service_instance_id)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      },
-      {
-        "name": "grpc_method",
-        "label": "gRPC Method",
-        "description": "Filter by gRPC method (GetLedger, GetLedgerData, GetLedgerDiff, GetLedgerEntry)",
-        "type": "query",
-        "query": "label_values(span_calls_total{span_name=~\"grpc\\\\..*\"}, method)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "includeAll": true,
-        "allValue": ".*",
-        "current": {
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "refresh": 2,
-        "sort": 1
-      }
-    ]
-  },
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "title": "RPC & Pathfinding",
-  "uid": "rpc-pathfinding",
-  "refresh": "30s"
-}

docker/telemetry/grafana/dashboards/validate_dashboards.py

@@ -1,196 +0,0 @@
-#!/usr/bin/env python3
-"""Dashboard lint: cumulative metrics rate()-wrapped; tier filters; sane panel grid."""
-
-import json, re, sys
-from pathlib import Path
-
-GRID_COLUMNS = 24
-
-# Prometheus gauges that hold a CUMULATIVE total -> must be rate()/increase()-wrapped.
-CUMULATIVE_PREFIXES = (
-    "total_bytes_",
-    "total_messages_",
-    "transactions_messages_",
-    "transactions_duplicate_messages_",
-    "proposals_",
-    "validations_",
-    "overlay_peer_disconnects",
-    "squelch_",
-    "overhead_",
-    "validator_lists_",
-    "set_get_",
-    "set_share_",
-    "have_transactions_",
-    "requested_transactions_",
-    "proof_path_",
-    "replay_delta_",
-    "ledger_data_",
-    "ledger_get_",
-    "ledger_share_",
-    "ledger_transaction_",
-    "ledger_account_state_",
-    "getobject_",
-    "jq_trans_overflow_total",
-)
-# nodestore_state{metric=...} cumulative sub-series (raw only inside these metric= labels).
-NODESTORE_CUMULATIVE = (
-    "node_reads_total",
-    "node_reads_hit",
-    "node_writes",
-    "node_read_bytes",
-    "node_written_bytes",
-    "node_reads_duration_us",
-    "node_writes_duration_us",
-)
-# state_accounting_*_duration are cumulative µs.
-STATE_DURATION = re.compile(r"state_accounting_\w+_duration")
-REQUIRED_FILTERS = (
-    "$node",
-    "$deployment_environment",
-    "$xrpl_network_type",
-    "$service_name",
-)
-
-
-def iter_panels(dash):
-    for p in dash.get("panels", []):
-        yield p
-        for sub in p.get("panels", []) or []:
-            yield sub
-
-
-def check_layout(path, dash):
-    """Duplicate panel ids and grid collisions -- both break Grafana's loader.
-
-    Grafana keys panels by id when it builds the dashboard model, so two panels
-    sharing an id make the load non-deterministic. Overlapping gridPos rectangles
-    have no valid layout. Only top-level panels are checked: panels nested inside
-    a collapsed row do not occupy the outer grid.
-    """
-    errs = []
-    top = dash.get("panels", []) or []
-
-    seen_ids = {}
-    for p in top:
-        pid = p.get("id")
-        if pid is None:
-            continue
-        if pid in seen_ids:
-            errs.append(
-                f"{path}: duplicate panel id {pid}: "
-                f"[{seen_ids[pid]}] and [{p.get('title', '<untitled>')}]"
-            )
-        else:
-            seen_ids[pid] = p.get("title", "<untitled>")
-
-    # Mark every grid cell each panel covers; a second claim on a cell is a collision.
-    owner = {}
-    for p in top:
-        g = p.get("gridPos") or {}
-        x, y = g.get("x", 0), g.get("y", 0)
-        w, h = g.get("w", 0), g.get("h", 0)
-        title = p.get("title", "<untitled>")
-        if x + w > GRID_COLUMNS:
-            errs.append(
-                f"{path} [{title}]: spans past the {GRID_COLUMNS}-column grid (x={x}, w={w})"
-            )
-        clashed = set()
-        for cy in range(y, y + h):
-            for cx in range(x, min(x + w, GRID_COLUMNS)):
-                prev = owner.get((cy, cx))
-                if prev is None:
-                    owner[(cy, cx)] = title
-                elif prev not in clashed:
-                    clashed.add(prev)
-                    errs.append(
-                        f"{path} [{title}]: grid overlap with [{prev}] at y={cy} x={cx}"
-                    )
-    return errs
-
-
-def expr_is_wrapped(expr):
-    return (
-        "rate(" in expr
-        or "increase(" in expr
-        or "irate(" in expr
-        or "histogram_quantile(" in expr
-    )
-
-
-def check(path, forbid_5m):
-    errs = []
-    try:
-        dash = json.load(open(path))
-    except Exception as e:
-        return [f"{path}: INVALID JSON: {e}"]
-    errs += check_layout(path, dash)
-    for p in iter_panels(dash):
-        title = p.get("title", "<untitled>")
-        for tg in p.get("targets", []) or []:
-            expr = (tg.get("expr") or "").strip()
-            if not expr:
-                continue
-            # tier filters
-            if "{" in expr:
-                for f in REQUIRED_FILTERS:
-                    if f not in expr:
-                        errs.append(f"{path} [{title}]: missing {f} in expr")
-            # cumulative metric plotted raw?
-            hits = [m for m in CUMULATIVE_PREFIXES if m in expr]
-            nod = [m for m in NODESTORE_CUMULATIVE if f'"{m}"' in expr]
-            statedur = STATE_DURATION.search(expr)
-            if (hits or nod or statedur) and not expr_is_wrapped(expr):
-                who = hits or nod or [statedur.group(0)]
-                errs.append(
-                    f"{path} [{title}]: cumulative metric {who} plotted RAW (needs rate/increase)"
-                )
-            if "deriv(" in expr or "idelta(" in expr:
-                errs.append(
-                    f"{path} [{title}]: uses deriv()/idelta() on a cumulative series"
-                )
-            if forbid_5m and ("[5m]" in expr or "[1h]" in expr):
-                # Only a fixed [5m]/[1h] inside a top-level rate()/increase()/irate() on a
-                # plain counter is a violation. Legitimate fixed-window exprs -- histogram
-                # buckets (rate(..._bucket[5m])), avg_over_time windows, and subqueries
-                # ([5m:]) -- are intentional and must not be flagged.
-                if not ("_bucket" in expr or "avg_over_time" in expr or ":]" in expr):
-                    errs.append(
-                        f"{path} [{title}]: hardcoded range window; use [$__rate_interval]"
-                    )
-        # unit check
-        unit = p.get("fieldConfig", {}).get("defaults", {}).get("unit", "")
-        if unit == "mps":
-            errs.append(f"{path} [{title}]: invalid Grafana unit 'mps' (use 'cps')")
-    return errs
-
-
-def main():
-    """Validate the dashboards named on the command line, or all of them.
-
-    Exit status: 0 every dashboard passed, 1 violations were found, 2 there was
-    nothing to validate. The last is an error rather than a pass: reporting
-    success without having read a single dashboard is indistinguishable from a
-    clean run, so a caller that mis-spells a path gets a green light for work
-    that never happened.
-    """
-    args = [a for a in sys.argv[1:] if not a.startswith("--")]
-    forbid_5m = "--no-5m" in sys.argv
-    if not args:
-        # Default to every dashboard beside this script, so a bare run checks
-        # the whole set instead of iterating an empty list.
-        args = sorted(str(p) for p in Path(__file__).resolve().parent.glob("*.json"))
-    if not args:
-        print("ERROR: no dashboard JSON files to validate", file=sys.stderr)
-        sys.exit(2)
-    all_errs = []
-    for path in args:
-        all_errs += check(path, forbid_5m)
-    if all_errs:
-        print("\n".join(all_errs))
-        print(f"\nFAIL: {len(all_errs)} violation(s)")
-        sys.exit(1)
-    print(f"OK: {len(args)} dashboard(s) passed")
-
-
-if __name__ == "__main__":
-    main()

docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml

@@ -1,112 +0,0 @@
-# Grafana contact-point provisioning for rippled OTel alerts.
-#
-# Alerting on health-critical internal metrics.
-#
-# A contact point is where a firing alert is delivered. Two are defined:
-#   xrpld-default   — Slack only; receives warning-severity alerts.
-#   xrpld-critical  — Slack + email; receives critical-severity alerts.
-# The severity split is wired in policies.yaml.
-#
-# ---------------------------------------------------------------------------
-# Supplying a real destination
-# ---------------------------------------------------------------------------
-# The values below are deliberately non-empty, invalid-but-valid-shaped
-# placeholders (an unroutable webhook host and a .invalid email). Grafana's
-# provisioning validator REQUIRES a non-empty Slack url and a non-empty email
-# `addresses`, so the placeholders keep the whole stack booting with zero
-# configuration; alerts simply route nowhere until a destination is supplied.
-#
-# To enable delivery, edit the two placeholder values IN PLACE with a real
-# webhook / address. Do NOT commit the result. Do not put
-# ${SLACK_WEBHOOK_URL} or any other variable in either placeholder:
-# Grafana expands ${VAR} but does NOT support ${VAR:-default}, so an unset
-# variable expands to empty, fails validation, and Grafana exits 1 — taking
-# the whole telemetry stack down, not just alerting.
-#
-# Two further traps when editing this file:
-#
-#   * Never leave a `receivers:` list empty to "disable" a tier. A contact
-#     point with no receivers ceases to exist, the policy tree then references
-#     a missing receiver, and Grafana refuses to boot.
-#
-#   * File provisioning is upsert-only. Deleting a receiver here does NOT
-#     remove it from an instance that already booted with it — the old
-#     receiver keeps delivering. Removal requires an explicit
-#     `deleteContactPoints:` block (see the commented example at the bottom).
-#
-# Email delivery additionally requires SMTP configured on the Grafana service.
-# The GF_SMTP_* variables live in docker/telemetry/.env.alerting (gitignored,
-# see .env.alerting.example) and reach Grafana via the compose `env_file`;
-# they are NOT set in docker-compose.yml. Email stays off until
-# GF_SMTP_ENABLED=true and the relay settings point somewhere real.
-#
-# Grafana Cloud does NOT read this file: Cloud has no provisioning
-# filesystem. Create the Cloud contact points and notification templates
-# through its API or UI, and keep them in step with this file and
-# templates.yaml by hand. Do not push a notification policy tree to a Cloud
-# stack: the PUT endpoint replaces the org's default policy tree wholesale.
-
-apiVersion: 1
-
-contactPoints:
-  # --- Warning tier: Slack only ---
-  - orgId: 1
-    name: xrpld-default
-    receivers:
-      - uid: xrpld-slack-default
-        type: slack
-        settings:
-          # Disabled placeholder: an unroutable webhook host. A non-empty url
-          # selects Slack webhook mode (no recipient/token required) and keeps
-          # provisioning valid. Replace with a real webhook to enable delivery.
-          url: https://hooks.slack.invalid/disabled
-          # Title and body both come from templates.yaml. `alertname` is the
-          # label Grafana sets on every alert; `service_instance_id` is a query
-          # label and is absent on NoData/Error evaluations, so the templates
-          # guard it rather than referencing it bare.
-          title: '{{ template "xrpld.title" . }}'
-          text: '{{ template "xrpld.slack.body" . }}'
-        disableResolveMessage: false
-
-  # --- Critical tier: Slack + email ---
-  - orgId: 1
-    name: xrpld-critical
-    receivers:
-      - uid: xrpld-slack-critical
-        type: slack
-        settings:
-          # Disabled placeholder — see xrpld-slack-default above.
-          url: https://hooks.slack.invalid/disabled
-          # The title template already carries the severity label, so this tier
-          # needs no separate "[CRITICAL]" prefix.
-          title: '{{ template "xrpld.title" . }}'
-          text: '{{ template "xrpld.slack.body" . }}'
-        disableResolveMessage: false
-      - uid: xrpld-email-critical
-        type: email
-        settings:
-          # Disabled placeholder: a .invalid address keeps the required
-          # `addresses` field non-empty so provisioning validates. Replace
-          # with a real address (and enable SMTP) to deliver. Multiple
-          # recipients are semicolon- or comma-separated.
-          addresses: alerts-disabled@xrpld.invalid
-          # One message listing all recipients, rather than one message each.
-          singleEmail: true
-          # DELIBERATELY no `subject`/`message` override here, unlike the Slack
-          # receivers above. Grafana's default email body is the good one: bold
-          # section headings, a styled label table, annotations rendered as real
-          # clickable links, and the View/Silence buttons. Overriding `message`
-          # replaces all of that with plain text, because email escapes HTML, so
-          # a custom body cannot reproduce any of it. The value line the default
-          # body carries is made readable by NAMING each rule's query steps (see
-          # rules.yaml), not by replacing the body.
-        disableResolveMessage: false
-
-# To retire a receiver that a running Grafana has already stored, uncomment
-# and list its uid here — deleting the block above is not sufficient:
-#
-# deleteContactPoints:
-#   - orgId: 1
-#     uid: xrpld-slack-default
-#   - orgId: 1
-#     uid: xrpld-slack-critical

docker/telemetry/grafana/provisioning/alerting/policies.yaml

@@ -1,44 +0,0 @@
-# Grafana notification-policy provisioning for rippled OTel alerts.
-#
-# Alerting on health-critical internal metrics.
-#
-# The notification policy tree decides which contact point receives a firing
-# alert and how alerts are batched. Routing is split by severity:
-#   - default (root)        → xrpld-default  (Slack) — catches warnings.
-#   - severity = critical   → xrpld-critical (Slack + email) — child route.
-# A critical alert matches the child route and is delivered to Slack AND email;
-# everything else falls through to the root receiver.
-#
-# Grouping by alertname + service_instance_id means one notification per
-# (alert, node) pair, so a fleet-wide problem does not collapse into a single
-# ambiguous message.
-
-apiVersion: 1
-
-policies:
-  - orgId: 1
-    receiver: xrpld-default
-    group_by:
-      - alertname
-      - service_instance_id
-    # Wait before sending the first notification for a new group, so a burst of
-    # related series is batched together.
-    group_wait: 30s
-    # Wait before adding new alerts to an already-notified group.
-    group_interval: 5m
-    # Minimum time before re-sending a notification for an alert that is still
-    # firing.
-    repeat_interval: 4h
-    routes:
-      # Critical alerts page harder: Slack + email. matchers uses the
-      # PromQL-style label match "severity = critical" set on the rules.
-      - receiver: xrpld-critical
-        matchers:
-          - severity = critical
-        group_by:
-          - alertname
-          - service_instance_id
-        group_wait: 30s
-        group_interval: 5m
-        # Re-page critical alerts more often than the 4h default while still firing.
-        repeat_interval: 1h

docker/telemetry/grafana/provisioning/alerting/templates.yaml

@@ -1,96 +0,0 @@
-# Notification templates for the xrpld OTel alerts.
-#
-# Why these exist: Grafana's built-in message body appends a raw dump of every
-# expression node's value and every label, so a notification reads
-# `Value: A=0, B=0, C=1` followed by five `key = value` lines. The refIds carry
-# no meaning to a reader, and the label dump repeats what the title already
-# says. These templates replace that body with the rule's own prose.
-#
-# ---------------------------------------------------------------------------
-# SLACK ONLY. Email deliberately has no template here.
-# ---------------------------------------------------------------------------
-# Measured, and the reason this file covers one channel rather than two:
-#
-#   * Email ESCAPES any HTML in the message body, so a literal `<br>` arrives
-#     as `&lt;br&gt;` and no tag or anchor is possible. A custom email body is
-#     therefore plain text and cannot reproduce Grafana's default layout, which
-#     already gives section headings, a label table, clickable annotation links
-#     and the View/Silence buttons. So email keeps the default body.
-#   * Slack markup is meaningless in email anyway: bold asterisks, backticks
-#     and `:rotating_light:` arrive as literal characters, and a `<url|label>`
-#     link cannot become a link, so it dumps the whole URL inline. Four panel
-#     links bury the prose in a wall of text. That is what a shared body did.
-#   * Real emoji CHARACTERS render in both channels; the `:shortcode:` form
-#     only works in Slack.
-#
-# The value line Grafana's default email body ends with is made readable by
-# NAMING each rule's query steps (see rules.yaml), not by replacing the body.
-#
-# ---------------------------------------------------------------------------
-# What a template may and may not use
-# ---------------------------------------------------------------------------
-# The function set is much smaller than Go's text/template plus sprig.
-# Available and used below: `match` (regexp), `reReplaceAll`, `title`, `printf`,
-# `len`, `index`, `eq`. NOT available — each fails the whole template with
-# `function "X" not defined`: `hasPrefix`, `hasSuffix`, `contains`, `humanize`,
-# `humanizeDuration`, `humanizePercentage`.
-#
-# A template failure is nearly silent, which is the trap. Measured: a missing
-# define logs one `failed to template email message` warning and then delivers
-# Grafana's DEFAULT body — the value dump and label list this file exists to
-# remove — while the rule still reports `health=ok`. So a change here is only
-# verified by reading a real delivered notification, never by rule state, and
-# never by the absence of an obvious error.
-#
-# ---------------------------------------------------------------------------
-# Two label traps
-# ---------------------------------------------------------------------------
-#   * `service_instance_id` is a QUERY label, so it is absent whenever Grafana
-#     evaluates NoData or Error — those evaluations carry only
-#     {datasource_uid, ref_id}. Two rules set `noDataState: Alerting` and so
-#     fire that way deliberately. `xrpld.node` below falls back rather than
-#     rendering an empty string.
-#   * `alertname` is set by Grafana on every alert. There is no `rulename`
-#     label, so referencing one renders blank.
-#
-# Values are deliberately NOT printed. Each rule's `description` already states
-# its own measurement, formatted and with its threshold, which is the readable
-# form of what the raw value dump was trying to say.
-
-apiVersion: 1
-
-templates:
-  - orgId: 1
-    name: xrpld.notifications
-    template: |
-      {{- /* Node identity, or a marker when the query label is absent. */ -}}
-      {{ define "xrpld.node" -}}
-      {{ if .Labels.service_instance_id }}{{ .Labels.service_instance_id }}{{ else }}unknown node (NoData/Error evaluation){{ end }}
-      {{- end }}
-
-      {{- /* One line. Slack title and email subject share this. */ -}}
-      {{ define "xrpld.title" -}}
-      {{ if .Alerts.Firing }}🔥 FIRING{{ else }}✅ RESOLVED{{ end }}
-      {{- with .CommonLabels.alertname }} · {{ . }}{{ end }}
-      {{- with .CommonLabels.severity }} ({{ . }}){{ end }}
-      {{- with .CommonLabels.service_instance_id }} · {{ . }}{{ end }}
-      {{- end }}
-
-      {{- /* SLACK body: mrkdwn, emoji shortcodes, <url|label> links. */ -}}
-      {{ define "xrpld.slack.body" -}}
-      {{ with .Alerts.Firing }}{{ range . }}
-      :rotating_light: *FIRING* · *{{ .Labels.alertname }}* on `{{ template "xrpld.node" . }}`
-      {{ .Annotations.description }}
-      {{- with .Annotations.action }}
-      *Why it matters:* {{ . }}
-      {{- end }}
-      {{- range $key, $url := .Annotations }}
-      {{- if match "^panel_" $key }}
-      :chart_with_upwards_trend: <{{ $url }}|{{ title (reReplaceAll "_" " " (reReplaceAll "^panel_" "" $key)) }}>
-      {{- end }}{{ end }}
-      {{ end }}{{ end }}
-      {{- with .Alerts.Resolved }}{{ range . }}
-      :white_check_mark: *RESOLVED* · *{{ .Labels.alertname }}* on `{{ template "xrpld.node" . }}`
-      {{ .Annotations.description }}
-      {{ end }}{{ end }}
-      {{- end }}

docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml

@@ -1,12 +0,0 @@
-apiVersion: 1
-
-providers:
-  - name: xrpld-telemetry
-    orgId: 1
-    folder: xrpld
-    type: file
-    disableDeletion: false
-    editable: true
-    options:
-      path: /var/lib/grafana/dashboards
-      foldersFromFilesStructure: false

docker/telemetry/grafana/provisioning/datasources/loki.yaml

@@ -1,35 +0,0 @@
-# Grafana Loki data source provisioning for rippled log-trace correlation.
-#
-# Loki ingests rippled logs via OTel Collector's file_log receiver.
-# The derivedFields config links trace_id values in log lines back to
-# Tempo traces, enabling one-click log-to-trace navigation in Grafana.
-
-apiVersion: 1
-
-datasources:
-  - name: Loki
-    type: loki
-    access: proxy
-    url: http://loki:3100
-    uid: loki
-    jsonData:
-      derivedFields:
-        # Trace ID is an OTel-issued 32-hex value emitted by Logs::format()
-        # as `trace_id=...`. Grafana treats the captured value as a Tempo
-        # trace ID, opening the trace directly via Tempo's trace-by-id API.
-        - datasourceUid: tempo
-          matcherRegex: "trace_id=(\\w+)"
-          name: TraceID
-          url: "$${__value.raw}"
-        # 64-char uppercase hex tokens in log bodies are XRPL ledger
-        # hashes (or tx hashes). They are NOT OTel trace IDs and cannot
-        # be resolved via the trace-by-id endpoint. Build a Grafana
-        # Explore deep-link that runs TraceQL on the span attribute
-        # `consensus_ledger_id` — set on `consensus.round` spans to
-        # the full prev_ledger hash. This finds the round span whose
-        # first 16 bytes were folded into the OTel trace_id by
-        # SpanGuard::hashSpan().
-        - matcherRegex: "\\b([A-F0-9]{64})\\b"
-          name: ConsensusLedgerHash
-          urlDisplayLabel: "Search Tempo"
-          url: '/explore?schemaVersion=1&orgId=1&panes={"tempo":{"datasource":"tempo","queries":[{"refId":"A","queryType":"traceql","query":"{span.consensus_ledger_id=\"$${__value.raw}\"}"}],"range":{"from":"now-1h","to":"now"}}}'

docker/telemetry/grafana/provisioning/datasources/prometheus.yaml

@@ -1,10 +0,0 @@
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    uid: prometheus
-    access: proxy
-    url: http://prometheus:9090
-    isDefault: true
-    editable: true

docker/telemetry/grafana/provisioning/datasources/tempo.yaml

@@ -1,282 +0,0 @@
-# Grafana datasource provisioning for Grafana Tempo.
-# Auto-configures Tempo as a trace data source on Grafana startup.
-# Access Grafana at http://localhost:3000, then use Explore -> Tempo
-# to browse xrpld traces using TraceQL.
-#
-# Search filters provide pre-configured dropdowns in the Explore UI.
-# Each phase adds filters for the span attributes it introduces.
-# Base filters — node identity, service, span name, status.
-# RPC command, status, role filters.
-# Path-finding request, mode and ledger filters.
-# Transaction hash, local/peer origin, status.
-# Consensus mode, round, ledger sequence, close time.
-
-apiVersion: 1
-
-datasources:
-  - name: Tempo
-    type: tempo
-    access: proxy
-    url: http://tempo:3200
-    uid: tempo
-    jsonData:
-      nodeGraph:
-        enabled: true
-      # Service map and traces-to-metrics require a Prometheus datasource
-      # (not included in this stack). These features are inactive until a
-      # Prometheus service is added to docker-compose.yml.
-      serviceMap:
-        datasourceUid: prometheus
-      # Trace-to-log correlation — enables one-click navigation
-      # from a Tempo trace to the corresponding Loki log lines. Filters
-      # by trace_id so only logs from the same trace are shown.
-      tracesToLogs:
-        datasourceUid: loki
-        filterByTraceID: true
-        filterBySpanID: false
-        tags: []
-      tracesToMetrics:
-        datasourceUid: prometheus
-        spanStartTimeShift: "-1h"
-        spanEndTimeShift: "1h"
-      search:
-        filters:
-          # --- Node identification filters ---
-          # service.name: logical service name (default: "xrpld").
-          #   Useful when running multiple service types in the same collector.
-          - id: service-name
-            tag: service.name
-            operator: "="
-            scope: resource
-            type: static
-          # service.instance.id: unique node identifier — configurable via
-          #   the service_instance_id setting in [telemetry], defaults to the
-          #   node's public key. E.g. "Node-1" or "nHB1X37...".
-          - id: node-id
-            tag: service.instance.id
-            operator: "="
-            scope: resource
-            type: dynamic
-          # service.version: xrpld build version (e.g., "2.4.0-b1").
-          #   Filter traces from specific software releases.
-          - id: node-version
-            tag: service.version
-            operator: "="
-            scope: resource
-            type: dynamic
-          # xrpl.network.id: numeric network identifier
-          #   (0 = mainnet, 1 = testnet, 2 = devnet, etc.).
-          #   Derived from the [network_id] config section.
-          - id: network-id
-            tag: xrpl.network.id
-            operator: "="
-            scope: resource
-            type: dynamic
-          # xrpl.network.type: human-readable network name derived from
-          #   network ID ("mainnet", "testnet", "devnet", "unknown").
-          - id: network-type
-            tag: xrpl.network.type
-            operator: "="
-            scope: resource
-            type: dynamic
-          # --- Span intrinsic filters ---
-          # name: the span operation name (e.g., "rpc.command.server_info").
-          #   Use to find traces for a specific RPC command or subsystem.
-          - id: span-name
-            tag: name
-            operator: "="
-            scope: intrinsic
-            type: dynamic
-          # status: span completion status ("ok", "error", "unset").
-          #   Filter for failed operations to diagnose errors.
-          - id: span-status
-            tag: status
-            operator: "="
-            scope: intrinsic
-            type: dynamic
-          # duration: span wall-clock duration. Use with ">" operator
-          #   to find slow operations (e.g., duration > 500ms).
-          - id: span-duration
-            tag: duration
-            operator: ">"
-            scope: intrinsic
-            type: dynamic
-          # RPC tracing filters
-          - id: rpc-command
-            tag: command
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: rpc-status
-            tag: rpc_status
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: rpc-role
-            tag: rpc_role
-            operator: "="
-            scope: span
-            type: dynamic
-          # Path-finding filters. Only the attributes that select a request get
-          # a dropdown; pathfind_num_paths, pathfind_num_requests and
-          # pathfind_num_source_assets are measurements read off a span, not
-          # things an operator searches by.
-          - id: pathfind-source-account
-            tag: pathfind_source_account
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: pathfind-dest-account
-            tag: pathfind_dest_account
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: pathfind-dest-currency
-            tag: pathfind_dest_currency
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: pathfind-fast
-            tag: pathfind_fast
-            operator: "="
-            scope: span
-            type: dynamic
-          # Changes every ledger, so it must be dynamic rather than static.
-          - id: pathfind-ledger-index
-            tag: pathfind_ledger_index
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: pathfind-search-level
-            tag: pathfind_search_level
-            operator: ">"
-            scope: span
-            type: dynamic
-          # Transaction tracing filters
-          - id: tx-hash
-            tag: tx_hash
-            operator: "="
-            scope: span
-            type: static
-          # One filter per account role a transaction names. tx_account is on
-          # every transaction; the rest appear only on the types that carry
-          # that field, so they are dynamic.
-          - id: tx-account
-            tag: tx_account
-            operator: "="
-            scope: span
-            type: static
-          - id: tx-destination
-            tag: tx_destination
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: tx-owner
-            tag: tx_owner
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: tx-issuer
-            tag: tx_issuer
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: tx-origin
-            tag: local
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: tx-status
-            tag: tx_status
-            operator: "="
-            scope: span
-            type: dynamic
-          # tx_type: transaction type (e.g., "Payment", "OfferCreate").
-          - id: tx-type
-            tag: tx_type
-            operator: "="
-            scope: span
-            type: dynamic
-          # Consensus tracing filters
-          - id: consensus-mode
-            tag: consensus_mode
-            operator: "="
-            scope: span
-            type: static
-          - id: consensus-round
-            tag: consensus_round
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: consensus-ledger-seq
-            tag: ledger_seq
-            operator: "="
-            scope: span
-            type: static
-          # ledger_hash: ledger hash — scope all spans to a specific closed ledger.
-          - id: ledger-hash
-            tag: ledger_hash
-            operator: "="
-            scope: span
-            type: static
-          - id: consensus-close-time-correct
-            tag: close_time_correct
-            operator: "="
-            scope: span
-            type: static
-          - id: consensus-state
-            tag: consensus_state
-            operator: "="
-            scope: span
-            type: static
-          - id: consensus-close-resolution
-            tag: close_resolution_ms
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: consensus-proposers
-            tag: proposers
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: consensus-result
-            tag: consensus_result
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: consensus-mode-old
-            tag: mode_old
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: consensus-mode-new
-            tag: mode_new
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: consensus-ledger-id
-            tag: consensus_ledger_id
-            operator: "="
-            scope: span
-            type: static
-          # Additional transaction and queue filters
-          - id: tx-path
-            tag: path
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: peer-version
-            tag: peer_version
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: txq-status
-            tag: txq_status
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: txq-ter-code
-            tag: ter_code
-            operator: "="
-            scope: span
-            type: dynamic

docker/telemetry/integration-test.sh

@@ -1,903 +0,0 @@
-#!/usr/bin/env bash
-# Integration test for rippled OpenTelemetry instrumentation.
-#
-# Launches a 6-node xrpld consensus network with telemetry enabled,
-# exercises RPC / transaction / consensus code paths, then verifies
-# that the expected spans and metrics appear in Tempo and Prometheus.
-#
-# Usage:
-#   bash docker/telemetry/integration-test.sh
-#
-# Prerequisites:
-#   - .build/xrpld built with telemetry=ON
-#   - docker compose (v2)
-#   - curl, jq
-#
-# The script leaves the observability stack and xrpld nodes running
-# so you can manually inspect Tempo (localhost:3200) and Grafana
-# (localhost:3000). Run with --cleanup to tear down instead.
-
-set -euo pipefail
-
-# ---------------------------------------------------------------------------
-# Configuration
-# ---------------------------------------------------------------------------
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
-XRPLD="$REPO_ROOT/.build/xrpld"
-COMPOSE_FILE="$SCRIPT_DIR/docker-compose.yml"
-STANDALONE_CFG="$SCRIPT_DIR/xrpld-telemetry.cfg"
-WORKDIR="${WORKDIR:-/tmp/xrpld-integration}"
-NUM_NODES=6
-PEER_PORT_BASE=51235
-RPC_PORT_BASE=5005
-CONSENSUS_TIMEOUT=120
-GENESIS_ACCOUNT="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"
-GENESIS_SEED="snoPBrXtMeMyMHUVTgbuqAfg1SUTb"
-DEST_ACCOUNT="" # Generated dynamically via wallet_propose
-TEMPO="http://localhost:3200"
-PROM="http://localhost:9090"
-LOKI="http://localhost:3100"
-# How long to wait for a log line to travel file -> file_log receiver -> batch
-# processor -> Loki. The batch timeout is 1s, so this is mostly ingestion slack.
-LOKI_INGEST_TIMEOUT=30
-
-# Hard ceiling on every curl probe below. curl has no overall timeout of its
-# own, so a server that accepts the connection and then never answers parks a
-# poll loop forever and its attempt count stops bounding anything. 5 s is well
-# above a healthy reply, so only a wedged server hits the ceiling.
-CURL_MAX_TIME=5
-
-# Counters for pass/fail
-PASS=0
-FAIL=0
-
-# Unix seconds just before this run's nodes start. Every Tempo search is
-# bounded to this run, so a previous run's traces cannot satisfy an assertion.
-# Set in Step 5; check_span refuses to run while it is empty.
-RUN_START=""
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-log() { printf "\033[1;34m[INFO]\033[0m  %s\n" "$*"; }
-ok() {
-    printf "\033[1;32m[PASS]\033[0m  %s\n" "$*"
-    PASS=$((PASS + 1))
-}
-fail() {
-    printf "\033[1;31m[FAIL]\033[0m  %s\n" "$*"
-    FAIL=$((FAIL + 1))
-}
-die() {
-    printf "\033[1;31m[ERROR]\033[0m %s\n" "$*" >&2
-    exit 1
-}
-
-check_span() {
-    local op="$1"
-    local count
-    # -G is required: it moves the urlencoded params into the query string.
-    # Without it curl POSTs them as a request body, and Tempo answers 200
-    # while ignoring the query — so every span name would look present.
-    #
-    # start/end bound the search to this run. Tempo keeps blocks for
-    # block_retention (tempo.yaml, 1h) on a named volume, so without a bound
-    # an older run's spans answer for this one. The end margin covers spans
-    # exported while this query is in flight.
-    [ -n "$RUN_START" ] || die "check_span called before RUN_START was set"
-    count=$(curl -sfG --max-time "$CURL_MAX_TIME" "$TEMPO/api/search" \
-        --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \
-        --data-urlencode "start=$RUN_START" \
-        --data-urlencode "end=$(($(date +%s) + 60))" \
-        --data-urlencode "limit=5" |
-        jq '.traces | length' 2>/dev/null || echo 0)
-    if [ "$count" -gt 0 ]; then
-        ok "$op  ($count traces)"
-    else
-        fail "$op  (0 traces)"
-    fi
-}
-
-# Verify trace_id injection in xrpld log output.
-# Greps all node debug.log files for the "trace_id=<hex> span_id=<hex>"
-# pattern that Logs::format() injects when an active OTel span exists.
-# Also cross-checks that a trace_id found in logs matches a trace in Tempo.
-check_log_correlation() {
-    log "Checking log-trace correlation..."
-
-    local total_matches=0
-    local files_scanned=0
-    local sample_trace_id=""
-
-    for i in $(seq 1 "$NUM_NODES"); do
-        local logfile="$WORKDIR/Node-$i/debug.log"
-        if [ ! -f "$logfile" ]; then
-            continue
-        fi
-        files_scanned=$((files_scanned + 1))
-        local matches
-        matches=$(grep -c 'trace_id=[a-f0-9]\{32\} span_id=[a-f0-9]\{16\}' "$logfile") || matches=0
-        total_matches=$((total_matches + matches))
-        # Capture the first trace_id we find for cross-referencing with Tempo
-        if [ -z "$sample_trace_id" ] && [ "$matches" -gt 0 ]; then
-            # -m1 makes grep stop after the first match and exit normally.
-            # Piping into `head -1` instead closes the pipe under grep, and
-            # under `set -o pipefail` the resulting SIGPIPE (141) aborts the
-            # whole run. It only bites once the log is bigger than the pipe
-            # buffer, so it reads as a flaky test.
-            sample_trace_id=$(grep -m1 -o 'trace_id=[a-f0-9]\{32\}' "$logfile" | cut -d= -f2)
-        fi
-    done
-
-    if [ "$files_scanned" -eq 0 ]; then
-        fail "Log correlation: no debug.log files found in $WORKDIR/Node-*/"
-        return
-    fi
-
-    if [ "$total_matches" -gt 0 ]; then
-        ok "Log correlation: found $total_matches log lines with trace_id ($files_scanned nodes scanned)"
-    else
-        fail "Log correlation: no trace_id found in any node debug.log ($files_scanned nodes scanned)"
-    fi
-
-    # Cross-check: verify the sample trace_id exists in Tempo
-    if [ -n "$sample_trace_id" ]; then
-        local trace_found
-        # Tempo /api/traces/{id} returns OTLP shape: {"batches":[...]}
-        trace_found=$(curl -sf "$TEMPO/api/traces/$sample_trace_id" |
-            jq '.batches | length' 2>/dev/null) || trace_found=0
-        if [ "$trace_found" -gt 0 ]; then
-            ok "Log-Tempo cross-check: trace_id=$sample_trace_id found in Tempo"
-        else
-            fail "Log-Tempo cross-check: trace_id=$sample_trace_id NOT found in Tempo"
-        fi
-
-        check_loki_ingestion "$sample_trace_id"
-    fi
-}
-
-# Verify the log line actually reached Loki, not just the local file.
-#
-# Without this the log-correlation check passes on a stack whose log mount is
-# wrong or whose Loki exporter is broken, because reading the file and reading
-# Tempo both still work. This is the only assertion that exercises the
-# file_log -> Loki hop, so it is what makes the log pipeline tested rather than
-# merely configured.
-#
-# Uses /query_range, not /query: Loki rejects a bare log selector on the instant
-# endpoint with HTTP 400 and a text/plain body, so jq could never parse it.
-# Bounds are unix nanoseconds, matching workload/validate_telemetry.py.
-check_loki_ingestion() {
-    local trace_id="$1"
-    local lines=0
-    local start_ns end_ns
-
-    for attempt in $(seq 1 "$LOKI_INGEST_TIMEOUT"); do
-        end_ns=$(($(date +%s) * 1000000000))
-        # Look back over the whole run, not a fixed window: the entry carries
-        # the timestamp parsed out of the log line, not its ingestion time.
-        start_ns=$((end_ns - 86400000000000))
-        lines=$(curl -sfG "$LOKI/loki/api/v1/query_range" \
-            --data-urlencode "query={service_name=\"xrpld\"} |= \"$trace_id\"" \
-            --data-urlencode "start=$start_ns" \
-            --data-urlencode "end=$end_ns" \
-            --data-urlencode "limit=5" \
-            --data-urlencode "direction=backward" |
-            jq '[.data.result[].values | length] | add // 0' 2>/dev/null) || lines=0
-        if [ "${lines:-0}" -gt 0 ]; then
-            ok "Loki ingestion: trace_id=$trace_id found in Loki ($lines lines, attempt $attempt)"
-            return
-        fi
-        sleep 1
-    done
-
-    fail "Loki ingestion: trace_id=$trace_id never reached Loki after ${LOKI_INGEST_TIMEOUT}s"
-}
-
-cleanup() {
-    log "Cleaning up..."
-    # Kill xrpld nodes
-    for i in $(seq 1 "$NUM_NODES"); do
-        local pidfile="$WORKDIR/Node-$i/xrpld.pid"
-        if [ -f "$pidfile" ]; then
-            kill "$(cat "$pidfile")" 2>/dev/null || true
-            rm -f "$pidfile"
-        fi
-    done
-    # Also kill any straggling xrpld processes from our workdir
-    pkill -f "$WORKDIR" 2>/dev/null || true
-    # Stop docker stack. -v also drops the tempo-data volume: plain `down`
-    # keeps it, and retained traces would then answer a later run's searches.
-    docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true
-    # Remove workdir
-    rm -rf "$WORKDIR"
-    log "Cleanup complete."
-}
-
-# Handle --cleanup flag
-if [ "${1:-}" = "--cleanup" ]; then
-    cleanup
-    exit 0
-fi
-
-# ---------------------------------------------------------------------------
-# Step 0: Prerequisites
-# ---------------------------------------------------------------------------
-log "Checking prerequisites..."
-
-command -v docker >/dev/null 2>&1 || die "docker not found"
-docker compose version >/dev/null 2>&1 || die "docker compose (v2) not found"
-command -v curl >/dev/null 2>&1 || die "curl not found"
-command -v jq >/dev/null 2>&1 || die "jq not found"
-[ -x "$XRPLD" ] || die "xrpld binary not found at $XRPLD (build with telemetry=ON)"
-[ -f "$COMPOSE_FILE" ] || die "docker-compose.yml not found at $COMPOSE_FILE"
-[ -f "$STANDALONE_CFG" ] || die "xrpld-telemetry.cfg not found at $STANDALONE_CFG"
-
-log "All prerequisites met."
-
-# ---------------------------------------------------------------------------
-# Step 1: Clean previous run
-# ---------------------------------------------------------------------------
-log "Cleaning previous run data..."
-for i in $(seq 1 "$NUM_NODES"); do
-    pidfile="$WORKDIR/Node-$i/xrpld.pid"
-    if [ -f "$pidfile" ]; then
-        kill "$(cat "$pidfile")" 2>/dev/null || true
-    fi
-done
-pkill -f "$WORKDIR" 2>/dev/null || true
-# Kill any xrpld using the standalone config (from key generation)
-pkill -f "xrpld-telemetry.cfg" 2>/dev/null || true
-sleep 2
-rm -rf "$WORKDIR"
-# A run that reached the summary left the stack up, so nothing has torn it
-# down. Do it here, with -v: Tempo's traces and Prometheus' samples must not
-# survive into this run, or an assertion can pass on the previous run's data.
-docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true
-mkdir -p "$WORKDIR"
-
-# ---------------------------------------------------------------------------
-# Step 2: Start observability stack
-# ---------------------------------------------------------------------------
-
-# From here on the script owns the docker stack and the xrpld nodes, so an
-# abort must tear them down instead of leaving them behind. A run that
-# reaches the summary deliberately leaves everything up for inspection
-# (see the header comment), so the trap only fires before that point.
-RUN_COMPLETED=0
-on_exit() {
-    local status=$?
-    if [ "$RUN_COMPLETED" -eq 0 ]; then
-        log "Aborted with exit status $status — tearing down."
-        cleanup
-    fi
-}
-trap on_exit EXIT
-trap 'exit 130' INT
-trap 'exit 143' TERM
-
-log "Starting observability stack..."
-# Point the collector's log mount at this test's workdir so it tails the
-# per-node debug.log files this script generates. The compose default
-# (./data/logs) is for user-run xrpld; the test owns its own log root.
-XRPLD_LOG_DIR="$WORKDIR" docker compose -f "$COMPOSE_FILE" up -d
-
-log "Waiting for otel-collector to be ready..."
-for attempt in $(seq 1 30); do
-    # The OTLP HTTP endpoint returns 405 for GET (expects POST), which
-    # means it is listening.  curl -sf would fail on 405, so we check
-    # the HTTP status code explicitly.
-    status=$(curl -so /dev/null -w '%{http_code}' --max-time "$CURL_MAX_TIME" http://localhost:4318/ 2>/dev/null || echo 000)
-    if [ "$status" != "000" ]; then
-        log "otel-collector ready (attempt $attempt, HTTP $status)."
-        break
-    fi
-    if [ "$attempt" -eq 30 ]; then
-        die "otel-collector not ready after 30s"
-    fi
-    sleep 1
-done
-
-log "Waiting for Tempo to be ready..."
-for attempt in $(seq 1 30); do
-    if curl -sf --max-time "$CURL_MAX_TIME" "$TEMPO/ready" >/dev/null 2>&1; then
-        log "Tempo ready (attempt $attempt)."
-        break
-    fi
-    if [ "$attempt" -eq 30 ]; then
-        die "Tempo not ready after 30s"
-    fi
-    sleep 1
-done
-
-log "Waiting for Loki to be ready..."
-for attempt in $(seq 1 60); do
-    if curl -sf "$LOKI/ready" >/dev/null 2>&1; then
-        log "Loki ready (attempt $attempt)."
-        break
-    fi
-    if [ "$attempt" -eq 60 ]; then
-        die "Loki not ready after 60s"
-    fi
-    sleep 1
-done
-
-# ---------------------------------------------------------------------------
-# Step 3: Generate validator keys
-# ---------------------------------------------------------------------------
-log "Generating $NUM_NODES validator key pairs..."
-
-# Start a temporary standalone xrpld for key generation
-TEMP_DATA="$WORKDIR/temp-keygen"
-mkdir -p "$TEMP_DATA"
-
-# Create a minimal temp config for key generation
-TEMP_CFG="$TEMP_DATA/xrpld.cfg"
-cat >"$TEMP_CFG" <<EOCFG
-[server]
-port_rpc_temp
-
-[port_rpc_temp]
-port = 5099
-ip = 127.0.0.1
-admin = 127.0.0.1
-protocol = http
-
-[node_db]
-type=NuDB
-path=$TEMP_DATA/nudb
-online_delete=256
-
-[database_path]
-$TEMP_DATA/db
-
-[debug_logfile]
-$TEMP_DATA/debug.log
-
-[ssl_verify]
-0
-EOCFG
-
-"$XRPLD" --conf "$TEMP_CFG" -a --start >"$TEMP_DATA/stdout.log" 2>&1 &
-TEMP_PID=$!
-log "Temporary xrpld started (PID $TEMP_PID), waiting for RPC..."
-
-for attempt in $(seq 1 30); do
-    if curl -sf --max-time "$CURL_MAX_TIME" http://localhost:5099 -d '{"method":"server_info"}' >/dev/null 2>&1; then
-        log "Temporary xrpld RPC ready (attempt $attempt)."
-        break
-    fi
-    if [ "$attempt" -eq 30 ]; then
-        kill "$TEMP_PID" 2>/dev/null || true
-        die "Temporary xrpld RPC not ready after 30s"
-    fi
-    sleep 1
-done
-
-declare -a SEEDS
-declare -a PUBKEYS
-
-for i in $(seq 1 "$NUM_NODES"); do
-    result=$(curl -sf --max-time "$CURL_MAX_TIME" http://localhost:5099 -d '{"method":"validation_create"}')
-    seed=$(echo "$result" | jq -r '.result.validation_seed')
-    pubkey=$(echo "$result" | jq -r '.result.validation_public_key')
-    if [ -z "$seed" ] || [ "$seed" = "null" ]; then
-        kill "$TEMP_PID" 2>/dev/null || true
-        die "Failed to generate key pair $i"
-    fi
-    SEEDS+=("$seed")
-    PUBKEYS+=("$pubkey")
-    log "  Node $i: $pubkey"
-done
-
-kill "$TEMP_PID" 2>/dev/null || true
-wait "$TEMP_PID" 2>/dev/null || true
-rm -rf "$TEMP_DATA"
-log "Key generation complete."
-
-# ---------------------------------------------------------------------------
-# Step 4: Generate node configs and validators.txt
-# ---------------------------------------------------------------------------
-log "Generating node configs..."
-
-# Create shared validators.txt
-VALIDATORS_FILE="$WORKDIR/validators.txt"
-{
-    echo "[validators]"
-    for i in $(seq 0 $((NUM_NODES - 1))); do
-        echo "${PUBKEYS[$i]}"
-    done
-} >"$VALIDATORS_FILE"
-
-# Create per-node configs
-for i in $(seq 1 "$NUM_NODES"); do
-    NODE_DIR="$WORKDIR/Node-$i"
-    mkdir -p "$NODE_DIR/nudb" "$NODE_DIR/db"
-
-    RPC_PORT=$((RPC_PORT_BASE + i - 1))
-    PEER_PORT=$((PEER_PORT_BASE + i - 1))
-    SEED="${SEEDS[$((i - 1))]}"
-
-    # Build ips_fixed list (all peers except self)
-    IPS_FIXED=""
-    for j in $(seq 1 "$NUM_NODES"); do
-        if [ "$j" -ne "$i" ]; then
-            IPS_FIXED="${IPS_FIXED}127.0.0.1 $((PEER_PORT_BASE + j - 1))
-"
-        fi
-    done
-
-    cat >"$NODE_DIR/xrpld.cfg" <<EOCFG
-[server]
-port_rpc
-port_peer
-
-[port_rpc]
-port = $RPC_PORT
-ip = 127.0.0.1
-admin = 127.0.0.1
-protocol = http
-
-[port_peer]
-port = $PEER_PORT
-ip = 0.0.0.0
-protocol = peer
-
-# A private id, so telemetry stamps xrpl.network.type=unknown. The config
-# default is id 0, which maps to "mainnet" -- this cluster's spans and metrics
-# would then share dashboard series with real mainnet data.
-[network_id]
-1025
-
-[node_db]
-type=NuDB
-path=$NODE_DIR/nudb
-online_delete=256
-
-[database_path]
-$NODE_DIR/db
-
-[debug_logfile]
-$NODE_DIR/debug.log
-
-[validation_seed]
-$SEED
-
-[validators_file]
-$VALIDATORS_FILE
-
-[ips_fixed]
-${IPS_FIXED}
-[peer_private]
-1
-
-[telemetry]
-enabled=1
-service_instance_id=Node-${i}
-traces_endpoint=http://localhost:4318/v1/traces
-metrics_endpoint=http://localhost:4318/v1/metrics
-batch_size=512
-batch_delay_ms=2000
-max_queue_size=2048
-trace_rpc=1
-trace_transactions=1
-trace_consensus=1
-trace_peer=1
-trace_ledger=1
-
-[insight]
-# server=otel is the only load-bearing key here -- it selects OTelCollector so
-# beast::insight metrics leave over OTLP. No prefix is set on purpose: on this
-# path it is inert, because OTelCollector's formatName() only lowercases the raw
-# instrument name and the one place the class reads prefix_ is its startup log
-# line. The service is identified by the OTel resource service.name.
-server=otel
-endpoint=http://localhost:4318/v1/metrics
-service_instance_id=Node-${i}
-
-[rpc_startup]
-# info, not warning: check_log_correlation() below greps these nodes' debug.log
-# for "trace_id=<hex> span_id=<hex>" and fails if it finds none. A line only
-# carries trace context when it is emitted while a span is current, and the one
-# line guaranteed to be is the consensus accept pair in RCLConsensus.cpp, which
-# logs at info inside the activation doAccept makes over its whole body. At
-# warning that pair is suppressed, leaving only whatever warn-or-worse line
-# happens to fire inside some active span -- so the check passed incidentally
-# rather than by construction. debug is deliberately not used here either, to
-# stay consistent with the workload harness.
-{ "command": "log_level", "severity": "info" }
-
-[ssl_verify]
-0
-EOCFG
-
-    log "  Node $i config: RPC=$RPC_PORT, Peer=$PEER_PORT"
-done
-
-# ---------------------------------------------------------------------------
-# Step 5: Start all 6 nodes
-# ---------------------------------------------------------------------------
-log "Starting $NUM_NODES xrpld nodes..."
-
-# Lower bound for every Tempo search below. Only these nodes have a
-# [telemetry] section, so nothing before this instant belongs to this run.
-RUN_START=$(date +%s)
-
-for i in $(seq 1 "$NUM_NODES"); do
-    NODE_DIR="$WORKDIR/Node-$i"
-    "$XRPLD" --conf "$NODE_DIR/xrpld.cfg" --start >"$NODE_DIR/stdout.log" 2>&1 &
-    echo $! >"$NODE_DIR/xrpld.pid"
-    log "  Node $i started (PID $(cat "$NODE_DIR/xrpld.pid"))"
-done
-
-# Give nodes a moment to initialize
-sleep 5
-
-# ---------------------------------------------------------------------------
-# Step 6: Wait for consensus
-# ---------------------------------------------------------------------------
-log "Waiting for nodes to reach 'proposing' state (timeout: ${CONSENSUS_TIMEOUT}s)..."
-
-start_time=$(date +%s)
-nodes_ready=0
-consensus_timed_out=0
-
-while [ "$nodes_ready" -lt "$NUM_NODES" ]; do
-    elapsed=$(($(date +%s) - start_time))
-    if [ "$elapsed" -ge "$CONSENSUS_TIMEOUT" ]; then
-        fail "Consensus timeout after ${CONSENSUS_TIMEOUT}s ($nodes_ready/$NUM_NODES nodes ready)"
-        log "Continuing with partial consensus..."
-        consensus_timed_out=1
-        break
-    fi
-
-    nodes_ready=0
-    for i in $(seq 1 "$NUM_NODES"); do
-        RPC_PORT=$((RPC_PORT_BASE + i - 1))
-        state=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT" \
-            -d '{"method":"server_info"}' 2>/dev/null |
-            jq -r '.result.info.server_state' 2>/dev/null || echo "unreachable")
-        if [ "$state" = "proposing" ]; then
-            nodes_ready=$((nodes_ready + 1))
-        fi
-    done
-    printf "\r  %d/%d nodes proposing (%ds elapsed)..." "$nodes_ready" "$NUM_NODES" "$elapsed"
-    if [ "$nodes_ready" -lt "$NUM_NODES" ]; then
-        sleep 3
-    fi
-done
-echo ""
-
-if [ "$nodes_ready" -eq "$NUM_NODES" ]; then
-    ok "All $NUM_NODES nodes reached 'proposing' state"
-elif [ "$consensus_timed_out" -eq 0 ]; then
-    # The timeout branch above already called fail(), so reporting again here
-    # would count one timeout twice. Only reachable if the loop ever gains
-    # another early exit.
-    fail "Only $nodes_ready/$NUM_NODES nodes reached 'proposing' state"
-fi
-
-# ---------------------------------------------------------------------------
-# Step 6b: Wait for validated ledger
-# ---------------------------------------------------------------------------
-log "Waiting for first validated ledger..."
-for attempt in $(seq 1 60); do
-    val_seq=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
-        -d '{"method":"server_info"}' 2>/dev/null |
-        jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
-    if [ "$val_seq" -gt 2 ] 2>/dev/null; then
-        ok "First validated ledger: seq $val_seq"
-        break
-    fi
-    if [ "$attempt" -eq 60 ]; then
-        fail "No validated ledger after 60s"
-    fi
-    sleep 1
-done
-
-# ---------------------------------------------------------------------------
-# Step 7: Exercise RPC spans
-# ---------------------------------------------------------------------------
-log "Exercising RPC spans..."
-
-curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
-    -d '{"method":"server_info"}' >/dev/null
-curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
-    -d '{"method":"server_state"}' >/dev/null
-curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
-    -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' >/dev/null
-
-log "RPC commands sent. Waiting 5s for batch export..."
-sleep 5
-
-# ---------------------------------------------------------------------------
-# Step 8: Submit transaction
-# ---------------------------------------------------------------------------
-log "Submitting Payment transaction..."
-
-# Generate a destination wallet
-log "  Generating destination wallet..."
-# Guarded: under set -e an unguarded curl failure would abort the whole
-# script, so the fallback below could never run.
-wallet_result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
-    -d '{"method":"wallet_propose"}') || wallet_result=""
-DEST_ACCOUNT=$(echo "$wallet_result" | jq -r '.result.account_id' 2>/dev/null || echo "")
-if [ -z "$DEST_ACCOUNT" ] || [ "$DEST_ACCOUNT" = "null" ]; then
-    fail "Could not generate destination wallet"
-    DEST_ACCOUNT="rrrrrrrrrrrrrrrrrrrrrhoLvTp" # ACCOUNT_ZERO fallback
-fi
-log "  Destination: $DEST_ACCOUNT"
-
-# Get genesis account info
-acct_result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
-    -d "{\"method\":\"account_info\",\"params\":[{\"account\":\"$GENESIS_ACCOUNT\"}]}") || acct_result=""
-seq_num=$(echo "$acct_result" | jq -r '.result.account_data.Sequence' 2>/dev/null || echo "unknown")
-log "  Genesis account sequence: $seq_num"
-
-# Submit payment
-submit_result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
-    -d "{\"method\":\"submit\",\"params\":[{\"secret\":\"$GENESIS_SEED\",\"tx_json\":{\"TransactionType\":\"Payment\",\"Account\":\"$GENESIS_ACCOUNT\",\"Destination\":\"$DEST_ACCOUNT\",\"Amount\":\"10000000\"}}]}") || submit_result=""
-
-engine_result=$(echo "$submit_result" | jq -r '.result.engine_result' 2>/dev/null || echo "unknown")
-tx_hash=$(echo "$submit_result" | jq -r '.result.tx_json.hash' 2>/dev/null || echo "unknown")
-
-if [ "$engine_result" = "tesSUCCESS" ] || [ "$engine_result" = "terQUEUED" ]; then
-    ok "Transaction submitted: $engine_result (hash: ${tx_hash:0:16}...)"
-else
-    fail "Transaction submission: $engine_result"
-    log "  Full response: $(echo "$submit_result" | jq -c .result 2>/dev/null)"
-fi
-
-log "Waiting 15s for consensus round + batch export..."
-sleep 15
-
-# ---------------------------------------------------------------------------
-# Step 9: Verify Tempo traces
-# ---------------------------------------------------------------------------
-log "Verifying spans in Tempo..."
-
-# Check service registration
-services=$(curl -sf --max-time "$CURL_MAX_TIME" "$TEMPO/api/v2/search/tag/resource.service.name/values" |
-    jq -r '.tagValues[].value' 2>/dev/null || echo "")
-# Whole-line match: a substring match would also accept a value that merely
-# contains "xrpld". This endpoint ignores start/end (measured), so its only
-# protection against a previous run is the teardown in Step 1.
-if echo "$services" | grep -Fxq "xrpld"; then
-    ok "Service 'xrpld' registered in Tempo"
-else
-    fail "Service 'xrpld' NOT found in Tempo (found: $services)"
-fi
-
-log ""
-log "--- RPC Spans ---"
-check_span "rpc.http_request"
-check_span "rpc.process"
-check_span "rpc.command.server_info"
-check_span "rpc.command.server_state"
-check_span "rpc.command.ledger"
-
-log ""
-log "--- Transaction Spans ---"
-check_span "tx.process"
-check_span "tx.receive"
-check_span "tx.apply"
-
-log ""
-log "--- Consensus Spans ---"
-check_span "consensus.proposal.send"
-check_span "consensus.ledger_close"
-check_span "consensus.accept"
-check_span "consensus.validation.send"
-
-log ""
-log "--- Ledger Spans ---"
-check_span "ledger.build"
-check_span "ledger.validate"
-check_span "ledger.store"
-
-log ""
-log "--- Peer Spans (trace_peer=1) ---"
-check_span "peer.proposal.receive"
-check_span "peer.validation.receive"
-
-# ---------------------------------------------------------------------------
-# Step 9b: Verify log-trace correlation
-# ---------------------------------------------------------------------------
-log ""
-log "--- Log-Trace Correlation ---"
-check_log_correlation
-
-# ---------------------------------------------------------------------------
-# Step 10: Verify Prometheus span_metrics
-# ---------------------------------------------------------------------------
-log ""
-log "--- Spanmetrics ---"
-log "Waiting 20s for Prometheus scrape cycle..."
-sleep 20
-
-# Names come from the spanmetrics connector's `namespace: "span"` in
-# otel-collector-config.yaml. Without that namespace the connector emits
-# traces_span_metrics_*, so these queries must move whenever it changes.
-calls_count=$(curl -sf --max-time "$CURL_MAX_TIME" "$PROM/api/v1/query?query=span_calls_total" |
-    jq '.data.result | length' 2>/dev/null || echo 0)
-if [ "$calls_count" -gt 0 ]; then
-    ok "Prometheus: span_calls_total ($calls_count series)"
-else
-    fail "Prometheus: span_calls_total (0 series)"
-fi
-
-duration_count=$(curl -sf --max-time "$CURL_MAX_TIME" "$PROM/api/v1/query?query=span_duration_milliseconds_count" |
-    jq '.data.result | length' 2>/dev/null || echo 0)
-if [ "$duration_count" -gt 0 ]; then
-    ok "Prometheus: duration histogram ($duration_count series)"
-else
-    fail "Prometheus: duration histogram (0 series)"
-fi
-
-# Check Grafana
-if curl -sf --max-time "$CURL_MAX_TIME" http://localhost:3000/api/health >/dev/null 2>&1; then
-    ok "Grafana: healthy at localhost:3000"
-else
-    fail "Grafana: not reachable at localhost:3000"
-fi
-
-# ---------------------------------------------------------------------------
-# Step 10b: Verify native OTel metrics in Prometheus (beast::insight)
-# ---------------------------------------------------------------------------
-log ""
-log "--- Native OTel Metrics (beast::insight via OTLP) ---"
-log "Waiting 20s for OTLP metric export + Prometheus scrape..."
-sleep 20
-
-check_otel_metric() {
-    local metric_name="$1"
-    local result
-    result=$(curl -sf --max-time "$CURL_MAX_TIME" "$PROM/api/v1/query?query=$metric_name" |
-        jq '.data.result | length' 2>/dev/null || echo 0)
-    if [ "$result" -gt 0 ]; then
-        ok "OTel: $metric_name ($result series)"
-    else
-        fail "OTel: $metric_name (0 series)"
-    fi
-}
-
-# Names are what OTelCollector::formatName() produces: the beast::insight
-# name lowercased with '.' and ' ' mapped to '_', any group() segment kept, and
-# no prefix. The [insight] prefix knob is logged at startup and never applied on
-# this path, and the collector's prometheus exporter sets no namespace, so a
-# name carrying a product prefix or capitals cannot match any exported series.
-
-# Node health gauges (ObservableGauge — no _total suffix)
-check_otel_metric "ledgermaster_validated_ledger_age"
-check_otel_metric "ledgermaster_published_ledger_age"
-check_otel_metric "jobq_job_count"
-
-# State accounting
-check_otel_metric "state_accounting_full_duration"
-
-# Peer finder
-check_otel_metric "peer_finder_active_inbound_peers"
-check_otel_metric "peer_finder_active_outbound_peers"
-
-# RPC counters (Counter — Prometheus adds _total suffix automatically)
-check_otel_metric "rpc_requests_total"
-
-# Overlay traffic — one series per TrafficCount category; "total" is the
-# aggregate category.
-check_otel_metric "total_bytes_in"
-
-# Verify StatsD receiver is NOT required (no statsd receiver in pipeline)
-log ""
-log "--- Verify StatsD receiver is not required ---"
-# StatsD listens on UDP 8125, so probe with a UDP-aware tool, not curl (TCP).
-if command -v ss >/dev/null 2>&1; then
-    if ss -ulnp 2>/dev/null | grep -q ":8125"; then
-        fail "StatsD port 8125 appears to be listening (should not be needed)"
-    else
-        ok "StatsD port 8125 is not listening (not required)"
-    fi
-else
-    log "ss not found -- skipping StatsD UDP port check"
-fi
-
-# ---------------------------------------------------------------------------
-# Step 10c: Verify OTel SDK Metrics
-# ---------------------------------------------------------------------------
-log ""
-log "--- OTel SDK Metrics (MetricsRegistry) ---"
-log "Waiting 15s for OTel metric export + Prometheus scrape..."
-sleep 15
-
-check_otel_metric() {
-    local metric_name="$1"
-    local result
-    result=$(curl -sf "$PROM/api/v1/query?query=$metric_name" |
-        jq '.data.result | length' 2>/dev/null || echo 0)
-    if [ "$result" -gt 0 ]; then
-        ok "OTel: $metric_name ($result series)"
-    else
-        fail "OTel: $metric_name (0 series)"
-    fi
-}
-
-# NodeStore I/O
-check_otel_metric 'nodestore_state{metric="node_reads_total"}'
-check_otel_metric 'nodestore_state{metric="write_load"}'
-
-# Cache hit rates
-check_otel_metric 'cache_metrics{metric="SLE_hit_rate"}'
-check_otel_metric 'cache_metrics{metric="treenode_cache_size"}'
-
-# TxQ metrics
-check_otel_metric 'txq_metrics{metric="txq_count"}'
-check_otel_metric 'txq_metrics{metric="txq_reference_fee_level"}'
-
-# Per-RPC metrics
-check_otel_metric "rpc_method_started_total"
-check_otel_metric "rpc_method_finished_total"
-
-# Per-job metrics
-check_otel_metric "job_queued_total"
-check_otel_metric "job_finished_total"
-
-# Counted object instances
-check_otel_metric "object_count"
-
-# Load factor breakdown
-check_otel_metric 'load_factor_metrics{metric="load_factor"}'
-check_otel_metric 'load_factor_metrics{metric="load_factor_server"}'
-
-# ValidationTracker rolling-window agreement gauge.
-# MetricsRegistry::registerValidationAgreementGauge() publishes
-# validation_agreement with a `metric` label for each window
-# (1h / 24h / 7d) plus the matching agreement/miss counts. The 7-day
-# window matches the external xrpl-validator-dashboard parity target.
-check_otel_metric 'validation_agreement{metric="agreement_pct_1h"}'
-check_otel_metric 'validation_agreement{metric="agreement_pct_24h"}'
-check_otel_metric 'validation_agreement{metric="agreement_pct_7d"}'
-check_otel_metric 'validation_agreement{metric="agreements_1h"}'
-check_otel_metric 'validation_agreement{metric="missed_1h"}'
-check_otel_metric 'validation_agreement{metric="agreements_24h"}'
-check_otel_metric 'validation_agreement{metric="missed_24h"}'
-check_otel_metric 'validation_agreement{metric="agreements_7d"}'
-check_otel_metric 'validation_agreement{metric="missed_7d"}'
-
-# ---------------------------------------------------------------------------
-# Step 11: Summary
-# ---------------------------------------------------------------------------
-
-# All checks are done, so the run counts as complete: keep the stack and the
-# nodes up for inspection even when some checks failed.
-RUN_COMPLETED=1
-
-echo ""
-echo "==========================================================="
-echo "  INTEGRATION TEST RESULTS"
-echo "==========================================================="
-printf "  \033[1;32mPASSED: %d\033[0m\n" "$PASS"
-printf "  \033[1;31mFAILED: %d\033[0m\n" "$FAIL"
-echo "==========================================================="
-echo ""
-echo "  Observability stack is running:"
-echo ""
-echo "    Tempo:         http://localhost:3200"
-echo "    Grafana:       http://localhost:3000"
-echo "    Prometheus:    http://localhost:9090"
-echo "    Loki:          http://localhost:3100"
-echo ""
-echo "  xrpld nodes (6) are running:"
-for i in $(seq 1 "$NUM_NODES"); do
-    RPC_PORT=$((RPC_PORT_BASE + i - 1))
-    PEER_PORT=$((PEER_PORT_BASE + i - 1))
-    echo "    Node $i: RPC=localhost:$RPC_PORT  Peer=:$PEER_PORT  PID=$(cat "$WORKDIR/Node-$i/xrpld.pid" 2>/dev/null || echo 'unknown')"
-done
-echo ""
-echo "  To tear down:"
-echo "    bash docker/telemetry/integration-test.sh --cleanup"
-echo ""
-echo "==========================================================="
-
-if [ "$FAIL" -gt 0 ]; then
-    exit 1
-fi

docker/telemetry/otel-collector-config.grafanacloud.yaml

@@ -1,421 +0,0 @@
-# OpenTelemetry Collector configuration — Grafana Cloud dual-export variant.
-#
-# Identical to otel-collector-config.yaml (local Tempo/Prometheus/Loki) but
-# ALSO ships all three signals to Grafana Cloud over a single OTLP/HTTP
-# endpoint. Local backends are kept so the on-box stack still works as a
-# fallback; for cloud-only, see the comment at the start of service.pipelines.
-#
-# Selecting this config is a RUNTIME choice — no code or image changes:
-#   Local only (default):
-#     docker compose -f docker/telemetry/docker-compose.yml up -d
-#   Local + Grafana Cloud:
-#     docker compose -f docker/telemetry/docker-compose.yml \
-#                    -f docker/telemetry/docker-compose.grafanacloud.yaml up -d
-#
-# The override file mounts THIS config over the collector's config path and
-# injects the three required secrets as environment variables:
-#   GRAFANA_CLOUD_OTLP_ENDPOINT  e.g. https://otlp-gateway-<zone>.grafana.net/otlp
-#   GRAFANA_CLOUD_INSTANCE_ID    numeric Grafana Cloud stack/instance id
-#   GRAFANA_CLOUD_API_TOKEN      a Cloud Access Policy token with metrics/traces/logs:write
-# Find all three under Grafana Cloud -> Connections -> "OpenTelemetry (OTLP)".
-
-extensions:
-  health_check:
-    endpoint: 0.0.0.0:13133
-  # Basic-auth for the Grafana Cloud OTLP gateway: username = instance id,
-  # password = API token. Values come from the container environment so no
-  # secret is committed to the repo.
-  basicauth/grafanacloud:
-    client_auth:
-      username: ${env:GRAFANA_CLOUD_INSTANCE_ID}
-      password: ${env:GRAFANA_CLOUD_API_TOKEN}
-
-receivers:
-  otlp:
-    protocols:
-      grpc:
-        endpoint: 0.0.0.0:4317
-      http:
-        endpoint: 0.0.0.0:4318
-  file_log:
-    include:
-      - /var/log/xrpld/*/debug.log
-    # Needed to recover which node a line came from. The subdirectory name is
-    # the only per-node signal in the log stream: Logs::format() writes
-    # trace_id and span_id but no node identity. Emitters name the directory
-    # after their own service_instance_id so the two agree.
-    include_file_path: true
-    operators:
-      - type: regex_parser
-        regex: '^(?P<timestamp>\S+\s+\S+)\s+\S+\s+(?:(?P<partition>\S+):)?(?P<severity>\S+)\s+(?:trace_id=(?P<trace_id>[a-f0-9]+)\s+span_id=(?P<span_id>[a-f0-9]+)\s+)?(?P<message>.*)$'
-        timestamp:
-          parse_from: attributes.timestamp
-          layout: "%Y-%b-%d %H:%M:%S.%f"
-          location: UTC
-      # Lift the per-node directory out of the file path and onto the
-      # RESOURCE. include_file_path alone is not enough: it produces a log
-      # RECORD attribute, and on OTLP ingest Loki promotes only an allow-list
-      # of RESOURCE attributes to indexed stream labels. A record attribute
-      # becomes structured metadata, which cannot be used in a {...} selector.
-      # service.instance.id is on that allow-list and arrives as the LogQL
-      # label service_instance_id, which is the label the dashboards filter on.
-      # Dotted keys need bracket syntax; dot notation would be read as a
-      # nested traversal and match nothing.
-      - type: regex_parser
-        parse_from: attributes["log.file.path"]
-        parse_to: attributes
-        regex: "^/var/log/xrpld/(?P<node_dir>[^/]+)/"
-      - type: move
-        from: attributes.node_dir
-        to: resource["service.instance.id"]
-      # Drop the raw path once the node name is on the resource. Keeping it
-      # would add a structured-metadata field to every line for no benefit.
-      - type: remove
-        field: attributes["log.file.path"]
-
-processors:
-  batch:
-    timeout: 1s
-    send_batch_size: 100
-  # Batching for the trace-storage pipeline only. Each request to Grafana
-  # Cloud costs a network round trip, so 100-span requests cap how fast
-  # stored traces can leave, and at high sampling the cloud exporter's queue
-  # fills and drops spans. send_batch_max_size caps one request: keep a full
-  # batch under the local Tempo receiver's 4 MiB gRPC message limit.
-  batch/store:
-    timeout: 1s
-    send_batch_size: 2048
-    send_batch_max_size: 2048
-  # Probabilistic tail sampling, on the trace-storage branch only. The
-  # spanmetrics branch sees every span, so span-derived RED metrics stay
-  # exact while stored trace volume is cut to bound cost on a long-running
-  # node. Operators who need full traces raise sampling_percentage to 100.
-  # Note: a log line carries a trace id only when its span was sampled at
-  # the SDK, so sampling here drops traces whose ids are already in the
-  # logs, and log-to-trace links for those resolve to nothing.
-  tail_sampling:
-    decision_wait: 10s
-    # A trace waits in memory for decision_wait plus up to one 1s tick
-    # before it is decided. If more than num_traces new traces arrive in that
-    # time, the oldest are dropped undecided and counted in
-    # sampling_trace_dropped_too_early. Size it as peak new traces per second
-    # x (decision_wait + 1s) x 2. Spans per second is a safe stand-in for new
-    # traces per second, since every trace has at least one span. 300000
-    # covers a peak of about 13,600 new traces per second. Each waiting trace
-    # costs a few KB, so a full table is about 1 GB.
-    num_traces: 300000
-    # With a decision cache, a decided trace is freed at once and only its id
-    # is kept here, which is smaller. Without one, decided traces stay until
-    # newer traces push them out, so sampling_traces_on_memory sits at
-    # num_traces and cannot warn before drops start. A late span still gets
-    # its trace's decision. Keep each cache at least num_traces, so a late
-    # span is matched for at least as long as it would be without a cache.
-    decision_cache:
-      sampled_cache_size: 300000
-      non_sampled_cache_size: 300000
-    policies:
-      - name: half-percent
-        type: probabilistic
-        probabilistic:
-          sampling_percentage: 0.5
-      # A nodestore rotation is the trace that proves a full->syncing flap,
-      # and it is rare, so keep every one rather than leaving it to the
-      # probabilistic policy. Children arrive before the root, so match on
-      # the family prefix rather than the root name.
-      - name: keep-rotation-traces
-        type: ottl_condition
-        ottl_condition:
-          error_mode: ignore
-          span:
-            - 'IsMatch(name, "^nodestore\\.rotate")'
-  resource/logs:
-    attributes:
-      - key: service.name
-        value: xrpld
-        action: upsert
-      - key: job
-        value: xrpld
-        action: upsert
-  # Deployment-tier tagging. Each collector serves ONE environment and ONE
-  # network, so it stamps both onto every signal it forwards to Grafana
-  # Cloud, letting one cloud stack hold data from many collectors and
-  # filter by tier.
-  #   - deployment.environment: the collector IS the environment (local, ci,
-  #     test, prod), so it is authoritative -> upsert (overwrite).
-  #   - xrpl.network.type: the xrpld node knows its own chain and already
-  #     stamps this, so the collector only fills it when absent -> insert.
-  #     This keeps a node's real network (e.g. a local node on mainnet)
-  #     from being overwritten by a collector's default.
-  # Replace the placeholder values per collector; see the "Deployment
-  # Tiers" section of docs/telemetry-runbook.md.
-  resource/tier:
-    attributes:
-      - key: deployment.environment
-        value: local
-        action: upsert
-      - key: xrpl.network.type
-        value: mainnet
-        action: insert
-  # Strip SDK-injected resource attributes (telemetry.sdk.language/name/version).
-  # The OpenTelemetry SDK auto-adds these to every Resource; they carry no
-  # operational value, so drop them for every signal on every backend.
-  resource/stripsdk:
-    attributes:
-      - key: telemetry.sdk.language
-        action: delete
-      - key: telemetry.sdk.name
-        action: delete
-      - key: telemetry.sdk.version
-        action: delete
-  # Grafana Cloud ingests metrics via OTLP (no Prometheus scrape), so the
-  # tier/instance resource attributes never become series labels the way the
-  # local prometheus exporter promotes them (resource_to_telemetry_conversion
-  # is Prometheus-exporter-only). Copy them onto datapoint labels so the
-  # dashboards' $node / $deployment_environment / $xrpl_network_type filters
-  # resolve on Cloud exactly as they do locally.
-  #
-  # NOTE: service.name is intentionally NOT copied here. Cloud's OTLP ingest
-  # already promotes service.name to the `service_name` label, and adding a
-  # datapoint attribute with the same name causes Mimir to concatenate both
-  # values as `xrpld;xrpld`. service.instance.id does not exhibit the same
-  # doubling, so the copy for $node is kept.
-  transform/cloudlabels:
-    metric_statements:
-      - context: datapoint
-        statements:
-          - set(attributes["service_instance_id"], resource.attributes["service.instance.id"])
-          - set(attributes["xrpl_node_id"], resource.attributes["xrpl.node.id"])
-          - set(attributes["deployment_environment"], resource.attributes["deployment.environment"])
-          - set(attributes["xrpl_network_type"], resource.attributes["xrpl.network.type"])
-
-connectors:
-  spanmetrics:
-    namespace: "span"
-    # Emit span-derived metrics every 15s (default is 60s). Grafana Cloud
-    # ingests these as one datapoint per flush, and dashboard rate() panels
-    # use [$__rate_interval], which at short ranges (<=1h) can be shorter than
-    # the flush interval. A 60s flush left <2 samples per rate window on Cloud,
-    # so those panels rendered "No data". 15s keeps >=2 samples per window at
-    # every dashboard range, at the cost of ~4x span-metric datapoints to Cloud.
-    metrics_flush_interval: 15s
-    # Resource attributes that define the resource-metrics grouping key.
-    # All resource attributes propagate to the output metrics regardless;
-    # label promotion is handled by resource_to_telemetry_conversion on the
-    # prometheus exporter. Listing the per-node id and tier keys keeps
-    # series from distinct nodes/tiers grouped separately.
-    resource_metrics_key_attributes:
-      - service.instance.id
-      - xrpl.node.id
-      - deployment.environment
-      - xrpl.network.type
-    histogram:
-      # Pin unit=ms so the metric stays span_duration_milliseconds_* and le
-      # labels stay in ms even if a future collector flips the default to
-      # seconds (connector.spanmetrics.useSecondAsDefaultMetricsUnit gate).
-      unit: ms
-      explicit:
-        # Buckets MUST stay strictly ascending (the connector binary-searches
-        # them and silently misbuckets otherwise) and MUST match the Alloy
-        # list and the other collector config. Sub-millisecond boundaries
-        # exist because most xrpld spans sit far below 1ms: a 1ms floor puts
-        # nearly every sample in the first bucket and pins p95/p99 to that
-        # bucket's edge. Sub-second boundaries cover RPC, tx and ledger spans.
-        # The 2s-4s edges resolve second-scale consensus spans, and 10s/30s
-        # give the ledger.acquire catch-up tail a home so its p95/p99 stop
-        # reading as +Inf. Each edge was chosen from a measured duration
-        # distribution for the spans it covers.
-        buckets:
-          [
-            0.01ms,
-            0.05ms,
-            0.1ms,
-            0.25ms,
-            0.5ms,
-            1ms,
-            5ms,
-            10ms,
-            25ms,
-            50ms,
-            100ms,
-            250ms,
-            500ms,
-            1s,
-            2s,
-            3s,
-            4s,
-            5s,
-            10s,
-            30s,
-          ]
-    dimensions:
-      - name: command
-      - name: rpc_status
-      - name: consensus_mode
-      # Only the boolean close_time_correct is promoted, never the
-      # close_time_ripple_epoch_s value itself: that attribute is a monotonic
-      # NetClock timestamp, so a dimension on it would mint a new metric series
-      # every ledger (unbounded cardinality). The close-time value is exposed as
-      # the server_info last_close_time gauge instead (for last-close age); the
-      # close interval comes from the ledgers_closed_total counter.
-      - name: close_time_correct
-      - name: consensus_state
-      - name: local
-      - name: proposal_trusted
-      - name: validation_trusted
-      - name: tx_type
-      - name: ter_result
-      - name: stage
-      - name: txq_status
-      - name: load_type
-      - name: is_batch
-      - name: mode_new
-      - name: consensus_stalled
-      - name: consensus_phase
-      - name: consensus_result
-      - name: method
-      - name: grpc_role
-      - name: grpc_status
-      # ledger.acquire dimensions. Must stay identical to the local
-      # otel-collector-config.yaml dimension set; see the cardinality note
-      # there for why ledger_hash / ledger_seq are span-only (per-ledger values
-      # would mint one metric series per ledger) and are indexed in Tempo
-      # instead. `outcome` includes `abandoned`, recorded when an acquire is
-      # swept or shut down before reaching a result.
-      - name: outcome
-      - name: acquire_reason
-      # Sync-diagnostic span dimensions. Must stay identical to the
-      # local otel-collector-config.yaml dimension set; see the cardinality
-      # note there for why txset_hash / remote_endpoint / missing_nodes /
-      # served_nodes / duration_ms are span-only (per-round, per-peer or
-      # per-integer values would each mint their own metric series) and are
-      # indexed in Tempo instead. `outcome` above is reused verbatim by all of
-      # txset.acquire, ledger.acquire.*, peer.dial and ledger.serve.
-      - name: timed_out
-      - name: object_type
-      # Trace-causality span dimensions, on
-      # consensus.validation.accept. Must stay identical to the local
-      # otel-collector-config.yaml dimension set; see the reasoning there for
-      # what each value set is and why the ledger_hash join key stays span-only
-      # (a per-ledger value would mint one metric series per ledger validated)
-      # and is indexed in Tempo instead. consensus_round_duration_ms needs no
-      # entry: it is a native histogram, so the collector only forwards it.
-      - name: validation_status
-      - name: accept_gated
-
-exporters:
-  otlp/tempo:
-    endpoint: tempo:4317
-    tls:
-      insecure: true
-    # Fed by batch/store, so size the queue in spans: 1000 requests of 2048
-    # spans would let a stalled Tempo hold about two million spans in memory.
-    sending_queue:
-      sizer: items
-      queue_size: 100000
-  otlp_http/loki:
-    endpoint: http://loki:3100/otlp
-  prometheus:
-    endpoint: 0.0.0.0:8889
-    # Promote resource attributes (deployment.environment, xrpl.network.type,
-    # service.name, service.instance.id) to Prometheus labels so the local
-    # scrape surface can be filtered by tier. (Grafana Cloud ingests the
-    # attributes natively via OTLP, so this only affects the local export.)
-    resource_to_telemetry_conversion:
-      enabled: true
-  # OTLP/HTTP exporter to Grafana Cloud for metrics and logs; stored traces
-  # use otlp_http/grafanacloud-traces below. The gateway fans the signals out
-  # to hosted Tempo (traces), Mimir/Prometheus (metrics), and Loki (logs).
-  # Retry + queue guard against transient gateway errors.
-  otlp_http/grafanacloud:
-    endpoint: ${env:GRAFANA_CLOUD_OTLP_ENDPOINT}
-    auth:
-      authenticator: basicauth/grafanacloud
-    retry_on_failure:
-      enabled: true
-    sending_queue:
-      enabled: true
-  # Stored traces get their own Grafana Cloud exporter, so their queue can be
-  # sized in spans without shrinking the metrics and logs queues above, whose
-  # requests vary in size. Throughput is about num_consumers x batch size per
-  # round trip to the gateway. The queue counts spans still being sent, so
-  # more consumers do not add memory.
-  otlp_http/grafanacloud-traces:
-    endpoint: ${env:GRAFANA_CLOUD_OTLP_ENDPOINT}
-    auth:
-      authenticator: basicauth/grafanacloud
-    retry_on_failure:
-      enabled: true
-    sending_queue:
-      enabled: true
-      sizer: items
-      queue_size: 100000
-      num_consumers: 20
-
-service:
-  extensions: [health_check, basicauth/grafanacloud]
-  pipelines:
-    # Local and Grafana Cloud exporters run side by side. For cloud-only,
-    # remove otlp/tempo from traces/store and otlp_http/loki from logs, and
-    # delete metrics/local (its only exporter is prometheus).
-    # 100% of spans feed the spanmetrics connector so span-derived RED
-    # metrics stay exact. No tail sampling on this branch.
-    traces/metrics:
-      receivers: [otlp]
-      processors: [resource/tier, resource/stripsdk, batch]
-      exporters: [spanmetrics]
-    # Trace-STORAGE branch: 0.5% probabilistic tail sampling before Tempo
-    # and Grafana Cloud, so stored trace volume is ~1/200 of ingested spans.
-    traces/store:
-      receivers: [otlp]
-      processors: [tail_sampling, resource/tier, resource/stripsdk, batch/store]
-      exporters: [otlp/tempo, otlp_http/grafanacloud-traces]
-    # The local Prometheus scrape promotes tier/instance resource attrs to
-    # labels via resource_to_telemetry_conversion; Grafana Cloud (OTLP) does
-    # not, so it runs a separate pipeline that copies them onto datapoint
-    # labels via transform/cloudlabels. Splitting avoids the local scrape and
-    # the transform both writing a service_instance_id label on the same series.
-    metrics/local:
-      receivers: [otlp, spanmetrics]
-      processors: [resource/tier, resource/stripsdk, batch]
-      exporters: [prometheus]
-    metrics/cloud:
-      receivers: [otlp, spanmetrics]
-      processors:
-        [resource/tier, resource/stripsdk, transform/cloudlabels, batch]
-      exporters: [otlp_http/grafanacloud]
-    logs:
-      receivers: [file_log]
-      processors: [resource/logs, resource/tier, resource/stripsdk, batch]
-      exporters: [otlp_http/loki, otlp_http/grafanacloud]
-  # The collector's own metrics (dropped traces, export queues and more).
-  # The periodic reader posts them to this collector's OTLP receiver, so they
-  # take the node metrics' path: resource/tier stamps deployment.environment
-  # on them, metrics/local serves them to the local Prometheus scrape and
-  # metrics/cloud sends them to Grafana Cloud. Naming the receiver by its
-  # compose service name, like the tempo and loki exporters, works whichever
-  # interface it binds. Declaring readers drops the default one, so the pull
-  # reader keeps the in-container localhost:8888 page. At shutdown the
-  # receiver stops first, so the reader's last push fails and the collector
-  # exits with status 1.
-  telemetry:
-    resource:
-      attributes:
-        # A fixed id, so a restart keeps the same series. The default is a new
-        # random id per start. Give each collector that shares a
-        # deployment.environment its own value.
-        - name: service.instance.id
-          value: otel-collector
-    metrics:
-      readers:
-        # 15s, the same as the spanmetrics flush and for the same reason.
-        - periodic:
-            interval: 15000
-            exporter:
-              otlp:
-                protocol: http/protobuf
-                endpoint: http://otel-collector:4318
-        - pull:
-            exporter:
-              prometheus:
-                host: localhost
-                port: 8888

docker/telemetry/otel-collector-config.yaml

@@ -1,354 +0,0 @@
-# OpenTelemetry Collector configuration for xrpld development.
-#
-# Pipelines:
-#   traces: OTLP receiver -> batch processor -> debug + Tempo + span_metrics
-#   metrics: OTLP receiver + span_metrics connector -> Prometheus exporter
-#   logs:   file_log receiver -> batch processor -> otlp_http/Loki
-#
-# xrpld sends traces via OTLP/HTTP to port 4318. The collector batches
-# them, forwards to Tempo, and derives RED metrics via the span_metrics
-# connector, which Prometheus scrapes on port 8889.
-#
-# xrpld sends beast::insight metrics natively via OTLP/HTTP to port 4318
-# (same endpoint as traces). The OTLP receiver feeds both the traces and
-# metrics pipelines. Metrics are exported to Prometheus alongside
-# span-derived metrics.
-#
-# The file_log receiver tails xrpld's debug.log files under
-# /var/log/xrpld/ (mounted from the host). A regex_parser operator
-# extracts timestamp, partition, severity, and optional trace_id/span_id
-# fields injected by Logs::format(). Parsed logs are exported to Grafana
-# Loki for log-trace correlation.
-
-receivers:
-  otlp:
-    protocols:
-      grpc:
-        endpoint: 0.0.0.0:4317
-      http:
-        endpoint: 0.0.0.0:4318
-  # Filelog receiver tails xrpld debug.log files for log-trace
-  # correlation. Extracts structured fields (timestamp, partition, severity,
-  # trace_id, span_id, message) via regex. The trace_id and span_id are
-  # optional — only present when the log was emitted within an active span.
-  file_log:
-    include: [/var/log/xrpld/*/debug.log]
-    # Needed to recover which node a line came from. The subdirectory name is
-    # the only per-node signal in the log stream: Logs::format() writes
-    # trace_id and span_id but no node identity. Emitters name the directory
-    # after their own service_instance_id so the two agree.
-    include_file_path: true
-    # Read each file from the start. The upstream default is `end`, which
-    # skips everything written before the receiver's first poll — so any log
-    # line a node emitted before the collector got to it would be lost, and
-    # nothing is read at all from a file that has stopped being written to.
-    #
-    # Offsets are kept in memory here, so a restarted collector re-reads the
-    # file. Stacks that keep their logs across restarts layer
-    # otel-collector-filestorage.yaml on top to persist them; ephemeral
-    # stacks get a fresh log directory each run and need nothing.
-    start_at: beginning
-    operators:
-      # Log format emitted by Logs::format() is:
-      #   YYYY-Mmm-DD HH:MM:SS.fffffffff UTC <partition>:<severity> [trace_id=... span_id=...] <message>
-      # The `partition:` prefix is omitted when partition is empty, so the
-      # capture group is non-capturing optional. The node emits nanosecond
-      # precision (9 digits); `%f` accepts any number of fractional digits.
-      - type: regex_parser
-        regex: '^(?P<timestamp>\S+\s+\S+)\s+\S+\s+(?:(?P<partition>\S+):)?(?P<severity>\S+)\s+(?:trace_id=(?P<trace_id>[a-f0-9]+)\s+span_id=(?P<span_id>[a-f0-9]+)\s+)?(?P<message>.*)$'
-        timestamp:
-          parse_from: attributes.timestamp
-          layout: "%Y-%b-%d %H:%M:%S.%f"
-          location: UTC
-      # Lift the per-node directory out of the file path and onto the
-      # RESOURCE. include_file_path alone is not enough: it produces a log
-      # RECORD attribute, and on OTLP ingest Loki promotes only an allow-list
-      # of RESOURCE attributes to indexed stream labels. A record attribute
-      # becomes structured metadata, which cannot be used in a {...} selector.
-      # service.instance.id is on that allow-list and arrives as the LogQL
-      # label service_instance_id, which is the label the dashboards filter on.
-      # Dotted keys need bracket syntax; dot notation would be read as a
-      # nested traversal and match nothing.
-      - type: regex_parser
-        parse_from: attributes["log.file.path"]
-        parse_to: attributes
-        regex: "^/var/log/xrpld/(?P<node_dir>[^/]+)/"
-      - type: move
-        from: attributes.node_dir
-        to: resource["service.instance.id"]
-      # Drop the raw path once the node name is on the resource. Keeping it
-      # would add a structured-metadata field to every line for no benefit.
-      - type: remove
-        field: attributes["log.file.path"]
-
-processors:
-  batch:
-    timeout: 1s
-    send_batch_size: 100
-  resource/logs:
-    attributes:
-      # Loki 3.x OTLP ingestion promotes only its own allow-list of resource
-      # attributes to stream (index) labels; `service.name` is on that list
-      # and arrives as the label `service_name`, which is what the LogQL
-      # examples in the runbook and TESTING.md select on.
-      #
-      # A custom `job` attribute is NOT on that list: the pinned
-      # grafana/loki:3.7.6 prints an 18-key default list under
-      # limits_config.otlp_config, and `job` is not one of them. Ingesting
-      # through this pipeline, /loki/api/v1/labels returned only `service_name`
-      # and `deployment_environment`, `{job="xrpld"}` matched 0 streams, and
-      # `job` appeared as structured metadata instead — which a `{...}`
-      # stream selector cannot match. Promoting it would mean mounting a Loki
-      # config and adding it to limits_config.otlp_config.resource_attributes
-      # (additive to Loki's defaults unless ignore_defaults is set), which is
-      # not worth a constant value — especially as Loki caps index labels at
-      # 15 and already promotes 18 by default. Select on `service_name`.
-      - key: service.name
-        value: xrpld
-        action: upsert
-  # Deployment-tier tagging. Each collector serves ONE environment and ONE
-  # network, so it stamps both onto every signal it forwards. This lets a
-  # single Grafana stack hold data from many collectors and filter by tier.
-  #   - deployment.environment: the collector IS the environment (local, ci,
-  #     test, prod), so it is authoritative -> upsert (overwrite).
-  #   - xrpl.network.type: the xrpld node knows its own chain and already
-  #     stamps this, so the collector only fills it when absent -> insert.
-  #     This keeps a node's real network (e.g. a local node on mainnet)
-  #     from being overwritten by a collector's default.
-  # Replace the placeholder values per collector; see docker/telemetry
-  # tier examples.
-  resource/tier:
-    attributes:
-      - key: deployment.environment
-        value: local
-        action: upsert
-      - key: xrpl.network.type
-        value: mainnet
-        action: insert
-  # Strip SDK-injected resource attributes (telemetry.sdk.language/name/version).
-  # The OpenTelemetry SDK auto-adds these to every Resource; they carry no
-  # operational value and clutter the attribute set on every backend, so drop
-  # them here for all signals.
-  resource/stripsdk:
-    attributes:
-      - key: telemetry.sdk.language
-        action: delete
-      - key: telemetry.sdk.name
-        action: delete
-      - key: telemetry.sdk.version
-        action: delete
-  # No attribute hashing or redaction. Account addresses in span attributes
-  # (pathfind_source_account, pathfind_dest_account) are public ledger
-  # identifiers and are stored as emitted so they join against explorers and
-  # logs. Do not add an attributes/hash processor for them.
-
-connectors:
-  span_metrics:
-    namespace: "span"
-    # Resource attributes that define the resource-metrics grouping key.
-    # All resource attributes are propagated onto the output metrics
-    # regardless; promotion to Prometheus labels is handled by the
-    # prometheus exporter's resource_to_telemetry_conversion. Listing the
-    # per-node id and the deployment-tier keys keeps series from distinct
-    # nodes/tiers grouped separately.
-    resource_metrics_key_attributes:
-      - service.instance.id
-      - xrpl.node.id
-      - deployment.environment
-      - xrpl.network.type
-    histogram:
-      # Pin unit=ms so the metric stays span_duration_milliseconds_* and le
-      # labels stay in ms even if a future collector flips the default to
-      # seconds (connector.spanmetrics.useSecondAsDefaultMetricsUnit gate).
-      unit: ms
-      explicit:
-        # Buckets MUST stay strictly ascending (the connector binary-searches
-        # them and silently misbuckets otherwise) and MUST match the Alloy
-        # list and the other collector config. Sub-millisecond boundaries
-        # exist because most xrpld spans sit far below 1ms: a 1ms floor puts
-        # nearly every sample in the first bucket and pins p95/p99 to that
-        # bucket's edge. Sub-second boundaries cover RPC, tx and ledger spans.
-        # The 2s-4s edges resolve second-scale consensus spans, and 10s/30s
-        # give the ledger.acquire catch-up tail a home so its p95/p99 stop
-        # reading as +Inf. Each edge was chosen from a measured duration
-        # distribution for the spans it covers.
-        buckets:
-          [
-            0.01ms,
-            0.05ms,
-            0.1ms,
-            0.25ms,
-            0.5ms,
-            1ms,
-            5ms,
-            10ms,
-            25ms,
-            50ms,
-            100ms,
-            250ms,
-            500ms,
-            1s,
-            2s,
-            3s,
-            4s,
-            5s,
-            10s,
-            30s,
-          ]
-    dimensions:
-      - name: command
-      - name: rpc_status
-      - name: consensus_mode
-      # Only the boolean close_time_correct is promoted, never the
-      # close_time_ripple_epoch_s value itself: that attribute is a monotonic
-      # NetClock timestamp, so a dimension on it would mint a new metric series
-      # every ledger (unbounded cardinality). The close-time value is exposed as
-      # the server_info last_close_time gauge instead (for last-close age); the
-      # close interval comes from the ledgers_closed_total counter.
-      - name: close_time_correct
-      - name: consensus_state
-      - name: local
-      - name: proposal_trusted
-      - name: validation_trusted
-      - name: tx_type
-      - name: ter_result
-      # Apply-pipeline stage (preflight|preclaim|apply) — splits the
-      # tx.preflight/tx.preclaim/tx.transactor span RED metrics per stage.
-      - name: stage
-      - name: txq_status
-      - name: load_type
-      - name: is_batch
-      # Consensus lifecycle dimensions (low cardinality, bounded value sets).
-      - name: mode_new
-      - name: consensus_stalled
-      - name: consensus_phase
-      - name: consensus_result
-      # gRPC surface dimensions (bounded: method names, role, status).
-      - name: method
-      - name: grpc_role
-      - name: grpc_status
-      # ledger.acquire dimensions (bounded: outcome, acquire reason).
-      # `outcome` carries the four terminal values complete|failed|timeout|
-      # abandoned. `abandoned` is the one recorded when an acquire is destroyed
-      # (swept or shut down) before reaching a result, so a stuck-then-swept
-      # fetch is counted here rather than silently dropped from the rate.
-      # Deliberately NOT promoted: ledger_hash and ledger_seq. Both are
-      # per-ledger values, so a dimension on either would mint a new metric
-      # series for every ledger acquired -- the same unbounded-cardinality rule
-      # already applied to close_time_ripple_epoch_s above. They stay span-only
-      # and are searchable in Tempo instead (see the parquet_dedicated_columns
-      # block in tempo.yaml), which gives per-ledger lookup with no metric
-      # series.
-      - name: outcome
-      - name: acquire_reason
-      # Sync-diagnostic span dimensions. Each is a small, closed value
-      # set, so each adds a bounded number of series per span name:
-      #   timed_out    - 2 values, on the ledger.acquire.* phase spans. Split
-      #                  by it and a phase-duration panel separates "slow but
-      #                  arriving" from "the retry budget ran out".
-      #   object_type  - 4 values (header|tx|as|txset), on ledger.serve. Which
-      #                  request kind this node serves its peers, so shedding
-      #                  account-state tree requests is visible apart from the
-      #                  cheap header replies.
-      # `outcome` above is reused verbatim by the four new spans
-      # (txset.acquire, the three ledger.acquire.* phases, peer.dial,
-      # ledger.serve), so it needs no second entry -- one dimension covers
-      # every span's outcome rate.
-      #
-      # Deliberately NOT promoted, same unbounded-cardinality rule as
-      # ledger_hash / ledger_seq above:
-      #   txset_hash      - one value per proposed tx set (per consensus round)
-      #   remote_endpoint - one value per peer address dialled
-      #   missing_nodes   - an arbitrary node count
-      #   served_nodes    - an arbitrary reply size
-      #   duration_ms     - an arbitrary elapsed time
-      #   current_ledger_hash / current_ledger_seq - the txset.acquire
-      #                     `round.request` event attributes, one value per
-      #                     requesting round. Per-round, so unbounded by the
-      #                     same rule -- but more fundamentally impossible:
-      #                     spanmetrics derives metrics from spans, not span
-      #                     events, so an event attribute cannot be a metric
-      #                     dimension at all. Read them in Tempo only.
-      # A dimension on any of these would mint a metric series per round, per
-      # peer or per distinct integer. They stay span-only and are searchable in
-      # Tempo instead (see parquet_dedicated_columns in tempo.yaml); the
-      # aggregate view of the same quantities comes from the spanmetrics
-      # duration histogram and from the native acquire gauges.
-      - name: timed_out
-      - name: object_type
-      # Trace-causality span dimensions, on
-      # consensus.validation.accept. Both are closed value sets:
-      #   validation_status - 6 values (the 5 ValStatus outcomes plus an
-      #                       unknown sentinel). Only `current` reaches the
-      #                       acceptance gate, so splitting by it separates a
-      #                       node whose arriving validations are counting from
-      #                       one whose validations are all rejected -- which
-      #                       from the outside look identical, both being a
-      #                       node that receives validations and never
-      #                       validates.
-      #   accept_gated      - 2 values. True when this validation did not reach
-      #                       the gate because another thread was already
-      #                       accepting the same ledger, which is why a trace
-      #                       can show a validation with no ledger.validate
-      #                       after it. Without the dimension that gap looks
-      #                       like a lost span.
-      #
-      # Deliberately NOT promoted, same rule as ledger_hash / ledger_seq above:
-      # nothing here carries a per-ledger value. The join key itself
-      # (ledger_hash) is exactly such a value and stays span-only, indexed in
-      # tempo.yaml, because the per-ledger lookup it exists for is a trace
-      # search, not a metric query -- and as a dimension it would mint one
-      # series per ledger validated.
-      #
-      # consensus_round_duration_ms needs NO entry here at all: it is a native
-      # histogram from the OTel SDK, so it rides the existing OTLP -> Prometheus
-      # path and the collector only forwards it. Only span-derived signals need
-      # a dimension.
-      - name: validation_status
-      - name: accept_gated
-
-exporters:
-  debug:
-    verbosity: detailed
-  otlp_grpc/tempo:
-    endpoint: tempo:4317
-    tls:
-      insecure: true
-  # Export logs to Grafana Loki via OTLP/HTTP. Loki 3.x supports
-  # native OTLP ingestion on its /otlp endpoint, replacing the removed
-  # loki exporter (dropped in otel-collector-contrib v0.147.0).
-  otlp_http/loki:
-    endpoint: http://loki:3100/otlp
-  prometheus:
-    endpoint: 0.0.0.0:8889
-    # Promote resource attributes (deployment.environment, xrpl.network.type,
-    # service.name, service.instance.id) to Prometheus labels. Without this
-    # the tier attributes stamped by resource/tier stay on the resource and
-    # never reach the metric series, so dashboards could not filter by them.
-    resource_to_telemetry_conversion:
-      enabled: true
-
-extensions:
-  health_check:
-    endpoint: 0.0.0.0:13133
-
-service:
-  extensions: [health_check]
-  pipelines:
-    traces:
-      receivers: [otlp]
-      processors: [resource/tier, resource/stripsdk, batch]
-      exporters: [debug, otlp_grpc/tempo, span_metrics]
-    metrics:
-      receivers: [otlp, span_metrics]
-      # batch keeps the OTLP metric path from exporting one request per
-      # instrument. It delays a sample by at most the batch timeout, which
-      # is well under the Prometheus scrape interval.
-      processors: [resource/tier, resource/stripsdk, batch]
-      exporters: [prometheus]
-    # Log pipeline ingests xrpld debug.log via file_log receiver,
-    # batches entries, and exports to Loki for log-trace correlation.
-    logs:
-      receivers: [file_log]
-      processors: [resource/logs, resource/tier, resource/stripsdk, batch]
-      exporters: [otlp_http/loki]

docker/telemetry/otel-collector-filestorage.grafanacloud.yaml

@@ -1,26 +0,0 @@
-# file_log offset persistence for the Grafana Cloud stack.
-#
-# Same purpose as otel-collector-filestorage.yaml, and needed for the same
-# reason, but it cannot be reused as-is. The collector replaces lists on merge
-# rather than appending, so whichever config is passed last decides
-# service.extensions outright. The base variant's list,
-# [health_check, file_storage/file_log], drops basicauth/grafanacloud and
-# leaves the cloud exporters unable to resolve their authenticator. The
-# collector then exits at startup with "authenticator not found".
-#
-# Applied by docker-compose.grafanacloud.yaml, which mounts it over the base
-# variant so the command line stays unchanged. Keep the two files in step.
-
-extensions:
-  file_storage/file_log:
-    directory: /var/lib/otelcol/file_storage
-    create_directory: true
-
-receivers:
-  file_log:
-    storage: file_storage/file_log
-
-# Must name every extension the cloud config needs, not just this file's own:
-# lists are replaced, so anything omitted here is omitted from the collector.
-service:
-  extensions: [health_check, file_storage/file_log, basicauth/grafanacloud]

docker/telemetry/otel-collector-filestorage.yaml

@@ -1,31 +0,0 @@
-# Collector overlay that persists file_log read offsets. Applied ONLY by the
-# developer stack (docker/telemetry/docker-compose.yml), as a second --config
-# after otel-collector-config.yaml; the collector deep-merges the two.
-#
-# Why this is an overlay rather than part of the base config: the base config
-# is shared by every stack that runs the collector, including the ephemeral
-# workload-validation stack, which creates a fresh log directory per run and so
-# has nothing to resume from. The extension needs a writable directory, and the
-# collector image runs as 10001:10001 with no writable path of its own, so
-# requiring it in the base config would force every stack to mount a volume
-# just to start. Keeping it here means the base config stays self-sufficient.
-#
-# The developer stack benefits because its log directory and this volume both
-# survive `docker compose down`, so a restart resumes at the last offset
-# instead of re-reading debug.log from the top.
-#
-# The Grafana Cloud stack mounts otel-collector-filestorage.grafanacloud.yaml
-# over this file, because it needs one more extension. Keep the two in step.
-
-extensions:
-  file_storage/file_log:
-    directory: /var/lib/otelcol/file_storage
-    create_directory: true
-
-receivers:
-  file_log:
-    storage: file_storage/file_log
-
-# Lists are replaced rather than merged, so this must repeat the base entry.
-service:
-  extensions: [health_check, file_storage/file_log]

docker/telemetry/prometheus.yml

@@ -1,9 +0,0 @@
-# Prometheus configuration for scraping span_metrics from OTel Collector.
-global:
-  scrape_interval: 15s
-  evaluation_interval: 15s
-
-scrape_configs:
-  - job_name: otel-collector
-    static_configs:
-      - targets: ["otel-collector:8889"]

docker/telemetry/tempo.yaml

@@ -1,149 +0,0 @@
-# Grafana Tempo configuration for xrpld telemetry stack.
-#
-# Runs in single-binary mode for local development.
-# Receives traces via OTLP/gRPC from the OTel Collector and stores
-# them locally. Queryable via Grafana Explore using the Tempo datasource.
-#
-# Search filters are configured on the Grafana datasource side
-# (grafana/provisioning/datasources/tempo.yaml). Tempo auto-indexes
-# all span attributes for search in single-binary mode.
-#
-# For production, replace local storage with S3/GCS backend and adjust
-# retention via the compactor settings. See:
-# https://grafana.com/docs/tempo/latest/configuration/
-
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-
-# Raise the TraceQL metrics query range limit. The default
-# query_frontend.metrics.max_duration is 3h, so a dashboard set to a longer
-# window (e.g. 6h/12h) fails with "range exceeds 3h0m0s". 168h matches the
-# search max_duration and gives dashboards generous headroom.
-query_frontend:
-  metrics:
-    max_duration: 168h
-
-distributor:
-  receivers:
-    otlp:
-      protocols:
-        grpc:
-          endpoint: 0.0.0.0:4317
-
-ingester:
-  max_block_duration: 5m
-
-compactor:
-  compaction:
-    block_retention: 1h
-
-# Enable metrics generator for service graph, span metrics, and the
-# local-blocks processor. Produces RED metrics (rate, errors, duration) per
-# service/span for the service map, and keeps recent trace blocks queryable so
-# TraceQL metrics queries (quantile_over_time, count_over_time, etc. via
-# /api/metrics/query_range) work.
-metrics_generator:
-  registry:
-    external_labels:
-      source: tempo
-  storage:
-    path: /var/tempo/generator/wal
-    # Uncomment and add a Prometheus service to docker-compose.yml
-    # to enable remote_write for service graph metrics:
-    # remote_write:
-    #   - url: http://prometheus:9090/api/v1/write
-  # Separate WAL the local-blocks processor flushes traces to for metrics
-  # queries. Required when flush_to_storage is true.
-  traces_storage:
-    path: /var/tempo/generator/traces
-  processor:
-    local_blocks:
-      # xrpld consensus/transaction spans are SPAN_KIND_INTERNAL. By default
-      # local-blocks keeps only server spans for TraceQL metrics, so attribute
-      # aggregations over internal spans return nothing. Keep all spans.
-      filter_server_spans: false
-      # Flush recent blocks to traces_storage so query_range can read them.
-      flush_to_storage: true
-
-overrides:
-  defaults:
-    metrics_generator:
-      processors:
-        - service-graphs
-        - span-metrics
-        - local-blocks
-
-storage:
-  trace:
-    backend: local
-    wal:
-      path: /var/tempo/wal
-    local:
-      path: /var/tempo/blocks
-    block:
-      # Give the highest-value span attributes their own Parquet columns so
-      # TraceQL scopes a search to one ledger without scanning the generic
-      # attribute list. Tempo indexes every attribute for search regardless;
-      # a dedicated column only changes the cost of filtering on it.
-      #
-      # ledger_hash is here rather than in the collector's spanmetrics
-      # `dimensions:` on purpose: it is a per-ledger value, so as a metric
-      # dimension it would mint one series per ledger acquired (unbounded
-      # cardinality). As a trace column it gives the same per-ledger lookup at
-      # zero metric cost -- which is how a single stuck or abandoned
-      # ledger.acquire is found: {name="ledger.acquire" && span.outcome=
-      # "abandoned" && span.ledger_hash="<hash>"}.
-      #
-      # txset_hash and remote_endpoint are here for the same reason and by the
-      # same rule: a tx-set root hash is one value per consensus round and a
-      # peer endpoint is one value per address dialled, so either as a
-      # spanmetrics dimension would mint an unbounded number of metric series.
-      # As trace columns they answer the per-object questions the metrics
-      # cannot -- which proposed set stalled a round:
-      #   {name="txset.acquire" && span.outcome="timeout" &&
-      #    span.txset_hash="<hash>"}
-      # and which peer a dial keeps failing against:
-      #   {name="peer.dial" && span.outcome="tcp_fail" &&
-      #    span.remote_endpoint="10.0.0.5:51235"}
-      #
-      # missing_nodes, served_nodes and duration_ms are deliberately NOT given
-      # columns. They are unbounded integers, so they are useless as equality
-      # filters; Tempo indexes every attribute for search regardless, so they
-      # remain queryable by range (e.g. span.missing_nodes > 0) at no
-      # storage cost, which is how they are actually read.
-      #
-      # The txset.acquire `round.request` event attributes get no column either,
-      # and cannot: this Tempo (2.9.4, docker-compose.yml) queries them via
-      # event:name / event.<attr>, but a dedicated column accepts only
-      # scope: span or scope: resource -- never event. Those event filters
-      # therefore scan rather than index, which is fine for the per-span lookup
-      # they are used for; do not add an event-scoped column to the list below.
-      #
-      # The ledger_hash column below carries a second job as of the per-ledger
-      # trace join, which is why it stays the highest-value column here.
-      # Every span for one ledger -- ledger.acquire, consensus.validation.accept,
-      # ledger.validate, ledger.store -- now derives its TRACE ID from that same
-      # ledger hash, so the whole story of one slow ledger is one trace:
-      #   {span.ledger_hash="<hash>"}
-      # returns all of them across the three threads that produced them. The
-      # column is what makes that lookup cheap; it also stays the way a reader
-      # confirms two spans really are the same ledger rather than a trace-id
-      # coincidence, since the attribute holds the full 32 bytes while the trace
-      # id is only the leading 16.
-      #
-      # validation_status and accept_gated get no column: both are small closed
-      # value sets promoted to spanmetrics dimensions instead (see the collector
-      # configs), so the aggregate question is answered by a metric query and no
-      # per-value trace lookup is needed.
-      parquet_dedicated_columns:
-        - name: ledger_hash
-          type: string
-          scope: span
-        - name: txset_hash
-          type: string
-          scope: span
-        - name: remote_endpoint
-          type: string
-          scope: span

docker/telemetry/workload/README.md

@@ -1,723 +0,0 @@
-# Telemetry Workload Tools
-
-Synthetic workload generation and validation tools for xrpld's OpenTelemetry telemetry stack. These tools validate that all spans, metrics, dashboards, and log-trace correlation work end-to-end under controlled load.
-
-## Quick Start
-
-```bash
-# Build xrpld with telemetry enabled (see BUILD.md for the full flow)
-mkdir -p .build && cd .build
-conan install .. --output-folder . --build missing \
-    --settings build_type=Release -o telemetry=True
-cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
-    -DCMAKE_BUILD_TYPE=Release -Dtelemetry=ON ..
-cmake --build . --parallel "$(nproc)" --target xrpld
-cd ..
-
-# Run full validation (starts everything, runs load, validates)
-docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld
-
-# Cleanup when done
-docker/telemetry/workload/run-full-validation.sh --cleanup
-```
-
-## Architecture
-
-The validation suite runs a multi-node xrpld cluster as local processes alongside
-a Docker Compose telemetry stack. The cluster exercises consensus, peer-to-peer
-spans (proposals, validations), and all metric pipelines.
-
-```
-run-full-validation.sh (shell orchestrator)
-  |
-  |-- docker-compose.workload.yaml
-  |     |-- otel-collector (otlp receiver: traces + beast::insight metrics;
-  |     |                  file_log receiver: node debug.log -> Loki)
-  |     |-- tempo (trace backend + TraceQL search API)
-  |     |-- prometheus (metrics scraping)
-  |     |-- loki (log aggregation for log-trace correlation)
-  |     |-- grafana (dashboards, provisioned automatically)
-  |
-  |-- generate-validator-keys.sh
-  |     -> validator-keys.json, validators.txt
-  |
-  |-- Nx xrpld nodes (local processes, full telemetry)
-  |     - Each node: [telemetry] enabled=1, all 5 trace_* categories on
-  |     - [insight] server=otel (beast::insight metrics over OTLP, no StatsD)
-  |     - [signing_support] true (server-side signing for tx_submitter)
-  |     - Peer discovery via [ips] (not [ips_fixed]) for active peer counts
-  |
-  |-- workload_orchestrator.py (phased load execution)
-  |     |-- rpc_load_generator.py (WebSocket RPC traffic)
-  |     |-- tx_submitter.py (transaction diversity)
-  |     -> workload-report.json + per-phase reports
-  |
-  |-- validate_telemetry.py (pass/fail checks)
-  |     -> validation-report.json
-  |
-  |-- benchmark.sh (baseline vs telemetry comparison)
-        |-- collect_system_metrics.sh (per-leg CPU/RSS/latency/TPS sampling)
-        -> benchmark-report-*.md
-```
-
-## Workload Profiles
-
-The workload orchestrator (`workload_orchestrator.py`) reads named profiles
-from `workload-profiles.json` and executes sequential load phases. Within
-each phase, the RPC generator and TX submitter run concurrently.
-
-### Available Profiles
-
-| Profile           | Phases | Duration                    | Purpose                                                                                          |
-| ----------------- | ------ | --------------------------- | ------------------------------------------------------------------------------------------------ |
-| `full-validation` | 7      | 4.5 min + 1 min propagation | Coverage for the full asserted span/metric/dashboard inventory, with burst/idle/plateau patterns |
-| `quick-smoke`     | 1      | 30s + 30s propagation       | Fast CI smoke test                                                                               |
-| `stress`          | 3      | 3.5 min + 1 min propagation | Heavy sustained load for benchmarking                                                            |
-
-Durations are the sum of the phase `duration_sec` values in
-`workload-profiles.json` plus that profile's `propagation_wait_sec`; they exclude
-cluster startup and the validation pass itself.
-
-### full-validation Phases
-
-| Phase        | RPC Rate           | TX TPS | Duration | Dashboard Coverage                                                                                                                                                                               |
-| ------------ | ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| warmup       | 5 RPS              | —      | 30s      | Node Health, Validator Health (baseline gauges)                                                                                                                                                  |
-| steady-state | 30 RPS             | 3 TPS  | 60s      | All dashboards (plateau data)                                                                                                                                                                    |
-| rpc-burst    | 100 RPS            | —      | 30s      | Job Queue, RPC Performance (latency spikes)                                                                                                                                                      |
-| tx-flood     | 5 RPS              | 20 TPS | 30s      | Fee Market & TxQ, Transaction Overview                                                                                                                                                           |
-| txq-burst    | 5 RPS (100% `fee`) | 60 TPS | 30s      | Fee Market & TxQ — single-type Payment burst that forces open-ledger fee escalation and TxQ queueing, exercising the `txq.*` spans (`txq.enqueue`, `txq.accept`, `txq.accept_tx`, `txq.cleanup`) |
-| mixed-peak   | 50 RPS             | 10 TPS | 60s      | Consensus Health, Ledger Operations                                                                                                                                                              |
-| cooldown     | 5 RPS              | —      | 30s      | Recovery patterns, state transitions                                                                                                                                                             |
-
-### Custom Profiles
-
-Add profiles to `workload-profiles.json`:
-
-```json
-{
-  "profiles": {
-    "my-custom": {
-      "description": "Custom profile for specific testing",
-      "phases": [
-        {
-          "name": "phase-name",
-          "description": "What this phase exercises",
-          "duration_sec": 60,
-          "rpc": { "rate": 50, "weights": { "server_info": 80, "fee": 20 } },
-          "tx": { "tps": 5, "weights": { "Payment": 100 } }
-        }
-      ],
-      "propagation_wait_sec": 30
-    }
-  }
-}
-```
-
-Set `"rpc"` or `"tx"` to `null` to skip that generator for a phase.
-Custom `"weights"` override the default command/transaction distribution.
-
-## Tools Reference
-
-### run-full-validation.sh
-
-Orchestrates the complete validation pipeline. Starts the telemetry stack, starts a multi-node xrpld cluster, generates load, and validates the results.
-
-```bash
-# Full validation with defaults (uses full-validation profile)
-./run-full-validation.sh --xrpld /path/to/xrpld
-
-# Quick smoke test
-./run-full-validation.sh --xrpld /path/to/xrpld --profile quick-smoke
-
-# Stress test with benchmarks
-./run-full-validation.sh --xrpld /path/to/xrpld --profile stress --with-benchmark
-
-# Skip Loki checks (if log export is not deployed)
-./run-full-validation.sh --xrpld /path/to/xrpld --skip-loki
-```
-
-### workload_orchestrator.py
-
-Reads a named profile from `workload-profiles.json` and executes sequential
-load phases. Within each phase, `rpc_load_generator.py` and `tx_submitter.py`
-run as concurrent subprocesses. Produces per-phase reports and a combined
-summary.
-
-```bash
-# Run with a specific profile
-python3 workload_orchestrator.py --profile full-validation
-
-# Multiple endpoints
-python3 workload_orchestrator.py --profile full-validation \
-    --endpoints ws://localhost:6006 ws://localhost:6007
-
-# Save combined report
-python3 workload_orchestrator.py --profile stress --report /tmp/report.json
-```
-
-### rpc_load_generator.py
-
-Generates RPC traffic matching realistic production distribution. Uses
-xrpld's **native WebSocket command format** (`{"command": ...}`) with flat
-parameters — the same format as `tx_submitter.py`.
-
-- 40% health checks (server_info, fee)
-- 30% wallet queries (account_info, account_lines, account_objects)
-- 15% explorer queries (ledger, ledger_data)
-- 10% transaction lookups (tx, account_tx)
-- 5% DEX queries (book_offers, amm_info)
-
-```bash
-# Basic usage
-python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120
-
-# Multiple endpoints (round-robin)
-python3 rpc_load_generator.py \
-    --endpoints ws://localhost:6006 ws://localhost:6007 \
-    --rate 100 --duration 300
-
-# Custom weights
-python3 rpc_load_generator.py --endpoints ws://localhost:6006 \
-    --weights '{"server_info": 80, "account_info": 20}'
-```
-
-### tx_submitter.py
-
-Submits diverse transaction types to exercise the full span and metric surface.
-Uses xrpld's **native WebSocket command format** (`{"command": ...}`) rather
-than JSON-RPC format. The response payload is inside the `"result"` key, with
-`"status"` at the top level.
-
-Supported transaction types:
-
-- Payment (XRP transfers) — exercises `tx.process`, `tx.receive`, `tx.apply`
-- OfferCreate / OfferCancel (DEX activity)
-- TrustSet (trust line creation)
-- NFTokenMint / NFTokenCreateOffer (NFT activity)
-- EscrowCreate / EscrowFinish (escrow lifecycle)
-- AMMCreate / AMMDeposit (AMM pool operations)
-
-Requires `[signing_support] true` in the node config for server-side signing.
-
-```bash
-# Basic usage
-python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120
-
-# Custom mix
-python3 tx_submitter.py --endpoint ws://localhost:6006 \
-    --weights '{"Payment": 60, "OfferCreate": 20, "TrustSet": 20}'
-```
-
-### validate_telemetry.py
-
-Automated validation that all expected telemetry data exists. Every metric in `expected_metrics.json` is required — if it doesn't fire, the validation fails. Spans are required unless the entry carries `"optional": true`.
-
-- **Span validation**: All span types from `expected_spans.json` with required attributes and parent-child hierarchies. The entries marked `"optional": true` are the ones the harness cannot guarantee: no gRPC client, no path-finding RPC (see [Pathfinding is not exercised](#pathfinding-is-not-exercised)), no missing-ledger fetch, no mode transition, no WebSocket handshake, no online-delete rotation (`nodestore.rotate.*`), and no transaction-set fetch (`txset.acquire`), because every node builds the same set locally. The six `txq.*` spans fire only when fee escalation puts something in the queue, and `ledger.serve` stays optional while every span that requests a ledger is optional. `rpc.http_request` and `rpc.process` are marked optional because the generator drives WebSocket rather than HTTP. Their absence is recorded as a passing skip, not a failure.
-- **Metric validation**: All metrics from `expected_metrics.json` — SpanMetrics, `beast::insight` gauges/counters/histograms, `MetricsRegistry` OTLP metrics. Every listed metric must have > 0 series. Uses the Prometheus `/api/v1/series` endpoint (not instant queries), polled until the metric appears or the poll window elapses, so a late-populating or quiet series is not a false negative. A counter that starts at 0 has a series before its first event, so existence cannot show that the event happened; a group's `min_values` entries check the value instead (see [expected_metrics.json Format](#expected_metricsjson-format)).
-- **Log-trace correlation**: trace_id/span_id in Loki logs (requires Loki). The two checks are `log.trace_id_present` and `log.trace_id_cross_reference`, and they exist only when `--skip-loki` is **not** passed — `run_validation()` builds them inside an `if not skip_loki` branch, so with the flag they are absent from the report rather than reported as skipped. The workflow passes no `--skip-loki`, so both checks are built and gated on every CI run — see [CI Integration](#ci-integration).
-- **Dashboard validation**: Every dashboard uid listed under `grafana_dashboards.uids` in `expected_metrics.json` loads with panels. That list covers every dashboard provisioned in `docker/telemetry/grafana/dashboards/`. Note the scope of this check: it asks the Grafana API whether the dashboard exists and returns a panel count — it does **not** run the panels' queries, so a dashboard can pass here while individual panels render empty.
-
-```bash
-# Run all validations
-python3 validate_telemetry.py --report /tmp/report.json
-
-# Skip Loki checks
-python3 validate_telemetry.py --skip-loki --report /tmp/report.json
-```
-
-### OTel Timings Regression Gate
-
-`capture_timings.py` + `compare_to_baseline.py` implement a regression gate
-that compares OTel-derived per-span/per-RPC/per-job timings against a
-committed baseline. Unlike `benchmark.sh` (which measures the overhead of
-enabling telemetry on the current binary), this gate catches **xrpld
-performance regressions over time** by diffing against a stored baseline
-from a prior run.
-
-How it runs inside the validation pipeline:
-
-1. `run-full-validation.sh` executes the normal workload and validation suite.
-2. After validation, `capture_timings.py` queries Prometheus for every
-   metric `regression-metrics.json` declares and does not list in
-   `excluded_keys`, then writes `reports/timings.json`. That file records
-   how much of the declared surface actually came back, in a `capture`
-   block alongside `metrics` — see [Capture completeness](#capture-completeness).
-3. `compare_to_baseline.py` reads `timings.json`,
-   `baselines/baseline-timings.json`, and `regression-thresholds.json`,
-   then either:
-   - Prints the paste-me JSON block (when the baseline is a placeholder
-     or empty and the capture is complete) and exits 0. An incomplete
-     capture is refused instead, with exit 2 and nothing on stdout.
-   - Prints a delta table, writes `reports/regression-report.json`, and
-     exits non-zero if any metric breached both the percentage AND
-     absolute bound.
-
-Bootstrapping a baseline:
-
-1. Push the branch. The `Telemetry Validation` CI run prints the full
-   timings JSON under "Paste into `baselines/baseline-timings.json`" in
-   the workflow Step Summary.
-2. Open a PR copying that JSON block verbatim into
-   `baselines/baseline-timings.json`. Reviewer approval is the audit gate.
-3. Subsequent runs compare against it; the gate fails on regression.
-
-#### Capture completeness
-
-`capture_timings.py` writes `timings.json` and only then enforces
-`--min-capture-ratio`, so a run that reached too little of Prometheus still
-leaves a file behind — one that exists, parses, and carries every declared key,
-some of them `null`. Nothing about it looks degraded.
-
-Every capture therefore states its own verdict:
-
-```json
-"capture": { "declared": 20, "captured": 20, "min_ratio": 0.5, "complete": true }
-```
-
-`complete` is exactly the condition `capture_timings.py` exits 0 on. Both
-paste-me paths — the workflow's Step Summary block and `compare_to_baseline.py`
-— read that flag and withhold the JSON unless it is `true`, because the
-placeholder path is the only route to a committed baseline and a thin capture
-pasted into one narrows the gate silently. An absent `capture` block (an
-artifact from before this existed) counts as not complete: completeness has to
-be proven, not assumed.
-
-Per-run tuning:
-
-- `--skip-regression` disables the gate (local exploration only).
-- `REGRESSION_WINDOW` env var overrides the default Prometheus `rate()`
-  window (`3m`). Keep close to the workload duration.
-- Metric surface lives in `regression-metrics.json`; thresholds in
-  `regression-thresholds.json`; both are reviewed changes. Each gated key's
-  absolute bound is `hi_next - baseline` — the distance from its baseline to the
-  top of the next bucket up — so refreshing the baseline obliges you to
-  re-derive the bounds. See `_absolute_bound_derivation` in that file;
-  `.github/scripts/telemetry/check_regression_bounds.py` enforces it in CI.
-- That bound budgets for **quantization** noise only, so a key whose run-to-run
-  variance is larger than it cannot be gated at all. **Six keys are excluded**
-  for that reason: `span.ledger.validate.p95` and `.p99`, plus
-  `span.tx.apply.p50`, `span.ledger.build.p50` and `.p99`, and
-  `span.consensus.ledger_close.p50`. Each carries
-  its measurements in `excluded_keys` in `regression-metrics.json`. Check a key's
-  observed maximum across runs against `baseline + bound` before gating it;
-  widening the bound is not the fix, and neither is re-baselining until a run
-  lands favourably. See `baselines/README.md`.
-- A refresh moves sensitivity in **both** directions, because the trip point is
-  derived from the baseline, and a single run carries no information about
-  spread. `job.acceptLedger.running.p95` has been measured with a 5.74x
-  detection floor on one baseline and 16.28x on another (it does not fire, so it
-  stays gated), and the three `p50` keys above have been measured both inside and
-  outside a bound that absorbs their spread — `span.tx.apply.p50` has read
-  0.7917 ms and 0.00597 ms on the same workload, 132x apart, which moves its
-  bound between 4.21 ms and 0.0440 ms. Gating those keys needs a **multi-run
-  baseline** (or a spread measurement captured beside it), not a new threshold.
-  All of it is measured in `baselines/README.md`; re-check after every refresh.
-
-See [`baselines/README.md`](./baselines/README.md) for the baseline
-lifecycle and refresh process.
-
-### benchmark.sh
-
-Compares baseline (no telemetry) vs telemetry-enabled performance:
-
-```bash
-./benchmark.sh --xrpld /path/to/xrpld --duration 300
-```
-
-Thresholds (configurable via environment):
-
-| Metric            | Threshold | Env Variable                |
-| ----------------- | --------- | --------------------------- |
-| CPU overhead      | < 3%      | BENCH_CPU_OVERHEAD_PCT      |
-| Memory overhead   | < 5MB     | BENCH_MEM_OVERHEAD_MB       |
-| RPC p99 latency   | < 2ms     | BENCH_RPC_LATENCY_IMPACT_MS |
-| Throughput impact | < 5%      | BENCH_TPS_IMPACT_PCT        |
-| Consensus impact  | < 1%      | BENCH_CONSENSUS_IMPACT_PCT  |
-
-Each report row is `PASS`, `FAIL`, or `INCONCLUSIVE`. The throughput and
-consensus rows are ratios of the baseline, so they have nothing to report when
-the baseline run measured zero — that row becomes `INCONCLUSIVE` and **counts
-as a failure**, because an undefined result must never read as a pass.
-
-Exit codes:
-
-| Code | Meaning                                                                                                                     |
-| ---- | --------------------------------------------------------------------------------------------------------------------------- |
-| 0    | Every metric was measured and is within its threshold                                                                       |
-| 1    | Every metric was measured and at least one exceeded its threshold                                                           |
-| 2    | The overhead could not be measured — missing prerequisite, cluster never reached consensus, or incomplete metric collection |
-
-`run-full-validation.sh` keeps the last two apart: 1 folds into its own
-"checks failed" exit, 2 into its "infrastructure error" exit. A run that
-measured nothing is therefore never reported as a performance regression.
-
-### collect_system_metrics.sh
-
-Samples CPU, peak RSS, RPC p99 latency, TPS and the mean inter-ledger interval
-from the running nodes, and writes them as JSON. `benchmark.sh` calls it once
-per leg; it is rarely run by hand.
-
-```bash
-./collect_system_metrics.sh 5020,5021,5022 300 /tmp/metrics.json [pids_csv]
-```
-
-Processes are selected by matching `argv[0]`'s basename against the daemon
-binary name; the pre-rename spelling is accepted too, so the sampler still
-works against an older deployment. A wrapper that merely names the binary in
-its arguments, and unrelated tools whose command line happens to contain the
-string, are not sampled — including them diluted the CPU average and
-attributed a foreign process's RSS to the node. `ps -C xrpld` is not usable
-for this: xrpld renames itself, so its `comm` is `xrpld-main`.
-
-A fourth argument narrows selection to an explicit pid list, and `benchmark.sh`
-always passes its own nodes' pids. It has to: `run-full-validation.sh` leaves its
-five validation nodes running while the benchmark's three start, so host-wide
-selection would average eight processes in both arms and report the largest of
-them as the RSS peak. Without the argument the scope is still the whole host, so
-a second xrpld from another checkout is sampled as well.
-
-The output carries a `metrics_complete` flag. It is `false` when any
-measurement source came back empty — no matching process, no successful RPC
-probe, or a ledger sequence that never advanced — and the affected metrics are
-then `0` placeholders. Since `0` clears every threshold, a `false` flag must be
-read as inconclusive, never as a pass.
-
-Exit codes:
-
-| Code | Meaning                                                                                                   |
-| ---- | --------------------------------------------------------------------------------------------------------- |
-| 0    | Every metric was measured; `metrics_complete` is `true`                                                   |
-| 1    | Cannot run: bad arguments, no GNU `date` with `%N`, or a failed process sample. No output file is written |
-| 3    | The output file was written, but `metrics_complete` is `false`                                            |
-
-`benchmark.sh` treats either non-zero code — and an explicit
-`"metrics_complete": false` in an otherwise successful run — as fatal, and
-exits 2 rather than comparing an incomplete run.
-
-A nanosecond clock is required. RPC latency is graded against a 2 ms
-threshold, and GNU `date +%s%N` is the only source cheap enough that the clock
-does not dominate what it measures, so the script refuses to start without it.
-
-## Reading Validation Reports
-
-The validation report (`validation-report.json`) is structured as follows. The
-counts below are illustrative — the real total is the sum of the span, metric,
-log, dashboard and parity checks for the run.
-
-```json
-{
-  "summary": {
-    "total": 45,
-    "passed": 42,
-    "failed": 3,
-    "all_passed": false
-  },
-  "checks": [
-    {
-      "name": "span.rpc.ws_message",
-      "category": "span",
-      "passed": true,
-      "message": "rpc.ws_message: 15 traces found",
-      "details": { "trace_count": 15 }
-    }
-  ]
-}
-```
-
-Categories:
-
-- **span**: Span type existence and attribute validation
-- **metric**: Prometheus metric existence, required labels, and minimum values
-- **log**: Log-trace correlation checks
-- **dashboard**: Grafana dashboard accessibility
-- **parity**: Span attributes required by the external-parity dashboard panels (validator-health, peer-quality, and friends)
-
-## CI Integration
-
-The validation runs as a GitHub Actions workflow (`.github/workflows/telemetry-validation.yml`):
-
-- Triggered manually (`workflow_dispatch`), or by any push touching the workflow's `paths` globs. There is no branch filter and no cron schedule.
-- Builds xrpld, starts the full stack, runs load, validates
-- Uploads reports as artifacts (and node logs when validation did not succeed)
-- Writes the validation summary and the regression-gate summary to the workflow **Step Summary** (`$GITHUB_STEP_SUMMARY`). It does **not** comment on the PR — the workflow declares no `permissions:` block and calls no GitHub API, so read the summary on the run page.
-
-Of the five `workflow_dispatch` inputs, only `run_benchmark` changes behaviour.
-`rpc_rate`, `rpc_duration`, `tx_tps` and `tx_duration` are forwarded to
-`run-full-validation.sh`, which parses them into shell variables and never reads
-them again — load shape comes entirely from `--profile` and
-`workload-profiles.json`. Their `description:` fields say so.
-
-### Log-trace correlation in CI
-
-The workflow passes no `--skip-loki`, so `log.trace_id_present` and
-`log.trace_id_cross_reference` are constructed and gated on every CI run. A green
-`Telemetry Validation` is evidence that log lines carry trace context and that a
-logged trace id resolves to an exported trace. `integration-test.sh` has
-its own `check_log_correlation()`, but no workflow runs that script.
-
-Correlation depends on four independent legs, and a failed check on its own names
-none of them: the node must write a `debug.log` line carrying trace ids, the
-collector container must see that file, its `file_log` receiver must parse and
-export the line, and Loki must return it for the validator's LogQL.
-`run-full-validation.sh` prints a per-leg diagnostic after the suite whenever the
-Loki checks are enabled — per-node correlated-line counts and severity mix, the
-container-side view of `/var/log/xrpld`, the receiver's watched files and
-internal log-record counters, and Loki's own entry counts for the selector with
-and without the line filter. Read that block first; it identifies the broken leg
-without reproducing anything.
-
-Those two entry counts **must** be wrapped in `sum()`. The `file_log` receiver's
-`regex_parser` leaves `message` and `timestamp` as log-record attributes, and
-Loki's OTLP path stores them as structured metadata that joins the label set of a
-metric query — so an unaggregated `count_over_time` returns one series per log
-line and Loki rejects it with `HTTP 400 maximum number of series (500) reached`
-past a few hundred lines. That is not hypothetical: it made both legs print
-`unavailable` on runs `32877465763` and `32964262700`, at which point the block
-distinguished nothing. `_loki_json` in `validate_telemetry.py` and
-`diag_loki_count` in `run-full-validation.sh` now print the HTTP status and
-Loki's own plain-text body, so a future rejection names its own cause instead of
-surfacing as a mimetype error.
-
-`log.trace_id_cross_reference` polls Tempo for up to `METRIC_POLL_TIMEOUT_SEC`
-(45 s, the same window and interval every other poll in the file uses) before
-reporting that a logged trace id does not resolve. A trace id reaches a log line
-when its span is created but is queryable only after export, ingest and indexing,
-so a single query races that pipeline. A failure now means the id was absent for
-the whole window.
-
-The check distinguishes three outcomes, not two, because "Tempo never answered"
-and "the spans were not exported" send a reader to different subsystems:
-
-| outcome                           | Tempo said                               | reported as                       |
-| --------------------------------- | ---------------------------------------- | --------------------------------- |
-| resolved                          | 200 with spans                           | pass                              |
-| absent for the whole window       | 404, or 200 with no spans, every attempt | "…do not resolve; not exported"   |
-| query failed and nothing resolved | any other non-200 (4xx/5xx)              | "could not verify … last error …" |
-
-`_tempo_get_trace` treats **404 as absence** and returns an empty list, because a
-trace id read from a log line is legitimately not yet indexed and every caller
-loops over candidates relying on that. Any **other** non-200 raises
-`TempoQueryError` — previously an error body was fed straight to `resp.json()`,
-so a JSON 5xx read as "0 spans" and a `text/plain` 5xx surfaced as a mimetype
-complaint. `_tempo_search` has no absence status at all (an empty match is 200
-with an empty list), so there every non-200 raises.
-
-The same block prints locally:
-
-```bash
-docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld
-```
-
-Re-run it after any change to log formatting, span activation, the collector's
-`file_log` receiver, or the Loki exporter.
-
-### Pathfinding is not exercised
-
-`rpc_load_generator.py` issues no path-finding RPC: `DEFAULT_WEIGHTS` carries no `ripple_path_find` entry and `build_rpc_request()` has no branch for it.
-
-**Why.** Pathfinding is disabled on every node this harness starts, so those calls could only ever fail:
-
-- `src/xrpld/core/detail/Config.cpp:725-726` sets `pathSearchMax = 0` whenever a `[validation_seed]` or `[validator_token]` section is present — "by default, validators don't have pathfinding enabled".
-- `run-full-validation.sh` writes `[validation_seed]` into every generated node cfg, and that script carries no `[path_search]`, `[path_search_fast]` or `[path_search_max]` section to put the default back.
-- `src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp:59-60` therefore returns `RpcNotSupported`; `PathFind.cpp:50-51` does the same for `path_find`.
-
-**Why the refusals would not be harmless.** They are not silent. `pathfind.request` is opened at `RipplePathFind.cpp:35`, **above** that guard, so a refused call still exports a span, and the enclosing `rpc.command.ripple_path_find` span carries `rpc_status=error`. At a 3% weight that is a steady ~3% error floor in `span_calls_total{status_code="STATUS_CODE_ERROR"}` — a figure that reads as an xrpld error rate and is not one. **An error-rate threshold derived from a harness run that does issue path-finding load is measuring the harness, not xrpld.**
-
-**What it costs.** Pathfinding has no coverage here at all. Four spans (`pathfind.request`, `.compute`, `.discover`, `.update_all`) and two histograms (`pathfind_fast_milliseconds`, `pathfind_full_milliseconds`) go unexercised, which is why `pathfind.request` is marked `"optional": true` in `expected_spans.json`. Verify pathfinding by hand instead: the **PathFind** row of [`../TESTING.md`](../TESTING.md) carries a `curl` recipe, and `../xrpld-telemetry.cfg` is a non-validator config that already enables pathfinding.
-
-**Enabling it.** All four steps are required. The first two alone produce the error floor described above:
-
-1. Add a `[path_search_max]` section to the node cfg `run-full-validation.sh` generates — or drop `[validation_seed]` and run a non-validator node. The `[path_search*]` block in `../xrpld-telemetry.cfg` is a working example.
-2. Add a `ripple_path_find` weight to `DEFAULT_WEIGHTS` and a branch for it in `build_rpc_request()`. `path_find` is a streaming subscription and needs its own phase instead — the generator is strictly one request, one reply.
-3. Set `pathfind.request` to required in `expected_spans.json`. Step 1 also makes `pathfind.compute` reachable, so the `pathfind.request -> pathfind.compute` relationship can lose its `"skip": true`.
-4. **Re-capture `baselines/baseline-timings.json`.** Adding the load changes the RPC mix, and `span.rpc.ws_message.{p50,p95,p99}` is a gated key — a baseline captured under a different mix is stale. See [OTel Timings Regression Gate](#otel-timings-regression-gate).
-
-## Configuration Files
-
-| File                              | Purpose                                                                                                                       |
-| --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- |
-| `workload-profiles.json`          | Named load profiles with phase definitions                                                                                    |
-| `expected_spans.json`             | Span inventory (names, attributes, hierarchies, config flags)                                                                 |
-| `expected_metrics.json`           | Metric inventory — every listed metric must be present — plus the `grafana_dashboards.uids` list the dashboard check iterates |
-| `test_accounts.json`              | Test account roles (keys generated at runtime)                                                                                |
-| `regression-metrics.json`         | Metric surface for the OTel regression gate                                                                                   |
-| `regression-thresholds.json`      | Per-metric regression bounds (pct AND abs)                                                                                    |
-| `baselines/baseline-timings.json` | Committed baseline — populated from first CI run                                                                              |
-| `requirements.txt`                | Python dependencies                                                                                                           |
-
-### expected_metrics.json Format
-
-```json
-{
-  "description": "Top-level doc string — skipped by the validator.",
-  "category_name": {
-    "description": "Human-readable description.",
-    "metrics": ["metric_1", "metric_2"],
-    "required_labels": ["label_1"],
-    "min_values": [
-      { "metric": "metric_2", "min": 1, "by": "service_instance_id" }
-    ]
-  },
-  "grafana_dashboards": {
-    "uids": ["rpc-performance", "node-health"]
-  },
-  "not_asserted": {
-    "description": "Why these are excluded.",
-    "metrics_excluded": { "metric_3": "reason" }
-  },
-  "accounted_patterns": [
-    {
-      "pattern": "^family_[a-z]+_(a|b)$",
-      "family": "Short label.",
-      "reason": "Why."
-    }
-  ]
-}
-```
-
-Every metric listed under a `metrics` array must produce > 0 Prometheus series during the validation run. If a metric doesn't fire, the workload generators need to produce enough load to trigger it.
-
-`required_labels` is optional and read for every category that declares one. Each label becomes one additional check, named `metric.<category>.label.<label>`, that at least one of that category's series carries the label with a non-empty value. It is matched as `<label>!=""` rather than `<label>=~".*"` because Prometheus cannot tell an absent label from an empty one, so a regex match would pass on a node that lost the label entirely. The guarantee rule is the same as for `metrics`: list a label only where the workload guarantees it. Declaring `required_labels` on a category with no `metrics` fails the check rather than skipping it, so the key can never sit unenforced.
-
-`min_values` is optional and read for every category, including one whose existence checks another validator owns. Each entry becomes one check, named `metric.<category>.min.<metric>`. The validator runs `sum(<metric>)`, or `sum by (<by>) (<metric>)` when `by` is given, and fails unless every returned series reaches `min`. Use it for a counter that starts at 0: its series exists from startup, so its `metrics` entry passes even when the event never happened. With `min: 1` and no `by`, it asks what an existence check asks of a counter created by its first event: did any node record the event? Add `by: "service_instance_id"` only where the workload guarantees the event on every node. With `by`, every returned series must carry that label, so a misspelled label or a node that exports without it fails the check instead of turning it into a cluster-wide sum. The entry itself is checked first, and a bad one fails without a query: `min_values` must be a list of objects, `metric` must appear in the same category's `metrics`, `min` must be a number above 0, `by`, when set, must be a non-empty string, and no other key is allowed, so a misspelled `by` fails instead of being ignored.
-
-`test_validate_telemetry.py` holds the rule for which counters need an entry: every counter selector under a `metrics` list needs a `min_values` entry, or a reason in its `COUNTERS_WITHOUT_MIN_VALUES` list. A counter added to a `metrics` list fails that test until one of the two is written.
-
-Four top-level keys are not metric categories:
-
-- `description` is a string, and `accounted_patterns` is a list, so
-  `_metric_check_targets` skips both structurally — it walks only top-level
-  values that are objects.
-- `grafana_dashboards` is an object but declares no `metrics`, so it contributes
-  no checks. `grafana_dashboards.uids` drives the dashboard check, so adding a
-  dashboard to `docker/telemetry/grafana/dashboards/` does **not** put it under
-  the gate until its uid is added here too.
-- `not_asserted` is skipped the same way: the loop reads
-  `category_data.get("metrics", [])`, and this group deliberately has no
-  `metrics` key — its entries live under `metrics_excluded` as a name-to-reason
-  map. It documents metrics that are emitted and dashboarded but left unasserted
-  because they are workload-gated or defect-gated (a check that fails on a
-  healthy run is worse than no check). Promote an entry into an asserted group
-  only after the workload is changed to guarantee it fires.
-- `accounted_patterns` asserts nothing. It is a list of `{pattern, family,
-reason}` entries feeding the reverse coverage check described below.
-
-### Reverse Coverage — emitted but not accounted for
-
-The checks above run in one direction only: they read the contract and ask the
-backend whether each listed name exists. That direction is blind to a name the
-contract omits, which is how a large metric gap and several unknown spans went
-unnoticed — both emitted inventories were already being fetched for the CI log,
-and neither was compared back.
-
-Two extra checks close the loop, `metric.reverse_coverage` and
-`span.reverse_coverage`. Each lists every name the backend reports that the
-contract never mentions, sorted, one per line in the log, with counts in the
-report's `details`.
-
-**They warn and never fail.** `passed` is hardcoded `True` in
-`_reverse_coverage_result`, so an unaccounted name cannot turn CI red. That is
-deliberate: downstream branches legitimately add telemetry an upstream contract
-has not seen yet, and a hard failure would redden every one of them for doing
-the right thing. The value is visibility, not enforcement.
-
-A metric family counts as accounted for when any of these is true:
-
-- a group's `metrics` array lists it (with any label matcher stripped),
-- a `not_asserted.metrics_excluded` key names it — deliberately unasserted is
-  not the same as unknown,
-- an `accounted_patterns` regex matches it (anchored, `fullmatch`).
-
-Exporter shapes are folded before matching, so a contract entry written for a
-family covers what the exporter derives from it: a histogram's
-`_bucket`/`_count`/`_sum` names fold back onto their base family, and counters
-are listed with the `_total` the exporter appends. Spans need no pattern list —
-`expected_spans.json` already carries globs such as `rpc.command.*`, and the
-reverse check reuses the same matcher the forward check uses.
-
-Use `accounted_patterns` only for a family whose membership is derived
-mechanically from a table in the code and so cannot be enumerated by hand — the
-per-job-type job-queue instruments and the overlay per-category traffic cross
-product are the two real cases, plus the Prometheus scrape plumbing that is not
-xrpld telemetry at all. Keep each pattern anchored and no wider than its family
-needs: a pattern that swallows unrelated names defeats the check.
-
-### expected_spans.json Format
-
-Each span entry defines its name, category, `allowed_parents`, required
-attributes, and the `config_flag` that must be enabled. A trailing `*` in `name`
-is a wildcard. The optional `"optional": true` field marks a span whose absence
-is a skip rather than a failure:
-
-```json
-{
-  "name": "rpc.command.*",
-  "category": "rpc",
-  "allowed_parents": ["rpc.ws_message", "rpc.process", "ROOT"],
-  "required_attributes": ["command", "version", "rpc_role", "rpc_status"],
-  "config_flag": "trace_rpc"
-}
-```
-
-`allowed_parents` is asserted, by `validate_span_parents`: every emitted instance
-of the span must be parented to one of the names listed, so a span declared a
-root that is emitted as somebody's child fails a check instead of passing
-silently. The list is derived from the span's creation factory plus every call
-path that reaches it — `SpanGuard::span(...)` and a plain `ScopedSpanGuard`
-inherit the ambient scope, so they take one entry per ambient scope their callers
-can be under; `freshRoot`, `linkedSpan` and the standalone `hashSpan` are roots;
-`childSpan(name, ctx)` names its parent exactly. `ROOT` means "no parent from
-this node", which covers a genuine root and a span whose parent was created on a
-different node, since a cross-node parent is the design for the receive spans and
-is never a violation. An entry may itself be a glob, matched the same way span
-names are.
-
-## Node Configuration Notes
-
-The orchestrator (`run-full-validation.sh`) generates node configs with:
-
-- `[telemetry] enabled=1` with all five trace categories: `trace_rpc`, `trace_transactions`, `trace_consensus`, `trace_peer`, `trace_ledger`
-- `[insight] server=otel` with `endpoint=http://localhost:4318/v1/metrics` — `beast::insight` metrics reach Prometheus over OTLP, because the collector declares no `statsd` receiver. No `prefix` is set: it would be inert, since `OTelCollector` applies no prefix to instrument names and exported names are the lowercased raw names (`jobq_job_count`, not `xrpld_jobq_job_count`)
-- `[signing_support] true` — required for `tx_submitter.py` to submit signed transactions via WebSocket
-- `[ips]` (not `[ips_fixed]`) — ensures peer connections are counted in the PeerFinder active-peer gauges, exported as `peer_finder_active_inbound_peers` / `peer_finder_active_outbound_peers` (fixed peers are excluded from these counters by design). The `beast::insight` group/name pair is `Peer_Finder` / `Active_Inbound_Peers`; `formatName()` lowercases it for export.
-
-## Gauge Export Behaviour
-
-The harness configures each node with `[insight] server=otel` (see the
-`[insight]` block generated by `run-full-validation.sh`), so `beast::insight`
-gauges go through `OTelGaugeImpl` in
-`src/libxrpl/beast/insight/OTelCollector.cpp`, not through the StatsD collector.
-That matters for how the validator queries Prometheus.
-
-**How `OTelGaugeImpl` exports.** It wraps an OTel **observable** (asynchronous)
-gauge. `set()` and `increment()` only store into an `std::atomic<int64_t>`;
-nothing is exported at call time. The SDK's collection thread invokes
-`gaugeCallback`, which runs the collector's hooks and then `Observe()`s whatever
-the atomic currently holds. So the gauge reports **every collection cycle,
-whether or not the value changed** — including a gauge that sits at 0 from
-startup. There is no dirty flag on this path, and no first-flush special case is
-needed.
-
-**Why the validator still uses `/api/v1/series`.** Two reasons survive the move
-to OTLP:
-
-1. **Late-populating series.** A gauge or counter may not have completed the
-   export → collector → Prometheus-scrape pipeline by the time validation runs.
-   `_check_prometheus_metric` in `validate_telemetry.py` therefore polls
-   `/api/v1/series` (which returns anything that existed anywhere in the query
-   window) until the metric appears or the poll window elapses, instead of
-   racing a single instant query.
-2. **Staleness robustness.** `/api/v1/series` does not care whether the newest
-   sample is inside Prometheus's ~5-minute staleness horizon, so the check
-   cannot be defeated by a quiet series.
-
-> **Note — the StatsD path is still in the tree but unused here.** If a node is
-> configured with `server=statsd`, `StatsDGaugeImpl` (in
-> `src/libxrpl/beast/insight/StatsDCollector.cpp`) does gate emission on a
-> `dirty_` flag that is only set by `set()`/`increment()`, and it is
-> initialised to `true` so the initial value is emitted on the first flush. The
-> collector configs shipped in `docker/telemetry/` declare no `statsd` receiver
-> (the metrics pipeline is `[otlp, spanmetrics]`) and the base
-> `docker-compose.yml` keeps its StatsD UDP port commented out, so nothing in
-> this harness can receive StatsD.

docker/telemetry/workload/baselines/README.md

@@ -1,496 +0,0 @@
-# Performance Baselines
-
-This directory holds the committed baseline file used by the OTel-driven regression gate.
-
-## How the gate works
-
-After the validation suite runs, `capture_timings.py` queries Prometheus for the timings
-declared in [`../regression-metrics.json`](../regression-metrics.json) and writes a
-`timings.json`. Then `compare_to_baseline.py` reads [`baseline-timings.json`](./baseline-timings.json),
-[`../regression-thresholds.json`](../regression-thresholds.json), and the captured
-`timings.json`. The comparator picks one of two modes automatically:
-
-- **Placeholder baseline** (`"placeholder": true` or empty `metrics`): the comparator
-  prints the captured timings JSON in exactly the format expected for this file, then
-  exits 0 without gating. This is how we bootstrap the baseline. It prints that block
-  only when the capture was complete — see
-  [An incomplete capture cannot seed a baseline](#an-incomplete-capture-cannot-seed-a-baseline).
-- **Populated baseline**: the comparator diffs per-metric, enforces the thresholds
-  (regression = current exceeds baseline on BOTH the percentage AND absolute bound),
-  and exits non-zero on any regression. The single exception is a baseline that is
-  not positive: the percentage change is undefined there, so the absolute bound
-  decides alone. Without that fallback the AND gate would be unreachable and a
-  0 ms → 500 ms jump would be reported as "within bounds".
-
-The regression gate runs against whatever workload profile `run-full-validation.sh`
-was invoked with. Capture and comparison are profile-agnostic — they only read
-Prometheus — so all existing profiles (`full-validation`, `quick-smoke`, `stress`)
-continue to work unchanged.
-
-## Current state: 19 metrics gate, on a baseline captured 2026-09-10
-
-`baseline-timings.json` holds real captured values for the 19 keys the harness gates. Every value is
-the **median of three clean CI runs** — `34495527952`, `34505215266` and `34507425933` — taken at
-`a0385c53cb`, profile `full-validation`, window `3m`. The file records that provenance itself, in
-its `source_runs` and `statistic` fields.
-
-The median of three is the point, not a detail. A single-run baseline is what disqualified five of
-the six excluded keys below: one sample carries no information about spread, and the bound is
-derived from that one sample alone.
-
-It replaced a capture taken 2026-08-26, before the account-funding race was fixed. Phases whose
-funding silently failed submitted little or no traffic, so that capture recorded artificially low
-ledger and transaction timings. Once funding worked, `span.ledger.build.p99` read 29.00 ms against
-its 9.109 ms baseline and turned the gate red on a run whose 200 span and metric checks all passed
-— which is why that key is now excluded.
-
-Two earlier generations stay retrievable from this file's git history: a 2026-08-24 capture, and
-before it entries captured 2026-06-05 that were voided into a placeholder because they were captured
-against the spanmetrics ladder's old 1 ms floor, which made every sub-millisecond quantile in that
-capture bucket-edge arithmetic rather than a latency (a p95 of `0.95` ms is `0.95 × 1 ms`). Because the
-comparator only flags a metric when the current value _exceeds_ the baseline, a stale-high baseline
-passes everything silently, so those entries had to be dropped rather than left in place.
-
-**A placeholder must not outlive one run.** CI stays green the whole time one stands, so an
-un-copied block is not a failure anyone will notice — it is a silent loss of regression coverage
-that looks identical to a passing gate. Voiding a baseline is the one hand edit that needs no CI
-block behind it. Setting one comes from a printed CI block, with a single documented exception —
-combining several runs into a median, which nothing automates yet (see [Schema](#schema)).
-
-## Absolute bounds are derived per metric, from the ladder
-
-`../regression-thresholds.json` gives every gated key its own `max_abs_increase_*`, equal to
-**`hi_next − baseline`**: locate the baseline in the half-open bucket `(lo, hi]` of its ladder,
-take `hi_next` as the next edge above `hi`, and the bound is the distance from the baseline to
-`hi_next`. The trip point is therefore exactly `hi_next` — the gate fires only once the reading
-clears the bucket **above** the baseline's own.
-
-That is what buys the guarantee. `histogram_quantile` returns a value interpolated inside
-whichever bucket the true quantile falls in, so any reading taken while the quantile is still in
-the baseline's bucket, or anywhere in the one immediately above, is at most `hi_next` and cannot
-fire. Firing needs the quantile to have moved at least two buckets up. A multiple of the
-_enclosing_ bucket's width cannot deliver this, because once the quantile crosses `hi` the
-interpolation happens across the **next** bucket, which on this ladder is up to 8x wider —
-`(0.5, 1]` is 0.5 ms wide and `(1, 5]` is 4 ms wide. The full derivation, both ladders, and a
-per-key table of the arithmetic are in that file's `_absolute_bound_derivation`,
-`_bucket_note` (both ladders) and `_derivation_table`.
-
-Two earlier generations of this bound were wrong, in opposite directions:
-
-| generation                     | bound                                        | 10x regression caught | single-crossing false positive reachable |
-| ------------------------------ | -------------------------------------------- | --------------------- | ---------------------------------------- |
-| flat                           | 10 ms `p50`/`p95`, 15 ms `p99`, 20000 us job | 5 / 28 keys           | 2 / 25 keys                              |
-| 2 × enclosing bucket width     | per metric                                   | 28 / 28 keys          | **21 / 25 keys**                         |
-| `hi_next − baseline` (current) | per metric                                   | **19 / 19 keys**      | **0 / 19 keys**                          |
-
-The first two rows were measured when 28 and 25 keys were gated, by injecting a 10x regression into
-each gated key in turn against a real CI `timings.json`. The current row is **derived, not sampled**,
-and holds for every key on the 2026-09-10 baseline. A key's detection floor is exactly
-`trip point ÷ baseline`, because the gate fires when the reading exceeds the trip point and a `k`x
-regression reads `k × baseline`. So a 10x regression is caught precisely when the floor is under 10x,
-and the weakest floor on this baseline is 7.41x — see
-[weakly guarded](#which-keys-are-only-weakly-guarded) below.
-
-That is a real improvement over the 2026-08-26 baseline, where `job.acceptLedger.running.p95` had a
-16.28x floor and was the one key a 10x regression missed. Its baseline rose from 6142.9 us to
-15967.7 us while `hi_next` stayed at 100000 us, which pulled its floor down to 6.26x. Note the
-direction this can move in: floors are a property of where each baseline lands on the ladder, so a
-refresh changes sensitivity without anything about the code changing. Re-derive this table on every
-refresh.
-
-The zero in the last column is by
-construction rather than by sampling: rule C in
-[`check_regression_bounds.py`](../../../../.github/scripts/telemetry/check_regression_bounds.py)
-fails the build unless every trip point is exactly `hi_next`, and a trip point at a bucket edge
-cannot be crossed by interpolation inside that bucket.
-
-The flat bound was calibrated for a 5-25 ms band the spans do not occupy: 18 of the 28
-quantiles gated at the time sat below 1 ms, so it sat 1.15x to 2000x above the metric it guarded,
-and because the rule is an `AND` the percentage bound could never carry a regression alone. A 100x
-regression injected into `span.ledger.store.p95` reported **0 regressions, exit 0**. The second
-generation fixed the magnitude but kept an assumption that does not hold — that the reading's
-excursion is bounded by the enclosing bucket's width — which put 21 of 25 trip points inside the
-adjacent bucket, so a single legitimate bucket crossing could turn CI red.
-
-**Refreshing the baseline means re-deriving the bounds**, because a refreshed value can land in a
-different bucket and so get a different `hi_next`. This is no longer a documentation-only rule:
-[`.github/scripts/telemetry/check_regression_bounds.py`](../../../../.github/scripts/telemetry/check_regression_bounds.py)
-fails CI when a bound is not the one its own baseline implies, when a gated key has no override,
-when a baseline key is not declared by `../regression-metrics.json` (or the reverse), when the
-percentage bound would become the operative one, and when a baseline carries the ladder-floor
-signature described below.
-
-### Which keys are only weakly guarded
-
-The guarantee costs sensitivity where the ladder is coarse: the detection floor is
-`hi_next / baseline`, so a baseline sitting just above an edge is guarded loosely. Over the
-2026-09-10 baseline the floor ranges 2.00x to 7.41x. Do **not** read these six as guarded:
-
-| key                               | baseline   | fires at  | floor | limiting ladder step |
-| --------------------------------- | ---------- | --------- | ----- | -------------------- |
-| `span.consensus.ledger_close.p95` | 0.6750 ms  | 5 ms      | 7.41x | 1 ms → 5 ms          |
-| `span.consensus.accept.p50`       | 1.4364 ms  | 10 ms     | 6.96x | 5 ms → 10 ms         |
-| `job.acceptLedger.running.p95`    | 15967.7 us | 100000 us | 6.26x | 25000 us → 100000 us |
-| `span.rpc.ws_message.p95`         | 0.8122 ms  | 5 ms      | 6.16x | 1 ms → 5 ms          |
-| `span.rpc.ws_message.p99`         | 0.9873 ms  | 5 ms      | 5.06x | 1 ms → 5 ms          |
-| `span.tx.process.p99`             | 0.9940 ms  | 5 ms      | 5.03x | 1 ms → 5 ms          |
-
-Four of the six are limited by the same `1 ms → 5 ms` step, which is where this ladder is coarsest
-relative to how the spans actually behave. All six stay gated; the weak floor is recorded here so it
-is visible rather than surprising.
-
-None of the six fires on an observed run **of this workload** — but the qualifier is load-bearing,
-and there is now a measurement behind it. Changing one line of the generated node config from
-`[ips]` to `[ips_fixed]`, which holds peer connections open instead of treating the list as a
-discovery hint, moved `span.consensus.ledger_close.p95` from 0.57 ms to 6.43 ms and tripped this
-gate, while every transaction-path metric fell. Nothing else in that commit touched the consensus
-path. So a weak floor is not the only way one of these keys reddens: a change to the cluster's
-topology is enough on its own, which is exactly why
-[Refreshing the baseline](#refreshing-the-baseline) treats a workload change as requiring a new
-baseline.
-
-The fix is a 2 ms edge (ideally 3 ms as well) in the collector's spanmetrics `buckets` list plus the
-matching entries in `kMillisecondBuckets`, and 2000 us plus 50000 us edges in `kMicrosecondBuckets`.
-That work belongs to the branch that owns the ladders.
-
-`job.transaction.running.p95` and `span.tx.process.p95` were on this list against the 2026-08-26
-baseline, at 8.33x and 8.20x, and both dropped off it in the refresh — 2.65x and 2.00x now.
-`span.tx.process.p95` is the tightest gated key on this baseline — 0.50 on baseline over trip point,
-and 0.76 on the observed maximum across the three source runs — so it is the first to re-measure if
-the gate reddens. The two ratios have different numerators; neither is the other.
-
-`span.tx.apply.p50` is absent from this table because it is **no longer gated at all** — see
-[what all six excluded keys have in common](#what-all-six-excluded-keys-have-in-common). Beyond
-its variance it had a second, independent problem: its baseline of `0.00597` ms sat inside the
-ladder's **first** bucket `(0, 0.01]`, so the reported figure was interpolation across that bucket,
-tracking the _fraction_ of applies finishing under 10 us rather than a latency — the same mechanism
-that disqualified `ledger.store` below. Rule E did not flag it, correctly: the value is not
-`quantile × first_edge` exactly, so some mass does sit above 0.01 ms. Restoring the key therefore
-needs a finer low-end ladder **as well as** a spread-aware baseline.
-
-## Known exclusion: `ledger.store` is below the ladder's resolution
-
-`span.ledger.store` is **not** gated. The 2026-08-24 capture returned p50/p95/p99 of exactly
-`0.005` / `0.0095` / `0.0099` ms, which is `0.5` / `0.95` / `0.99 × 0.01` ms — the ladder's first
-edge times the quantile, the signature of every sample landing in the first bucket. Those numbers
-are interpolation arithmetic on the bucket floor, not latencies. It is physically plausible:
-[`LedgerMaster.cpp:470`](../../../../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L470) wraps an
-in-memory `ledgerHistory_.insert`, which completes in single-digit microseconds.
-
-While all the mass stays under 10 us the reported quantile cannot move materially, so **no
-absolute bound can gate this key** — every `ledger.store` slowing from 2 us to 9 us, a 4.5x
-regression, leaves the reported value unchanged. Three keys that read as covered but cannot fire
-are worse than no keys, the same argument that excluded `rpc.process`, so they were removed from
-`../regression-metrics.json` rather than left in with a bound that looks derived.
-
-Restoring the key needs sub-10 us edges on the collector's spanmetrics ladder (for example
-`0.001ms` and `0.005ms`) plus the matching entries in `HistogramBuckets.h`. `ledger.store`
-presence is still asserted by `../expected_spans.json` and `docker/telemetry/integration-test.sh`,
-and its rate is still on the ledger-operations dashboard; only the latency gate drops it.
-`check_regression_bounds.py` rule E fails the build if a key with this signature is gated again.
-
-## Known exclusion: `ledger.validate` p95 and p99 vary more than any bound can absorb
-
-`span.ledger.validate.p95` and `.p99` are **not** gated. `p50` still is. They are the first
-exclusion at _quantile_ rather than _span_ granularity, which is why
-[`../regression-metrics.json`](../regression-metrics.json) grew an `excluded_keys` map — `spans.names`
-lists span names and `_quantiles` is shared across all of them, so removing two quantiles of one
-span cannot be expressed by deleting a name.
-
-Measured across four CI runs, against the baseline in force **when the two keys were excluded**.
-The `p50` row is the only one still gated. Its baseline was 0.0779 ms then and is 0.0598 ms now, with
-the same 0.25 ms trip point either way, so the argument is unchanged:
-
-| key                               | baseline at exclusion | trip point | observed min | observed max | spread |
-| --------------------------------- | --------------------- | ---------- | ------------ | ------------ | ------ |
-| `span.ledger.validate.p50` (kept) | 0.0779 ms             | 0.25 ms    | 0.0484 ms    | 0.0778 ms    | 1.6x   |
-| `span.ledger.validate.p95`        | 0.2404 ms             | 0.5 ms     | 0.1281 ms    | 0.7500 ms    | 5.9x   |
-| `span.ledger.validate.p99`        | 1.0600 ms             | 10 ms      | 0.3875 ms    | 25.8750 ms   | 66.8x  |
-
-Both excluded quantiles reach past their trip point on an ordinary run, so CI reddened twice with
-no code change: run `32867433073` read `p95` = 0.7500 ms (+212%) and run `32862589645` read
-`p99` = 25.8750 ms (+2341%). The two failures landed on **different** quantiles in different runs
-while the other quantile stayed well inside its bound in the same run — the signature of variance,
-not of a regression.
-
-The mechanism is arrival timing, not slow code. The span opens only once a quorum-completing
-validation arrives ([`LedgerMaster.cpp:1003`](../../../../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L1003),
-inside `checkAccept`, past the `tvc < minVal` early return) and wraps the promotion work that
-follows — `setValidated`, `setFull`, `setValidLedger`, `pendSaveValidated`. Its duration therefore
-tracks when peer validations arrive in a 5-node cluster and what promotion then schedules, so a
-single slow consensus round dominates the tail of a 3 m rate window, and which round that is
-differs every run.
-
-**Widening the bound is not an option and must not be attempted.** Tolerating 25.8750 ms against a
-1.0600 ms baseline needs a bound of ~24.8 ms, i.e. a gate that fires at nothing a regression could
-plausibly reach. A bound that admits every healthy run's worst case admits every regression too.
-`check_regression_bounds.py` rule F fails the build if either key is re-gated with a bound while
-still listed in `excluded_keys`, and the per-key reasons in that map record this in full.
-
-### The general rule this exposed
-
-`hi_next − baseline` is derived from the **ladder**, so it budgets for **quantization** noise — one
-bucket of interpolation headroom — and for nothing else. It knows nothing about how far the metric
-itself moves between runs on identical code. Where run-to-run workload variance is the larger term,
-the bound is simply the wrong size and the gate reddens on a healthy run.
-
-**Before gating any key, check its observed maximum across several runs against its trip point
-(`baseline + bound`), and gate it only if the maximum stays below that with margin.** Spread alone
-proves nothing; it is spread **relative to the trip point** that decides. And because the trip
-point is derived from the baseline, a baseline that lands at the **low end** of a metric's own
-range shrinks that trip point without anything about the metric having changed.
-
-That is what happened to three `p50` keys on the **2026-08-26** baseline, and **all three are
-excluded** — this rule being applied, not a new exception. Measured across the three CI runs
-`32862589645`, `32867433073` and `32964262700` (the last of which produced that baseline):
-
-| key                               | bound     | trip point | observed max | max ÷ trip | spread |
-| --------------------------------- | --------- | ---------- | ------------ | ---------- | ------ |
-| `span.tx.apply.p50`               | 0.0440 ms | 0.05 ms    | 2.3378 ms    | **46.76x** | 391.8x |
-| `span.ledger.build.p50`           | 0.3849 ms | 0.5 ms     | 2.3826 ms    | **4.77x**  | 20.7x  |
-| `span.consensus.ledger_close.p50` | 0.0613 ms | 0.1 ms     | 0.2377 ms    | **2.38x**  | 6.1x   |
-
-Before the exclusion, replaying **either** older run against that baseline reported exactly those
-three and nothing else — and run `32867433073` carries the same post-path-finding-removal workload
-as the baseline itself, so the movement was metric variance, not a workload difference. Those two
-runs are what would have reddened CI. After the exclusion both replay clean.
-
-The evidence that settles it is `span.tx.apply.p50`'s own history. It read **0.7917 ms** in the
-2026-08-24 baseline and **0.00597 ms** in the 2026-08-26 one — a 132x difference between two runs of
-the same workload. At the old value the identical `hi_next − baseline` rule produced a 4.21 ms bound
-whose 5 ms trip point absorbed the entire range; at the new value it produces 0.0440 ms and cannot.
-Nothing about the metric changed. **Whether the gate functioned was decided by where in its own
-distribution the captured run happened to land** — which is not a threshold that needs tuning, it
-is a key that cannot be gated from a single-run baseline at all.
-
-So the remedy is the `excluded_keys` entry with the measurement behind it, exactly as
-`ledger.validate` p95 and p99 got — **not** a widened bound, and **not** re-baselining until a run
-lands favourably. A key that fails this test is never fixed by widening its bound. The remaining
-19 gated keys sit between 0.14 and 0.50 of their **baseline** over their trip point, the tightest
-being `span.tx.process.p95` at 0.50. That ratio is derivable from the two committed JSON files, so it
-is checkable; a headroom figure against each key's observed maximum is not, because no per-run
-`timings.json` is committed.
-
-### What all six excluded keys have in common
-
-| key                               | trip point | observed max | mechanism                             |
-| --------------------------------- | ---------- | ------------ | ------------------------------------- |
-| `span.tx.apply.p50`               | 0.05 ms    | 2.3378 ms    | baseline in the ladder's first bucket |
-| `span.consensus.ledger_close.p50` | 0.1 ms     | 0.2377 ms    | baseline in a low bucket              |
-| `span.ledger.build.p50`           | 0.5 ms     | 2.3826 ms    | baseline in a low bucket              |
-| `span.ledger.validate.p95`        | 0.5 ms     | 0.7500 ms    | baseline in a low bucket              |
-| `span.ledger.validate.p99`        | 10 ms      | 25.8750 ms   | spread too large for any bound        |
-| `span.ledger.build.p99`           | 25 ms      | 29.0000 ms   | spread too large for any bound        |
-
-One invariant covers all six: **the observed maximum exceeds `baseline + bound`**, so an ordinary
-run clears the trip point with nothing having regressed. Two mechanisms produce it. Four of the six
-have a baseline sitting low in the ladder, where the derived bound is tiny because the bound _is_
-the distance to the next edge up. The other two fail despite generous bounds:
-`ledger.validate.p99` has 8.94 ms and a 66.8x spread that reaches 25.875 ms against a 10 ms trip
-point, and `ledger.build.p99` has 16.056 ms and a 4.11x spread whose maximum is 1.16x its trip
-point.
-
-`span.ledger.build.p99` is the newest of the six and the clearest illustration of the rule, because
-the previous baseline **hid** it: at 9.109 ms the same rule also gave a 25 ms trip point, and the key
-read as gated only because both the capture and the comparison runs happened to land low. Ledger
-construction keeps coverage through `span.ledger.build.p95`, whose baseline sits at 0.48 of its trip
-point.
-
-**The follow-up that would restore coverage**, stated rather than left implied: a bound derived from
-the ladder alone cannot support these keys, because it carries no information about spread. What
-would let them be gated again is a bound sized against **observed variance** — a spread measurement
-captured alongside the baseline, rather than the ladder distance only. The 2026-09-10 baseline is
-already a median of three runs, which is the raw material for that; using the spread to size bounds
-is the part that is not implemented, and it is the design change these six exclusions are waiting
-on.
-
-## Bootstrapping the baseline
-
-1. Merge a CI run with a `"placeholder": true` baseline. The telemetry-validation
-   workflow runs, fails no gate, and prints the captured timings block to the workflow
-   Step Summary under the heading `### Paste into baselines/baseline-timings.json`.
-2. Open a new PR. Copy the full JSON block from the Step Summary (or download the
-   `timings.json` artifact) into this file, replacing the placeholder contents. The
-   JSON is emitted in the exact byte-for-byte format this file expects — sorted keys,
-   2-space indent, trailing newline.
-3. The committed baseline PR needs reviewer approval just like any other code change.
-   This is the primary audit point for "who moved the performance bar."
-
-### An incomplete capture cannot seed a baseline
-
-`capture_timings.py` writes `timings.json` **before** it enforces
-`--min-capture-ratio`, so a run that reached too little of Prometheus still leaves a
-file behind — one that exists, parses, and carries every declared key, some of them
-`null`. Nothing about it reads as degraded, and the obvious reaction to a red gate is
-to refresh the baseline, so this is exactly the file a person is most likely to paste.
-
-Every capture therefore records its own verdict in a `capture` block (see
-[Schema](#schema)), and `complete` there is exactly the condition
-`capture_timings.py` exits 0 on. Both routes to a baseline read that flag and print
-nothing to paste unless it is `true`:
-
-- the workflow's Step Summary heading becomes "Baseline NOT refreshable from this run",
-  carrying the captured/declared counts and an `::error::` annotation;
-- `compare_to_baseline.py` writes the same explanation to stderr, leaves stdout empty
-  so a `>` redirect cannot produce a plausible-looking file, and exits 2.
-
-An artifact with no `capture` block — one produced before this existed — counts as not
-complete. Completeness has to be proven, not assumed.
-
-This only guards the paste. Against a populated baseline a thin capture still compares
-normally and its uncaptured keys are reported as `not captured in current run`, which
-is the pre-existing behaviour described under [Schema](#schema) below.
-
-## Refreshing the baseline
-
-Refresh when a legitimate performance change lands on `develop` (for example, a
-deliberate rewrite that changes a span's structure). The process is identical to
-bootstrapping: run CI with the current baseline, inspect the delta, and if the
-new numbers should become the norm, open a PR pasting the fresh timings into
-`baseline-timings.json`. The reviewer decides whether the new baseline is acceptable.
-
-Do **not** edit `baseline-timings.json` by hand outside of this process — every entry
-should trace back to a real CI run so variance characteristics are preserved.
-
-Refreshing the baseline also obliges you to re-derive the absolute bounds in
-`../regression-thresholds.json`, per
-[Absolute bounds are derived per metric](#absolute-bounds-are-derived-per-metric-from-the-ladder).
-A value that moves into a different bucket needs a different bound, and a bound left behind
-either stops catching regressions or starts firing on quantization noise.
-
-It also obliges you to re-check each key's run-to-run spread against its new trip point, per
-[The general rule this exposed](#the-general-rule-this-exposed). A refreshed baseline can land in a
-bucket whose `hi_next` no longer clears the metric's own variance, which turns the key into a
-recurring false positive — the failure that excluded `ledger.validate` p95 and p99.
-
-## The baseline is only valid at the log level it was captured at
-
-Every timing here is coupled to the `log_level` that `run-full-validation.sh` writes into
-each node's `[rpc_startup]` stanza. Logging is **synchronous**, and several of the gated
-spans contain log statements, so the configured level is part of the measurement:
-
-- `ledger.build` contains [`BuildLedger.cpp:81`](../../../../src/xrpld/app/ledger/detail/BuildLedger.cpp#L81) (debug).
-- `consensus.accept` contains [RCLConsensus.cpp:683/687/698/715](../../../../src/xrpld/app/consensus/RCLConsensus.cpp#L715) (debug) — `:715` logs **once per transaction** in the canonical set.
-- `tx.apply` and the other `spans.names` entries in [`../regression-metrics.json`](../regression-metrics.json) are affected the same way.
-
-Raising the level admits more of those statements and inflates the p50/p95/p99 of the very
-spans the gate measures; lowering it deflates them. Neither shows up as a regression, because
-the baseline moves with it — the gate simply starts measuring a different configuration.
-
-**Changing the workload log level therefore invalidates this baseline and requires
-re-capturing it.** Treat it exactly like a deliberate performance change: follow
-[Refreshing the baseline](#refreshing-the-baseline), and note the level change in the PR so
-the reviewer knows why the numbers moved. In particular, do not capture a baseline while the
-harness is running at `debug` — see the runbook's "Why not `debug`" note; if you need
-debug-level detail, enable it per partition **after** the baseline exists.
-
-## Schema
-
-```json
-{
-  "schema_version": 1,
-  "captured_at": "2026-04-24T17:30:00Z",
-  "window": "3m",
-  "git_sha": "<SHA of the commit that produced these numbers>",
-  "profile": "<workload profile used>",
-  "source_runs": [34495527952, 34505215266, 34507425933],
-  "statistic": "median of three clean full-validation runs",
-  "capture": {
-    "declared": 19,
-    "captured": 19,
-    "min_ratio": 0.5,
-    "complete": true
-  },
-  "metrics": {
-    "span.tx.process.p99": { "value": 12.4, "unit": "ms" },
-    "job.transaction.queued.p95": { "value": 1500.0, "unit": "us" }
-  }
-}
-```
-
-`source_runs` and `statistic` record how the numbers were arrived at, and the committed baseline
-carries both. They matter because the bound-derivation rule reads the baseline as a single number:
-if `statistic` says the values are a median of several runs, a reviewer knows the spread was
-observable, and if it is absent the baseline came from one run and every key on it is exposed to the
-single-run problem described under
-[The general rule this exposed](#the-general-rule-this-exposed). Neither field is read by any
-script; they are provenance, like `git_sha`.
-
-`capture_timings.py` emits neither field, and no script in this directory combines several runs, so
-a multi-run baseline is currently assembled by hand and these two fields are how that is declared.
-That is a gap, not a workflow: it sits outside the paste-from-CI rule the rest of this file
-describes, so the median and the run ids are only as trustworthy as the PR that introduced them.
-Automating the combination — and having it write both fields — is part of the multi-run baseline work
-the exclusions are waiting on.
-
-`capture` describes the capture that produced the file, not the metrics in it:
-`declared` is how many keys the surface asked for, `captured` how many came back with a
-value, `min_ratio` the bar they were judged against, and `complete` the verdict. It is a
-sibling of `metrics`, never an entry inside it, so it is neither a metric key nor a
-gated entry — `check_regression_bounds.py` and `compare_to_baseline.py` both iterate
-`metrics` alone and never see it. Because a committed baseline is a verbatim copy of a
-capture, the block lands here too; it is metadata about provenance, exactly like
-`git_sha`. Entries committed before it existed simply do not carry it.
-
-Keys follow `{category}.{name}.p{quantile}`. Only two categories are actually
-produced today — `span.*` and `job.*` — because `build_query_plan()` in
-`prom_queries.py` reads the `spans` and `job_queue` groups of
-`regression-metrics.json`, and that file defines only those two.
-
-Placeholder baselines additionally include `"placeholder": true`. The comparator
-detects this field (or an empty `metrics` object) to switch into "populate" mode
-instead of enforcing thresholds. Remove the `placeholder` key when pasting real
-captured timings.
-
-Missing metrics (value `null`) in a captured run do not count as regressions. In
-`regression-report.json`, `summary.missing_in_current` is a **count** only; the
-identities are in the `metrics[]` array, as the entries whose `note` is
-`"not captured in current run"`. Filter for those to see which keys went missing:
-
-```bash
-jq -r '.metrics[] | select(.note == "not captured in current run") | .key' \
-    /tmp/xrpld-validation/reports/regression-report.json
-```
-
-This keeps the gate robust when a profile doesn't exercise every span on every run.
-
-## Known gap: no `rpc.*` metric can gate (FU-4)
-
-Per-RPC-method timings are **not** gated, and would not gate even if they were
-captured. Two independent blockers:
-
-1. **Nothing emits an `rpc.*` key.** `build_query_plan()` in `prom_queries.py`
-   builds `rpc.*` entries from `cfg.get("rpc_methods", {})`, and
-   `regression-metrics.json` has no `rpc_methods` block — so the group resolves
-   to empty and no `rpc.*` key ever reaches `timings.json` or this baseline.
-2. **Even a captured `rpc.*` key would silently not gate.** `resolve_thresholds()`
-   in `compare_to_baseline.py` maps the `rpc` category to the threshold group
-   `rpc_method`, but `regression-thresholds.json` defines only
-   `defaults.span` and `defaults.job_queue`. With no `rpc_method` block the
-   lookup returns `(None, None)`, which the comparator treats as "no threshold
-   configured" — the metric is reported but can never fail the build.
-
-Closing this needs **both** an `rpc_methods` group in `regression-metrics.json`
-and a `defaults.rpc_method` block in `regression-thresholds.json`. Adding only
-the first produces metrics that look gated in the report but are not.
-
-## Known exclusion: `rpc.process` is not captured
-
-`rpc.process` is deliberately absent from the `spans.names` list in
-`regression-metrics.json`, so no `span.rpc.process.*` key appears in this
-baseline. The span is created only in `ServerHandler::processRequest()`
-(`src/xrpld/rpc/detail/ServerHandler.cpp:718`), which is reached only from the
-HTTP/JSON-RPC session path. The harness load generator is WebSocket-only and
-that path never calls `processRequest`, so the span is never emitted under any
-workload profile here — `expected_spans.json` marks it `"optional": true` for
-the same reason.
-
-While it was listed, the three quantiles were captured as `null` on every run
-and the comparator short-circuited them as `"new metric (not in baseline)"` —
-so a 9999 ms value would still have reported `regressed: false`. Three keys
-that can never gate are worse than no keys: they inflate `summary.total` and
-read as covered.
-
-If per-request HTTP timings are wanted, the fix is to give the harness an
-HTTP/JSON-RPC load path first, then re-add `rpc.process` and bootstrap a real
-baseline for it.

docker/telemetry/workload/baselines/baseline-timings.json

@@ -1,87 +0,0 @@
-{
-  "captured_at": "2026-09-10T18:05:00Z",
-  "git_sha": "a0385c53cb8f1e9ab77379b5584367a92d0a9014",
-  "metrics": {
-    "job.acceptLedger.queued.p95": {
-      "unit": "us",
-      "value": 95.58333333333331
-    },
-    "job.acceptLedger.running.p95": {
-      "unit": "us",
-      "value": 15967.741935483866
-    },
-    "job.transaction.queued.p95": {
-      "unit": "us",
-      "value": 403.74177631578937
-    },
-    "job.transaction.running.p95": {
-      "unit": "us",
-      "value": 376.7512077294688
-    },
-    "span.consensus.accept.p50": {
-      "unit": "ms",
-      "value": 1.4363636363636365
-    },
-    "span.consensus.accept.p95": {
-      "unit": "ms",
-      "value": 8.811111111111114
-    },
-    "span.consensus.accept.p99": {
-      "unit": "ms",
-      "value": 20.928571428571445
-    },
-    "span.consensus.ledger_close.p95": {
-      "unit": "ms",
-      "value": 0.6749999999999998
-    },
-    "span.consensus.ledger_close.p99": {
-      "unit": "ms",
-      "value": 3.049999999999991
-    },
-    "span.ledger.build.p95": {
-      "unit": "ms",
-      "value": 4.7714285714285705
-    },
-    "span.ledger.validate.p50": {
-      "unit": "ms",
-      "value": 0.059761904761904766
-    },
-    "span.rpc.ws_message.p50": {
-      "unit": "ms",
-      "value": 0.16282758747645082
-    },
-    "span.rpc.ws_message.p95": {
-      "unit": "ms",
-      "value": 0.8122454466253443
-    },
-    "span.rpc.ws_message.p99": {
-      "unit": "ms",
-      "value": 0.9872660098522166
-    },
-    "span.tx.apply.p95": {
-      "unit": "ms",
-      "value": 4.639716312056738
-    },
-    "span.tx.apply.p99": {
-      "unit": "ms",
-      "value": 4.9733333333333345
-    },
-    "span.tx.process.p50": {
-      "unit": "ms",
-      "value": 0.21390674968918655
-    },
-    "span.tx.process.p95": {
-      "unit": "ms",
-      "value": 0.49949970576841685
-    },
-    "span.tx.process.p99": {
-      "unit": "ms",
-      "value": 0.9939697679594013
-    }
-  },
-  "profile": "full-validation",
-  "schema_version": 1,
-  "source_runs": [34495527952, 34505215266, 34507425933],
-  "statistic": "median of three clean full-validation runs",
-  "window": "3m"
-}

docker/telemetry/workload/benchmark.sh

@@ -1,734 +0,0 @@
-#!/usr/bin/env bash
-# benchmark.sh — Performance benchmark for rippled telemetry overhead.
-#
-# Runs the same client workload twice against a rippled cluster:
-#   1. Baseline: telemetry disabled ([telemetry] enabled=0)
-#   2. Telemetry: full telemetry enabled (traces + native OTel metrics)
-#
-# Both arms drive rpc_load_generator.py and tx_submitter.py at one fixed rate for
-# the whole sample window, so the delta is attributable to telemetry rather than
-# to a difference in offered load. The workload is not optional: with only the
-# sampler's own ~1 request/sec of server_info, the tx.*, txq.*, transactor-stage
-# and non-server_info rpc.command.* spans are never entered, and those are where
-# a per-operation span cost shows up. A pass on an idle cluster says nothing.
-#
-# Compares CPU, memory, RPC latency, TPS, and mean consensus round time.
-# Outputs a Markdown table with pass/fail against configured thresholds.
-#
-# Usage:
-#   ./benchmark.sh --xrpld /path/to/xrpld --duration 300
-#
-# Thresholds (configurable via environment variables):
-#   BENCH_CPU_OVERHEAD_PCT=3       CPU overhead < 3%
-#   BENCH_MEM_OVERHEAD_MB=5        Memory overhead < 5MB
-#   BENCH_RPC_LATENCY_IMPACT_MS=2  RPC p99 latency impact < 2ms
-#   BENCH_TPS_IMPACT_PCT=5         Throughput impact < 5%
-#   BENCH_CONSENSUS_IMPACT_PCT=1   Consensus round time impact < 1%
-#
-# Exit codes:
-#   0  Every overhead metric was measured and is within its threshold.
-#   1  Every overhead metric was measured and at least one exceeded its
-#      threshold. This is the only "telemetry is too expensive" signal.
-#      Also returned for a command-line usage error, which the pipeline
-#      cannot trigger (run-full-validation.sh passes a fixed flag list).
-#   2  The overhead could not be measured at all — a missing prerequisite, a
-#      cluster that never reached consensus, or an incomplete metric
-#      collection. Nothing was compared, so nothing was breached.
-#
-# run-full-validation.sh depends on that split: it folds 1 into its own
-# "checks failed" exit and 2 into its "infrastructure error" exit. Reporting a
-# run that measured nothing as a threshold breach would be a false regression.
-
-set -euo pipefail
-
-# ---------------------------------------------------------------------------
-# Colored output helpers
-# ---------------------------------------------------------------------------
-log() { printf "\033[1;34m[BENCH]\033[0m  %s\n" "$*"; }
-ok() { printf "\033[1;32m[BENCH]\033[0m  %s\n" "$*"; }
-warn() { printf "\033[1;33m[BENCH]\033[0m  %s\n" "$*"; }
-fail() { printf "\033[1;31m[BENCH]\033[0m  %s\n" "$*"; }
-
-# Usage error. Exit 1 by shell convention; see the exit-code block above.
-die() {
-    fail "$*" >&2
-    exit 1
-}
-
-# Fatal, and no overhead figure was produced. Exit 2 keeps exit 1 exclusively
-# for a measured threshold breach, so the caller never grades a run that
-# measured nothing as a performance regression.
-cannot_measure() {
-    fail "$*" >&2
-    exit 2
-}
-
-# ---------------------------------------------------------------------------
-# Defaults and thresholds
-# ---------------------------------------------------------------------------
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
-
-# Configurable thresholds via environment variables.
-CPU_THRESHOLD="${BENCH_CPU_OVERHEAD_PCT:-3}"
-MEM_THRESHOLD="${BENCH_MEM_OVERHEAD_MB:-5}"
-RPC_THRESHOLD="${BENCH_RPC_LATENCY_IMPACT_MS:-2}"
-TPS_THRESHOLD="${BENCH_TPS_IMPACT_PCT:-5}"
-CONSENSUS_THRESHOLD="${BENCH_CONSENSUS_IMPACT_PCT:-1}"
-
-XRPLD="${BENCH_XRPLD:-$REPO_ROOT/.build/xrpld}"
-DURATION=300
-NUM_NODES=3
-WORKDIR="/tmp/xrpld-benchmark"
-RESULTS_DIR="$SCRIPT_DIR/benchmark-results"
-RPC_PORT_BASE=5020
-PEER_PORT_BASE=51250
-# Above run-full-validation.sh's 6006.. so both harnesses can share a box.
-WS_PORT_BASE=6020
-
-# Name stem of each node's directory: node i lives in $WORKDIR/$NODE_PREFIX-$i.
-# Every node path and kill pattern below derives from it, and so does the
-# service_instance_id the telemetry arm reports. The telemetry arm exports to the
-# same endpoint the workload harness uses, so a stem that differs from that
-# harness's is what keeps the two clusters apart in the backend. Deriving both
-# from one value is what stops the directory and the reported id drifting.
-NODE_PREFIX="bench-node"
-
-# Head start the generators get before the sampler opens its window.
-# tx_submitter.py creates and funds eight accounts from genesis and then waits
-# for those payments to validate, so without a lead the first seconds of every
-# window carry no transaction load. Identical in both arms, so it cancels out.
-WORKLOAD_LEAD_SEC=20
-
-# One flat offered rate, not a workload-profiles.json profile: both arms must
-# issue the same work for the delta to mean anything, and a profile's phase
-# shaping only adds variance. Payment-only for the same reason -- a rejected
-# transaction costs a different amount of work than an applied one.
-WORKLOAD_RPC_RATE="${BENCH_RPC_RATE:-30}"
-WORKLOAD_TX_TPS="${BENCH_TX_TPS:-3}"
-
-# This arm's generator pids, reaped by wait_workload and killed by the trap.
-WORKLOAD_PIDS=()
-
-# Hard ceiling on every RPC probe below. curl applies no overall timeout of its
-# own, so a node that accepts the connection and then stops answering parks the
-# poll loop for the rest of the run. The loops here count attempts, not seconds,
-# so without this their stated timeouts are not bounds at all.
-CURL_MAX_TIME="${CURL_MAX_TIME:-5}"
-
-# ---------------------------------------------------------------------------
-# Argument parsing
-# ---------------------------------------------------------------------------
-usage() {
-    echo "Usage: $0 [OPTIONS]"
-    echo ""
-    echo "Options:"
-    echo "  --xrpld PATH      Path to xrpld binary (default: \$REPO_ROOT/.build/xrpld)"
-    echo "  --duration SECS   Benchmark duration per run (default: 300)"
-    echo "  --nodes NUM       Number of validator nodes (default: 3)"
-    echo "  --output DIR      Results output directory"
-    echo "  -h, --help        Show this help"
-    exit 0
-}
-
-while [ $# -gt 0 ]; do
-    case "$1" in
-        --xrpld)
-            XRPLD="$2"
-            shift 2
-            ;;
-        --duration)
-            DURATION="$2"
-            shift 2
-            ;;
-        --nodes)
-            NUM_NODES="$2"
-            shift 2
-            ;;
-        --output)
-            RESULTS_DIR="$2"
-            shift 2
-            ;;
-        -h | --help) usage ;;
-        *) die "Unknown option: $1" ;;
-    esac
-done
-
-# Validate prerequisites. A missing binary or tool means no measurement can be
-# taken, which is "cannot measure", not "too slow".
-# Validate the two numeric options before anything derives ports, loop counts or
-# pid-count guards from them. --nodes 0 is the dangerous one: node_pids_csv then
-# yields an empty list, awk reports 0 fields, and the "found N of NUM_NODES pids"
-# guard compares 0 with 0 and passes, handing the sampler no pids at all.
-case "$NUM_NODES" in
-    '' | *[!0-9]*) cannot_measure "--nodes must be a positive integer, got '$NUM_NODES'" ;;
-esac
-[ "$NUM_NODES" -ge 1 ] || cannot_measure "--nodes must be at least 1, got '$NUM_NODES'"
-case "$DURATION" in
-    '' | *[!0-9]*) cannot_measure "--duration must be a positive integer, got '$DURATION'" ;;
-esac
-[ "$DURATION" -ge 1 ] || cannot_measure "--duration must be at least 1, got '$DURATION'"
-
-[ -x "$XRPLD" ] || cannot_measure "xrpld not found at $XRPLD"
-command -v jq >/dev/null 2>&1 || cannot_measure "jq not found"
-command -v bc >/dev/null 2>&1 || cannot_measure "bc not found"
-command -v curl >/dev/null 2>&1 || cannot_measure "curl not found"
-command -v python3 >/dev/null 2>&1 ||
-    cannot_measure "python3 not found (the load generators need it)"
-python3 -c 'import websockets' 2>/dev/null ||
-    cannot_measure "python3 'websockets' package not found -- pip install -r $SCRIPT_DIR/requirements.txt"
-
-mkdir -p "$RESULTS_DIR" || cannot_measure "Could not create the results directory $RESULTS_DIR"
-TIMESTAMP=$(date +%Y%m%d_%H%M%S)
-
-# ---------------------------------------------------------------------------
-# Node cluster management
-# ---------------------------------------------------------------------------
-
-# True while xrpld children spawned by start_cluster may still be alive.
-# Read by stop_cluster so it can be called any number of times.
-CLUSTER_RUNNING=false
-
-start_cluster() {
-    local telemetry_enabled="$1"
-    local label="$2"
-
-    log "Starting $NUM_NODES-node cluster ($label, telemetry=$telemetry_enabled)..."
-
-    rm -rf "$WORKDIR" || cannot_measure "Could not clear the workdir $WORKDIR"
-    mkdir -p "$WORKDIR" || cannot_measure "Could not create the workdir $WORKDIR"
-
-    # Generate keys using a temporary standalone node. The helper fails through
-    # its own die(), which exits 1 -- the code this script reserves for a
-    # measured breach. Remap it, or a keygen failure reads as "too expensive".
-    bash "$SCRIPT_DIR/generate-validator-keys.sh" "$XRPLD" "$NUM_NODES" "$WORKDIR" ||
-        cannot_measure "generate-validator-keys.sh failed; no keys for the $NUM_NODES-node cluster"
-
-    # Set before the spawn loop so a failure part-way through it still gets
-    # cleaned up by the EXIT trap.
-    CLUSTER_RUNNING=true
-
-    # Build per-node configs.
-    for i in $(seq 1 "$NUM_NODES"); do
-        local node_dir="$WORKDIR/$NODE_PREFIX-$i"
-        mkdir -p "$node_dir/nudb" "$node_dir/db" ||
-            cannot_measure "Could not create node$i directories under $node_dir"
-
-        local rpc_port
-        rpc_port=$((RPC_PORT_BASE + i - 1))
-        local peer_port
-        peer_port=$((PEER_PORT_BASE + i - 1))
-        local ws_port
-        ws_port=$((WS_PORT_BASE + i - 1))
-        # Split from the declaration on purpose: `local seed=$(...)` reports
-        # local's status, not jq's, so the guard below would never fire.
-        local seed
-        seed=$(jq -r ".[$((i - 1))].seed" "$WORKDIR/validator-keys.json") ||
-            cannot_measure "Could not read node$i's seed from $WORKDIR/validator-keys.json"
-        # jq prints "null" and exits 0 when the array is short, so the exit
-        # status alone does not catch a truncated key file.
-        case "$seed" in
-            "" | null)
-                cannot_measure "node$i has no seed in $WORKDIR/validator-keys.json"
-                ;;
-        esac
-
-        # Build ips_fixed list.
-        local ips_fixed=""
-        for j in $(seq 1 "$NUM_NODES"); do
-            if [ "$j" -ne "$i" ]; then
-                ips_fixed="${ips_fixed}127.0.0.1 $((PEER_PORT_BASE + j - 1))
-"
-            fi
-        done
-
-        # Build telemetry section.
-        local telemetry_section=""
-        if [ "$telemetry_enabled" = "1" ]; then
-            telemetry_section="
-[telemetry]
-enabled=1
-service_instance_id=$NODE_PREFIX-${i}
-endpoint=http://localhost:4318/v1/traces
-exporter=otlp_http
-batch_size=512
-batch_delay_ms=2000
-max_queue_size=2048
-trace_rpc=1
-trace_transactions=1
-trace_consensus=1
-trace_peer=1
-trace_ledger=1
-
-[insight]
-# No prefix is set on purpose: OTelCollector applies no prefix to instrument
-# names, so setting one here would be inert. Same reasoning as
-# run-full-validation.sh.
-server=otel
-endpoint=http://localhost:4318/v1/metrics"
-        else
-            telemetry_section="
-[telemetry]
-enabled=0"
-        fi
-
-        # Guarded like every other fallible write. An unwritable node_dir would
-        # otherwise surface as cat's status 1, the code this script reserves for a
-        # measured threshold breach.
-        cat >"$node_dir/xrpld.cfg" <<EOCFG || cannot_measure "Could not write $node_dir/xrpld.cfg"
-[server]
-port_rpc
-port_ws
-port_peer
-
-[port_rpc]
-port = $rpc_port
-ip = 127.0.0.1
-admin = 127.0.0.1
-protocol = http
-
-# The generators speak WebSocket only. admin is required rather than cosmetic:
-# tx_submitter.py calls wallet_propose and submits with "secret". Declared in
-# BOTH arms, so the listener itself is not part of the measured delta.
-[port_ws]
-port = $ws_port
-ip = 127.0.0.1
-admin = 127.0.0.1
-protocol = ws
-
-[port_peer]
-port = $peer_port
-ip = 0.0.0.0
-protocol = peer
-
-[node_db]
-type=NuDB
-path=$node_dir/nudb
-online_delete=256
-
-[database_path]
-$node_dir/db
-
-[debug_logfile]
-$node_dir/debug.log
-
-[validation_seed]
-$seed
-
-[validators_file]
-$WORKDIR/validators.txt
-
-[ips_fixed]
-${ips_fixed}
-[peer_private]
-1
-${telemetry_section}
-
-[rpc_startup]
-# The log level deliberately diverges from run-full-validation.sh, which uses
-# info. This script measures telemetry OVERHEAD as the delta
-# between a telemetry-off and a telemetry-on arm, so it needs a quiet, identical
-# baseline in both arms rather than good observability. Logging is synchronous, so
-# admitting more log lines would add I/O to both arms and inflate the CPU,
-# memory and latency figures the thresholds gate. Do not "align" this with the
-# workload harness -- that would silently degrade the measurement.
-{ "command": "log_level", "severity": "warning" }
-
-[ssl_verify]
-0
-EOCFG
-
-        "$XRPLD" --conf "$node_dir/xrpld.cfg" --start >"$node_dir/stdout.log" 2>&1 &
-        echo $! >"$node_dir/xrpld.pid"
-    done
-
-    # Wait for consensus. Reaching the limit is fatal: numbers taken from a
-    # cluster that never got to "proposing" would silently corrupt the
-    # baseline-vs-telemetry comparison.
-    local max_wait=120
-    log "Waiting for consensus (up to ${max_wait}s)..."
-    for attempt in $(seq 1 "$max_wait"); do
-        local ready=0
-        for i in $(seq 1 "$NUM_NODES"); do
-            local port
-            port=$((RPC_PORT_BASE + i - 1))
-            local state
-            state=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$port" \
-                -d '{"method":"server_info"}' 2>/dev/null |
-                jq -r '.result.info.server_state' 2>/dev/null || echo "")
-            if [ "$state" = "proposing" ]; then
-                ready=$((ready + 1))
-            fi
-        done
-        if [ "$ready" -ge "$NUM_NODES" ]; then
-            ok "All $NUM_NODES nodes proposing (attempt $attempt)"
-            break
-        fi
-        if [ "$attempt" -eq "$max_wait" ]; then
-            cannot_measure "Consensus timeout — only $ready/$NUM_NODES nodes proposing after ${max_wait}s"
-        fi
-        sleep 1
-    done
-
-    # Let the cluster stabilize.
-    sleep 5
-}
-
-stop_cluster() {
-    # Idempotent. The happy path calls this directly and the EXIT trap calls
-    # it again, so a second call must not re-kill or log a misleading message.
-    [ "$CLUSTER_RUNNING" = true ] || return 0
-    CLUSTER_RUNNING=false
-
-    log "Stopping cluster..."
-    for i in $(seq 1 "$NUM_NODES"); do
-        local pidfile="$WORKDIR/$NODE_PREFIX-$i/xrpld.pid"
-        if [ -f "$pidfile" ]; then
-            kill "$(cat "$pidfile")" 2>/dev/null || true
-        fi
-    done
-    # Belt and braces for a node whose pidfile is missing or stale. Matched on the
-    # per-node config path start_cluster launches with, not the workdir alone: a
-    # workdir-only pattern would also match anything that merely mentions the
-    # tree, such as a developer's `tail -f` on a node's debug.log.
-    pkill -f "$WORKDIR/$NODE_PREFIX-[0-9]+/xrpld\.cfg" 2>/dev/null || true
-
-    # Guarded on purpose. This runs as the EXIT trap, where any unguarded
-    # failure makes `set -e` exit with that command's status and discard the
-    # status the script meant to report — a threshold breach would surface as
-    # a plain 1 and "cannot measure" would lose its 2. With every command here
-    # guarded, the explicit `return 0` below is reachable and authoritative.
-    sleep 3 || true
-
-    return 0
-}
-
-# Build RPC ports CSV string.
-rpc_ports_csv() {
-    local ports=""
-    for i in $(seq 1 "$NUM_NODES"); do
-        [ -n "$ports" ] && ports="$ports,"
-        ports="$ports$((RPC_PORT_BASE + i - 1))"
-    done
-    echo "$ports"
-}
-
-# Collects one leg of the benchmark.
-#
-# The collector exits non-zero when it cannot run (1) or when a measurement
-# source came back empty (3). An all-zero or partial sample set clears every
-# threshold, so an incomplete leg aborts with "cannot measure" instead of being
-# compared and passed.
-# Echoes one ws:// endpoint per node, space separated.
-ws_endpoints() {
-    local i out=""
-    for i in $(seq 1 "$NUM_NODES"); do
-        out="$out ws://localhost:$((WS_PORT_BASE + i - 1))"
-    done
-    printf '%s' "${out# }"
-}
-
-# This cluster's xrpld pids, comma separated, for the sampler's process filter.
-# Without it the sampler matches every xrpld on the host: run-full-validation.sh
-# leaves five validation nodes running while these three start, so both arms
-# average eight processes and the delta is diluted away.
-node_pids_csv() {
-    local i out="" pid
-    for i in $(seq 1 "$NUM_NODES"); do
-        pid=$(cat "$WORKDIR/$NODE_PREFIX-$i/xrpld.pid" 2>/dev/null) || continue
-        [ -n "$pid" ] && out="$out,$pid"
-    done
-    printf '%s' "${out#,}"
-}
-
-# Starts this arm's generators, then waits out the funding lead so the sampler's
-# whole window is under load. Logs and JSON summaries go to RESULTS_DIR, not
-# WORKDIR: the next arm's start_cluster rm -rf's WORKDIR.
-start_workload() {
-    local label="$1"
-    local gen_duration=$((DURATION + WORKLOAD_LEAD_SEC))
-    local logdir="$RESULTS_DIR/workload-${TIMESTAMP}"
-    mkdir -p "$logdir" || cannot_measure "Could not create the workload log dir $logdir"
-
-    log "Starting workload ($label): ${WORKLOAD_RPC_RATE} rpc/s + ${WORKLOAD_TX_TPS} tps for ${gen_duration}s..."
-
-    # shellcheck disable=SC2046 # --endpoints takes a list; splitting is intended
-    python3 "$SCRIPT_DIR/rpc_load_generator.py" \
-        --endpoints $(ws_endpoints) \
-        --rate "$WORKLOAD_RPC_RATE" \
-        --duration "$gen_duration" \
-        --output "$logdir/$label-rpc.json" \
-        >"$logdir/$label-rpc.log" 2>&1 &
-    WORKLOAD_PIDS+=("$!")
-
-    python3 "$SCRIPT_DIR/tx_submitter.py" \
-        --endpoint "ws://localhost:$WS_PORT_BASE" \
-        --tps "$WORKLOAD_TX_TPS" \
-        --duration "$gen_duration" \
-        --weights '{"Payment": 100}' \
-        --output "$logdir/$label-tx.json" \
-        >"$logdir/$label-tx.log" 2>&1 &
-    WORKLOAD_PIDS+=("$!")
-
-    sleep "$WORKLOAD_LEAD_SEC"
-}
-
-# Reaps this arm's generators. A non-zero generator means the arms did not do
-# the same work, so nothing is attributable: "cannot measure", never "too slow".
-wait_workload() {
-    local label="$1"
-    local pid status=0
-    for pid in ${WORKLOAD_PIDS[@]+"${WORKLOAD_PIDS[@]}"}; do
-        wait "$pid" || status=$?
-    done
-    WORKLOAD_PIDS=()
-    [ "$status" -eq 0 ] ||
-        cannot_measure "$label workload generator exited $status; the arms did not do identical work -- see $RESULTS_DIR/workload-${TIMESTAMP}/"
-}
-
-# Kills any generator still running, so an aborted arm leaves no python3 holding
-# WebSocket connections. Guarded like stop_cluster's commands: this runs from the
-# EXIT trap, where an unguarded failure would discard the real exit status.
-stop_workload() {
-    local pid
-    for pid in ${WORKLOAD_PIDS[@]+"${WORKLOAD_PIDS[@]}"}; do
-        kill "$pid" 2>/dev/null || true
-    done
-    WORKLOAD_PIDS=()
-    return 0
-}
-
-# Reap the workload and the cluster on every exit path. Installed below both
-# handlers, not after argument parsing: if the trap fires while either name is
-# still undefined, errexit aborts the handler on "command not found" and the
-# rest of it -- the cluster reap -- never runs. Without the trap, any failure
-# between start_cluster and stop_cluster leaks the xrpld children along with
-# their RPC ports (5020+) and peer ports (51250+).
-trap 'stop_workload; stop_cluster' EXIT
-
-collect_metrics() {
-    local label="$1"
-    local out_file="$2"
-
-    local status=0
-    # An empty or short list would silently fall back to host-wide sampling,
-    # which is the outcome the pid argument exists to prevent.
-    local pids
-    pids=$(node_pids_csv)
-    local n_pids
-    n_pids=$(printf '%s' "$pids" | awk -F, '{print NF}')
-    [ "${n_pids:-0}" -eq "$NUM_NODES" ] ||
-        cannot_measure "$label: found $n_pids of $NUM_NODES node pids, so the sampler cannot be scoped to this cluster"
-
-    bash "$SCRIPT_DIR/collect_system_metrics.sh" \
-        "$(rpc_ports_csv)" "$DURATION" "$out_file" "$pids" || status=$?
-    [ "$status" -eq 0 ] ||
-        cannot_measure "$label metric collection failed (exit $status) — refusing to compare an incomplete run"
-
-    # Only an explicit false counts. The flag is absent from older artifacts,
-    # and jq's "//" operator would turn a real false into the default.
-    local complete
-    complete=$(jq -r '.metrics_complete' "$out_file" 2>/dev/null || echo "null")
-    [ "$complete" != "false" ] ||
-        cannot_measure "$label metrics are flagged incomplete — refusing to compare an incomplete run"
-}
-
-# ---------------------------------------------------------------------------
-# Run benchmark
-# ---------------------------------------------------------------------------
-log "="
-log "  rippled Telemetry Performance Benchmark"
-log "  Nodes: $NUM_NODES | Duration: ${DURATION}s | Binary: $XRPLD"
-log "="
-
-# --- Baseline run ---
-BASELINE_FILE="$RESULTS_DIR/baseline-${TIMESTAMP}.json"
-start_cluster "0" "baseline"
-start_workload "baseline"
-collect_metrics "baseline" "$BASELINE_FILE"
-wait_workload "baseline"
-stop_cluster
-
-# --- Telemetry run ---
-TELEMETRY_FILE="$RESULTS_DIR/telemetry-${TIMESTAMP}.json"
-start_cluster "1" "telemetry"
-start_workload "telemetry"
-collect_metrics "telemetry" "$TELEMETRY_FILE"
-wait_workload "telemetry"
-stop_cluster
-
-# ---------------------------------------------------------------------------
-# Compare results
-# ---------------------------------------------------------------------------
-log "Comparing results..."
-
-# Written into an impact variable when its baseline could not be used.
-# check_threshold turns this into an INCONCLUSIVE verdict.
-INCONCLUSIVE="n/a"
-
-read_metric() {
-    local file="$1"
-    local key="$2"
-    # jq exits 5 on malformed JSON and 2 on a missing file. Neither is one of
-    # this script's three documented codes, and a bare assignment would let it
-    # escape through errexit, so route both through cannot_measure.
-    jq -r ".$key // 0" "$file" 2>/dev/null ||
-        cannot_measure "$file is not readable JSON — refusing to compare an unreadable run"
-}
-
-BASE_CPU=$(read_metric "$BASELINE_FILE" "cpu_pct_avg")
-TELE_CPU=$(read_metric "$TELEMETRY_FILE" "cpu_pct_avg")
-CPU_DELTA=$(echo "scale=2; $TELE_CPU - $BASE_CPU" | bc 2>/dev/null || echo "$INCONCLUSIVE")
-
-BASE_MEM=$(read_metric "$BASELINE_FILE" "memory_rss_mb_peak")
-TELE_MEM=$(read_metric "$TELEMETRY_FILE" "memory_rss_mb_peak")
-MEM_DELTA=$(echo "scale=2; $TELE_MEM - $BASE_MEM" | bc 2>/dev/null || echo "$INCONCLUSIVE")
-
-BASE_RPC=$(read_metric "$BASELINE_FILE" "rpc_p99_ms")
-TELE_RPC=$(read_metric "$TELEMETRY_FILE" "rpc_p99_ms")
-RPC_DELTA=$(echo "scale=2; $TELE_RPC - $BASE_RPC" | bc 2>/dev/null || echo "$INCONCLUSIVE")
-
-# Both impacts below are ratios of the baseline, so a non-positive baseline
-# leaves them undefined. Reporting that as "0% impact" would clear the
-# threshold and hide a failed baseline run. The reachable case is not the
-# collector's tps=0 placeholder -- that marks the run incomplete and stops it
-# earlier -- but quantization: tps is printed to two decimals, so a barely
-# advancing cluster yields "0.00" with the metrics still flagged complete.
-#
-# Both expressions scale by 100 before dividing. bc truncates at "scale" after
-# every operation, so dividing first would floor the ratio to 2 decimals and
-# then multiply the lost precision by 100 — a real 1.25% consensus impact came
-# out as exactly 1.00 and passed the 1% threshold.
-BASE_TPS=$(read_metric "$BASELINE_FILE" "tps")
-TELE_TPS=$(read_metric "$TELEMETRY_FILE" "tps")
-if [[ "$(echo "$BASE_TPS > 0" | bc 2>/dev/null)" = "1" ]]; then
-    TPS_IMPACT=$(echo "scale=2; ($BASE_TPS - $TELE_TPS) * 100 / $BASE_TPS" | bc 2>/dev/null || echo "$INCONCLUSIVE")
-else
-    TPS_IMPACT="$INCONCLUSIVE"
-fi
-
-BASE_CONS=$(read_metric "$BASELINE_FILE" "consensus_round_mean_ms")
-TELE_CONS=$(read_metric "$TELEMETRY_FILE" "consensus_round_mean_ms")
-if [[ "$(echo "$BASE_CONS > 0" | bc 2>/dev/null)" = "1" ]]; then
-    CONS_IMPACT=$(echo "scale=2; ($TELE_CONS - $BASE_CONS) * 100 / $BASE_CONS" | bc 2>/dev/null || echo "$INCONCLUSIVE")
-else
-    CONS_IMPACT="$INCONCLUSIVE"
-fi
-
-# ---------------------------------------------------------------------------
-# Pass/fail checks
-# ---------------------------------------------------------------------------
-PASS_COUNT=0
-FAIL_COUNT=0
-INCONCLUSIVE_COUNT=0
-
-# Records the verdict for one row of the report.
-#
-# Arguments: metric name, measured value, threshold, unit, and the name of the
-# variable to write the bare verdict into.
-#
-# The verdict travels through that named variable and every diagnostic goes to
-# stderr. Calling this through a command substitution would run it in a
-# subshell, which drops the counter updates and captures the colored log line
-# into the caller's variable.
-check_threshold() {
-    local name="$1"
-    local actual="$2"
-    local threshold="$3"
-    local unit="$4"
-    local result_var="$5"
-
-    # Unusable measurement. Counted as a failure so the exit gate fires: an
-    # undefined result must never read as a pass.
-    if [ "$actual" = "$INCONCLUSIVE" ]; then
-        fail "$name: INCONCLUSIVE — the baseline was zero or missing, or the arithmetic failed" >&2
-        FAIL_COUNT=$((FAIL_COUNT + 1))
-        INCONCLUSIVE_COUNT=$((INCONCLUSIVE_COUNT + 1))
-        printf -v "$result_var" 'INCONCLUSIVE'
-        return
-    fi
-
-    # Compare: actual <= threshold
-    if [[ "$(echo "$actual <= $threshold" | bc 2>/dev/null)" = "1" ]]; then
-        ok "$name: ${actual}${unit} <= ${threshold}${unit} PASS" >&2
-        PASS_COUNT=$((PASS_COUNT + 1))
-        printf -v "$result_var" 'PASS'
-    else
-        fail "$name: ${actual}${unit} > ${threshold}${unit} FAIL" >&2
-        FAIL_COUNT=$((FAIL_COUNT + 1))
-        printf -v "$result_var" 'FAIL'
-    fi
-}
-
-# Formats a delta for the report table: appends the unit to a real number and
-# leaves the INCONCLUSIVE placeholder bare.
-fmt_delta() {
-    local value="$1"
-    local unit="$2"
-    if [ "$value" = "$INCONCLUSIVE" ]; then
-        printf '%s' "$value"
-    else
-        printf '%s%s' "$value" "$unit"
-    fi
-}
-
-check_threshold "CPU overhead" "$CPU_DELTA" "$CPU_THRESHOLD" "%" CPU_RESULT
-check_threshold "Memory overhead" "$MEM_DELTA" "$MEM_THRESHOLD" "MB" MEM_RESULT
-check_threshold "RPC p99 impact" "$RPC_DELTA" "$RPC_THRESHOLD" "ms" RPC_RESULT
-check_threshold "TPS impact" "$TPS_IMPACT" "$TPS_THRESHOLD" "%" TPS_RESULT
-check_threshold "Consensus impact" "$CONS_IMPACT" "$CONSENSUS_THRESHOLD" "%" CONS_RESULT
-
-# ---------------------------------------------------------------------------
-# Output Markdown table
-# ---------------------------------------------------------------------------
-REPORT_FILE="$RESULTS_DIR/benchmark-report-${TIMESTAMP}.md"
-
-cat >"$REPORT_FILE" <<EOMD
-# Telemetry Performance Benchmark Report
-
-**Date**: $(date -u +"%Y-%m-%d %H:%M:%S UTC")
-**Nodes**: $NUM_NODES | **Duration**: ${DURATION}s per run
-**Binary**: $XRPLD
-
-## Results
-
-| Metric | Baseline | Telemetry | Delta | Threshold | Result |
-|--------|----------|-----------|-------|-----------|--------|
-| CPU (avg %) | ${BASE_CPU}% | ${TELE_CPU}% | ${CPU_DELTA}% | < ${CPU_THRESHOLD}% | ${CPU_RESULT} |
-| Memory RSS (peak MB) | ${BASE_MEM} MB | ${TELE_MEM} MB | ${MEM_DELTA} MB | < ${MEM_THRESHOLD} MB | ${MEM_RESULT} |
-| RPC p99 Latency (ms) | ${BASE_RPC} ms | ${TELE_RPC} ms | ${RPC_DELTA} ms | < ${RPC_THRESHOLD} ms | ${RPC_RESULT} |
-| Throughput (TPS) | ${BASE_TPS} | ${TELE_TPS} | $(fmt_delta "$TPS_IMPACT" "%") | < ${TPS_THRESHOLD}% | ${TPS_RESULT} |
-| Consensus Round Mean (ms) | ${BASE_CONS} ms | ${TELE_CONS} ms | $(fmt_delta "$CONS_IMPACT" "%") | < ${CONSENSUS_THRESHOLD}% | ${CONS_RESULT} |
-
-\`INCONCLUSIVE\` means the baseline for that row was zero or missing, so the
-impact could not be computed. Such rows count as failures.
-
-\`Consensus Round Mean\` is the mean inter-ledger interval derived from the
-collector's 2 s ledger-sequence samples, not a percentile.
-
-## Summary
-
-- **Passed**: $PASS_COUNT / $((PASS_COUNT + FAIL_COUNT))
-- **Failed**: $FAIL_COUNT / $((PASS_COUNT + FAIL_COUNT))
-- **Inconclusive**: $INCONCLUSIVE_COUNT (included in Failed)
-
-## Raw Data
-
-- Baseline: \`$(basename "$BASELINE_FILE")\`
-- Telemetry: \`$(basename "$TELEMETRY_FILE")\`
-EOMD
-
-ok "Benchmark report written to $REPORT_FILE"
-cat "$REPORT_FILE"
-
-# Exit per the code table at the top of this file. A breach and an unmeasurable
-# row both have to fail the gate, but they are different codes: exit 1 promises
-# every metric was measured, so a run carrying an INCONCLUSIVE row reports 2
-# instead. FAIL_COUNT includes the inconclusive rows, so subtract them to see
-# whether any real threshold was exceeded.
-if [ "$((FAIL_COUNT - INCONCLUSIVE_COUNT))" -gt 0 ]; then
-    exit 1
-elif [ "$INCONCLUSIVE_COUNT" -gt 0 ]; then
-    fail "$INCONCLUSIVE_COUNT overhead row(s) could not be measured — reporting an infrastructure error, not a breach" >&2
-    exit 2
-fi

docker/telemetry/workload/capture_timings.py

@@ -1,273 +0,0 @@
-#!/usr/bin/env python3
-"""Capture OTel-derived timings from Prometheus for the regression gate.
-
-Queries Prometheus for every metric declared in ``regression-metrics.json``
-and writes the results to a JSON file in the exact schema
-``baseline-timings.json`` expects. When a user wants to refresh the
-baseline, they copy a CI run's ``timings.json`` artifact (or the block
-printed to the workflow step summary) into
-``baselines/baseline-timings.json`` in a reviewable PR.
-
-Output schema (stable — ``compare_to_baseline.py`` reads it verbatim)::
-
-    {
-        "schema_version": 1,
-        "captured_at": "2026-04-24T17:30:00Z",
-        "window": "3m",
-        "git_sha": "<from $GITHUB_SHA or `git rev-parse HEAD`>",
-        "profile": "full-validation",
-        "capture": {
-            "declared": 20,
-            "captured": 20,
-            "min_ratio": 0.5,
-            "complete": true
-        },
-        "metrics": {
-            "span.tx.process.p99": {"value": 12.4, "unit": "ms"},
-            "job.transaction.queued.p95": {"value": 850.0, "unit": "us"},
-            ...
-        }
-    }
-
-The ``capture`` block is what makes this file safe to use as baseline material.
-The output is written BEFORE ``--min-capture-ratio`` is enforced, so a run that
-reached too little of Prometheus still leaves a ``timings.json`` behind. That
-file exists, parses, and carries every declared key — some with ``value: null``
-— so a thin capture is indistinguishable from a good one to a reader who only
-checks that the file is there. Pasted into
-``baselines/baseline-timings.json`` it would narrow the gate to whichever keys
-happened to come back, with nothing reporting that the gate had narrowed.
-
-``complete`` is exactly the condition this script exits 0 on. It is computed
-once, in ``_capture_status``, and drives both the exit code and the block, so
-the two cannot disagree. Consumers read the flag rather than re-deriving the
-ratio rule for themselves: the workflow's "Print regression summary" step, the
-paste-me path in ``compare_to_baseline.py``, and a human reading the artifact
-all get the same answer from one place. ``declared``, ``captured`` and
-``min_ratio`` sit alongside it so a rejected capture can be judged without
-re-running it.
-
-The block is additive — a sibling of ``metrics``, never an entry inside it — so
-it is neither a metric key nor a gated entry, and readers that predate it are
-unaffected. Its ABSENCE means an artifact from before it existed, whose
-completeness cannot be established; the paste-me paths treat that as not
-complete rather than as complete.
-
-Usage::
-
-    python3 capture_timings.py \\
-        --prometheus http://localhost:9090 \\
-        --metrics regression-metrics.json \\
-        --output /tmp/timings.json \\
-        --window 3m \\
-        --profile regression
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import json
-import logging
-import os
-import subprocess
-import sys
-from datetime import datetime, timezone
-from pathlib import Path
-
-import aiohttp
-
-from prom_queries import build_query_plan, run_query_plan
-
-logger = logging.getLogger("capture_timings")
-
-SCHEMA_VERSION = 1
-
-
-async def capture(
-    prom_url: str,
-    metrics_path: Path,
-    window: str,
-    profile: str,
-    min_capture_ratio: float,
-) -> dict:
-    """Build and execute the query plan, return the full report dict.
-
-    ``min_capture_ratio`` is recorded in the report rather than only applied to
-    the exit code, so the artifact states the bar it was judged against.
-    """
-    plan = build_query_plan(metrics_path, window=window)
-    logger.info("Capturing %d metrics from %s (window=%s)", len(plan), prom_url, window)
-
-    async with aiohttp.ClientSession() as session:
-        metrics = await run_query_plan(session, prom_url, plan)
-
-    metrics = dict(sorted(metrics.items()))
-    return {
-        "schema_version": SCHEMA_VERSION,
-        "captured_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
-        "window": window,
-        "git_sha": _detect_git_sha(),
-        "profile": profile,
-        "capture": _capture_status(metrics, min_capture_ratio),
-        "metrics": metrics,
-    }
-
-
-def _capture_status(metrics: dict, min_ratio: float) -> dict:
-    """Summarise how much of the declared surface this capture actually got.
-
-    ``declared`` is every key the surface asked for; ``captured`` is how many
-    came back with a value. ``complete`` is the single fact every consumer
-    keys on, and it is the same predicate that decides this script's exit
-    code — see the module docstring for why it lives in the artifact.
-
-    An empty surface is NOT complete. ``declared == 0`` means the capture asked
-    Prometheus for nothing: ``build_query_plan`` returns an empty plan, without
-    complaining, for any config that yields no ``spans``/``rpc_methods``/
-    ``job_queue`` entries -- a ``--metrics`` path pointing at the wrong file, a
-    truncated one, or every key excluded. Nothing about that run is evidence
-    the pipeline works, so treating it as vacuously complete would exit 0 and
-    hand the paste-me path a ``metrics: {}`` artifact to offer as baseline
-    material. Pasted in, it still reads as a placeholder, so the gate stays off
-    while the workflow reports the baseline as activated -- the silent-green
-    outcome the whole ``capture`` block exists to prevent. The exit code reads
-    this same flag, so the two still cannot disagree.
-    """
-    declared = len(metrics)
-    captured = sum(1 for entry in metrics.values() if entry["value"] is not None)
-    return {
-        "declared": declared,
-        "captured": captured,
-        "min_ratio": min_ratio,
-        "complete": declared > 0 and (captured / declared) >= min_ratio,
-    }
-
-
-def _detect_git_sha() -> str:
-    """Return the current commit SHA from env or git, else ``"unknown"``.
-
-    Prefers ``GITHUB_SHA`` (set in Actions), falls back to ``git rev-parse``.
-    Silent fallback is fine here — a missing SHA only affects the captured
-    metadata, not the comparison logic.
-    """
-    env_sha = os.environ.get("GITHUB_SHA")
-    if env_sha:
-        return env_sha
-    try:
-        result = subprocess.run(
-            ["git", "rev-parse", "HEAD"],
-            capture_output=True,
-            text=True,
-            timeout=5,
-            check=False,
-        )
-        if result.returncode == 0:
-            return result.stdout.strip()
-    except (OSError, subprocess.SubprocessError):
-        pass
-    return "unknown"
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument(
-        "--prometheus",
-        default="http://localhost:9090",
-        help="Prometheus base URL (default: http://localhost:9090)",
-    )
-    parser.add_argument(
-        "--metrics",
-        type=Path,
-        default=Path(__file__).parent / "regression-metrics.json",
-        help="Path to regression-metrics.json",
-    )
-    parser.add_argument(
-        "--output",
-        type=Path,
-        required=True,
-        help="Where to write the captured timings JSON",
-    )
-    parser.add_argument(
-        "--window",
-        default="3m",
-        help="Prometheus rate() window (default: 3m)",
-    )
-    parser.add_argument(
-        "--profile",
-        default="full-validation",
-        help=(
-            "Workload profile used during capture, recorded as metadata in the "
-            "timings file (default: full-validation). Must name a profile in "
-            "workload-profiles.json; run-full-validation.sh always passes this "
-            "explicitly."
-        ),
-    )
-    parser.add_argument(
-        "--min-capture-ratio",
-        type=float,
-        default=0.5,
-        help="Fail if fewer than this fraction of metrics are captured (default: 0.5)",
-    )
-    parser.add_argument(
-        "--verbose",
-        action="store_true",
-        help="Enable debug logging",
-    )
-    args = parser.parse_args()
-
-    logging.basicConfig(
-        level=logging.DEBUG if args.verbose else logging.INFO,
-        format="%(levelname)s %(name)s: %(message)s",
-    )
-
-    report = asyncio.run(
-        capture(
-            prom_url=args.prometheus,
-            metrics_path=args.metrics,
-            window=args.window,
-            profile=args.profile,
-            min_capture_ratio=args.min_capture_ratio,
-        )
-    )
-
-    args.output.parent.mkdir(parents=True, exist_ok=True)
-    with open(args.output, "w") as f:
-        json.dump(report, f, indent=2, sort_keys=True)
-        f.write("\n")
-
-    # The exit code is read off the same flag the artifact carries, so a file
-    # marked complete is always one this script exited 0 on.
-    status = report["capture"]
-    captured, total = status["captured"], status["declared"]
-    logger.info("Wrote %s (%d/%d metrics captured)", args.output, captured, total)
-
-    if not status["complete"]:
-        if total == 0:
-            # No ratio to report: nothing was asked for, so the shortfall is the
-            # declared surface, not Prometheus. Named separately because the
-            # percentage below would divide by zero.
-            logger.error(
-                "No metrics were declared, so nothing was captured. Does %s "
-                "declare spans/rpc_methods/job_queue names, and does "
-                "excluded_keys leave any of them gated? The file is marked "
-                "capture.complete=false and must not be pasted into the baseline.",
-                args.metrics,
-            )
-        else:
-            logger.error(
-                "Only %d/%d (%.0f%%) metrics captured — below the %.0f%% minimum. "
-                "Is Prometheus reachable at %s? The file is marked "
-                "capture.complete=false and must not be pasted into the baseline.",
-                captured,
-                total,
-                captured / total * 100,
-                args.min_capture_ratio * 100,
-                args.prometheus,
-            )
-        return 1
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

docker/telemetry/workload/collect_system_metrics.sh

@@ -1,408 +0,0 @@
-#!/usr/bin/env bash
-# collect_system_metrics.sh — Collect CPU, memory, and RPC latency metrics
-# from running xrpld nodes for benchmark comparison.
-#
-# Samples system metrics at regular intervals and writes a JSON summary.
-# Used by benchmark.sh for baseline vs telemetry comparison.
-#
-# Usage:
-#   ./collect_system_metrics.sh <rpc_ports_csv> <duration_seconds> <output_file> [pids_csv]
-#
-# pids_csv narrows process sampling to exactly those pids. Without it the scope
-# is every xrpld on the host, which averages in any other cluster's nodes and
-# reports the largest of them as the RSS peak. benchmark.sh passes its own pids
-# because run-full-validation.sh leaves five validation nodes running while the
-# benchmark's three start, and diluting the arms alike hides the delta.
-#
-# Example:
-#   ./collect_system_metrics.sh "5005,5006,5007" 300 /tmp/metrics-baseline.json
-#   ./collect_system_metrics.sh "5020,5021,5022" 120 /tmp/m.json "8801,8802,8803"
-#
-# Output JSON format:
-#   {
-#     "cpu_pct_avg": 12.5,
-#     "memory_rss_mb_peak": 450.2,
-#     "rpc_p99_ms": 15.3,
-#     "tps": 4.8,
-#     "consensus_round_mean_ms": 3200,
-#     "metrics_complete": true,
-#     "samples": 60
-#   }
-#
-# Exit codes:
-#   0  Every metric was measured; "metrics_complete" is true.
-#   1  Cannot run at all: bad arguments, no GNU date with %N, or a failed
-#      process sample. No output file is written.
-#   3  Output file was written, but at least one measurement source was empty,
-#      so the affected metrics are 0 placeholders and "metrics_complete" is
-#      false. Callers must treat this as inconclusive, never as a pass.
-
-set -euo pipefail
-
-# ---------------------------------------------------------------------------
-# Colored output helpers
-# ---------------------------------------------------------------------------
-log() { printf "\033[1;34m[METRICS]\033[0m %s\n" "$*"; }
-ok() { printf "\033[1;32m[METRICS]\033[0m %s\n" "$*"; }
-# Warnings go to stderr so they never mix into the JSON echoed on stdout.
-warn() { printf "\033[1;33m[METRICS]\033[0m %s\n" "$*" >&2; }
-die() {
-    printf "\033[1;31m[METRICS]\033[0m %s\n" "$*" >&2
-    exit 1
-}
-
-# ---------------------------------------------------------------------------
-# Argument parsing
-# ---------------------------------------------------------------------------
-usage() {
-    echo "Usage: $0 <rpc_ports_csv> <duration_seconds> <output_file>"
-    echo ""
-    echo "Arguments:"
-    echo "  rpc_ports_csv     Comma-separated RPC ports (e.g., 5005,5006,5007)"
-    echo "  duration_seconds  How long to collect metrics"
-    echo "  output_file       Path to write JSON results"
-    echo "  pids_csv          Optional: sample only these pids, not every host xrpld"
-    exit 1
-}
-
-if [ $# -lt 3 ]; then
-    usage
-fi
-
-RPC_PORTS_CSV="$1"
-DURATION="$2"
-OUTPUT_FILE="$3"
-PIDS_CSV="${4:-}"
-
-# Reject a malformed pid list rather than silently sampling the whole host,
-# which is the outcome this argument exists to prevent.
-case "$PIDS_CSV" in
-    '') ;;
-    *[!0-9,]*) die "pids_csv must be a comma-separated list of pids, got '$PIDS_CSV'" ;;
-esac
-
-IFS=',' read -ra RPC_PORTS <<<"$RPC_PORTS_CSV"
-
-# consensus_round_mean_ms below counts DISTINCT ledger sequences seen by this
-# loop, so it cannot resolve a close interval shorter than SAMPLE_INTERVAL. At
-# 5 s it read back 5000 ms for every close time from 2 s to 5 s, so a 10%
-# regression measured 0% and the benchmark's 1% consensus gate could never fire.
-# 2 s is the close-time floor itself (ledgerMinClose, ConsensusParms.h:93), which
-# is enough: measured against this file's own arithmetic, a 10% regression shows
-# up as at least 9.3% anywhere in the 2 s to 5 s band. 1 s only doubles the
-# probe load for slightly worse numbers.
-SAMPLE_INTERVAL=2
-
-# Hard ceiling on every RPC probe below. curl applies no overall timeout of its
-# own, so a node that accepts the connection and then stops answering — what a
-# stalled job queue looks like from outside — parks the sampling loop for the
-# rest of the run. 5 s is some thousands of times a healthy server_info, so it
-# bounds a wedged node's cost to a couple of lost samples while never
-# truncating a real reply. A probe that hits the ceiling exits
-# non-zero and is therefore skipped rather than recorded, which is the same
-# rule the latency loop already applies to a refused connection; if every
-# probe hits it, the empty file trips the placeholder warning below.
-CURL_MAX_TIME=5
-
-# Reject anything the sample arithmetic cannot use, instead of silently
-# treating it as 0.
-case "$DURATION" in
-    '' | *[!0-9]*) die "duration_seconds must be a positive integer, got '$DURATION'" ;;
-esac
-
-# Normalise to base 10 once, right after the digits check. Bash arithmetic
-# reads a leading zero as octal, so "08" aborted with "value too great for
-# base" and "0100" was silently taken as 64. Doing it here also keeps the
-# value a valid JSON number in the output below, where "08" is not.
-DURATION=$((10#$DURATION))
-
-# Round up, so a duration shorter than one interval still takes one sample
-# rather than truncating to zero and emitting an all-zero JSON.
-SAMPLES=$(((DURATION + SAMPLE_INTERVAL - 1) / SAMPLE_INTERVAL))
-if [ "$SAMPLES" -lt 1 ]; then
-    die "duration_seconds=$DURATION yields $SAMPLES samples; need at least 1"
-fi
-
-log "Collecting metrics for ${DURATION}s (${SAMPLES} samples, ${#RPC_PORTS[@]} nodes)..."
-
-# ---------------------------------------------------------------------------
-# Nanosecond clock
-#
-# GNU date supports "+%s%N". BSD/macOS date has no %N and echoes it back
-# literally, which used to abort the sampling loop under `set -e` and still
-# exit 0 with an all-zero JSON — a silent false pass.
-#
-# The clock has to be cheap as well as precise, because the latency it
-# measures is compared against a 2 ms threshold. Measured on one Linux host:
-# `date +%s%N` costs ~1.2 ms per call, forking python3 for the same value ~13 ms.
-# Two calls bracket every request, so a python3 fallback would add ~26 ms of
-# its own overhead to a 2 ms budget and make the number meaningless. There is
-# no cheap alternative worth having, so probe once and refuse to run without
-# GNU date rather than report a figure that is quietly an order of magnitude
-# wrong.
-# ---------------------------------------------------------------------------
-if [[ ! "$(date +%s%N 2>/dev/null)" =~ ^[0-9]+$ ]]; then
-    die "GNU coreutils date with %N is required for RPC latency timing; this date does not support it"
-fi
-
-# Echo the current time in nanoseconds since the epoch.
-now_ns() {
-    date +%s%N
-}
-
-# ---------------------------------------------------------------------------
-# Temporary files for aggregation
-# ---------------------------------------------------------------------------
-TMPDIR_METRICS="$(mktemp -d)"
-CPU_FILE="$TMPDIR_METRICS/cpu.txt"
-MEM_FILE="$TMPDIR_METRICS/mem.txt"
-RPC_FILE="$TMPDIR_METRICS/rpc.txt"
-LEDGER_FILE="$TMPDIR_METRICS/ledger.txt"
-
-touch "$CPU_FILE" "$MEM_FILE" "$RPC_FILE" "$LEDGER_FILE"
-
-cleanup() {
-    rm -rf "$TMPDIR_METRICS"
-}
-trap cleanup EXIT
-
-# ---------------------------------------------------------------------------
-# Get initial ledger sequence for TPS calculation
-# ---------------------------------------------------------------------------
-INITIAL_SEQ=0
-INITIAL_TIME=$(date +%s)
-for port in "${RPC_PORTS[@]}"; do
-    seq=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$port" \
-        -d '{"method":"server_info"}' 2>/dev/null |
-        jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
-    if [ "$seq" -gt "$INITIAL_SEQ" ]; then
-        INITIAL_SEQ=$seq
-    fi
-done
-log "Initial validated ledger seq: $INITIAL_SEQ"
-
-# ---------------------------------------------------------------------------
-# Sampling loop
-# ---------------------------------------------------------------------------
-for sample in $(seq 1 "$SAMPLES"); do
-    # Sample CPU% and RSS for the xrpld processes. One ps pass feeds both
-    # files, so the two numbers always come from the same instant.
-    #
-    # Selection is on argv[0]'s basename, NOT on "the command line mentions
-    # xrpld". The loose form matched every bystander whose command line
-    # happened to contain the string: this harness's own launcher (invoked as
-    # `--xrpld .build/xrpld`, whose argv[0] is bash), an editor's clangd, and
-    # any shell sitting in a directory with xrpld in its path. Measured
-    # against a live 5-node cluster, that pulled in 8-15 processes instead of
-    # 5, halved the CPU average with idle bystanders, and reported clangd's
-    # 1.1 GB RSS as xrpld's peak against a 5 MB threshold. `ps -C xrpld` is
-    # not an alternative: xrpld renames itself, so its comm is "xrpld-main"
-    # and -C matches nothing. "rippled" is accepted alongside "xrpld" so a
-    # rename of the binary cannot silently zero the collector.
-    #
-    # Scope is the pid list when one was given, and the whole host otherwise.
-    # Host scope averages in any other cluster's xrpld and reports the largest
-    # of them as the RSS peak, which reads as noise against a 5 MB threshold.
-    #
-    # A %cpu of exactly 0.0 is a real reading and is counted — dropping idle
-    # samples would inflate the average — while non-numeric output is
-    # rejected by the pattern. An RSS of 0 is not a live process, so it
-    # contributes no memory sample; counting it would leave the file non-empty
-    # and mark a dead cluster's 0 MB peak as a complete measurement.
-    ps -eo pid=,%cpu=,rss=,args= |
-        awk -v cpu_file="$CPU_FILE" -v mem_file="$MEM_FILE" -v pids="$PIDS_CSV" '
-            BEGIN { if (pids != "") { n = split(pids, a, ","); for (i = 1; i <= n; i++) want[a[i]] = 1 } }
-            pids != "" && !($1 in want) { next }
-            pids == "" && $4 !~ /(^|\/)(xrpld|rippled)$/ { next }
-            $2 ~ /^[0-9]+(\.[0-9]+)?$/ { cpu_sum += $2; cpu_n++ }
-            $3 ~ /^[0-9]+$/ && $3 + 0 > 0 { printf("%.2f\n", $3 / 1024) >> mem_file }
-            END {
-                # With an explicit pid list the expected count is known, so a
-                # dead node is detectable. Without it, exit 1 would fire on any
-                # host with no xrpld at all, which the empty-file check below
-                # already reports.
-                if (pids != "" && cpu_n != n) exit 1
-                if (cpu_n > 0) printf("%.2f\n", cpu_sum / cpu_n) >> cpu_file
-            }
-        ' || die "process sampling failed on sample $sample/$SAMPLES: expected $(echo "$PIDS_CSV" | tr -cd , | wc -c)+1 live pids"
-
-    # Collect RPC latency from each node. Only a successful call is a latency
-    # measurement: a refused connection returns in well under a millisecond,
-    # and recording that as ~0 ms would pull the reported p99 down.
-    for port in "${RPC_PORTS[@]}"; do
-        start_ns=$(now_ns)
-        if curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$port" \
-            -d '{"method":"server_info"}' >/dev/null 2>&1; then
-            end_ns=$(now_ns)
-            latency_ms=$(((end_ns - start_ns) / 1000000))
-            echo "$latency_ms" >>"$RPC_FILE"
-        fi
-    done
-
-    # Record current validated ledger seq.
-    for port in "${RPC_PORTS[@]}"; do
-        seq=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$port" \
-            -d '{"method":"server_info"}' 2>/dev/null |
-            jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
-        echo "$seq" >>"$LEDGER_FILE"
-        break # Only need one node's seq per sample.
-    done
-
-    # Progress indicator.
-    if [ $((sample % 10)) -eq 0 ]; then
-        log "  Sample $sample/$SAMPLES..."
-    fi
-
-    sleep "$SAMPLE_INTERVAL"
-done
-
-# ---------------------------------------------------------------------------
-# Compute aggregated metrics
-# ---------------------------------------------------------------------------
-log "Computing aggregated metrics..."
-
-# Cleared by any empty measurement source. A 0 metric is otherwise
-# indistinguishable from a real reading, so the flag is exported in the JSON
-# and drives the exit-3 contract documented in the header.
-METRICS_COMPLETE=true
-
-# CPU average.
-if [ -s "$CPU_FILE" ]; then
-    CPU_AVG=$(awk '{ sum += $1; n++ } END { if (n>0) printf "%.2f", sum/n; else print "0" }' "$CPU_FILE")
-else
-    # Now that the selector cannot match this harness's own processes, an
-    # empty file means no xrpld process was running for any sample.
-    warn "No CPU samples collected (no xrpld process matched); cpu_pct_avg is a 0 placeholder"
-    CPU_AVG="0"
-    METRICS_COMPLETE=false
-fi
-
-# Memory peak RSS (MB).
-if [ -s "$MEM_FILE" ]; then
-    MEM_PEAK=$(sort -n "$MEM_FILE" | tail -1)
-else
-    warn "No memory samples collected (no xrpld process matched); memory_rss_mb_peak is a 0 placeholder"
-    MEM_PEAK="0"
-    METRICS_COMPLETE=false
-fi
-
-# RPC latency p99 (ms).
-if [ -s "$RPC_FILE" ]; then
-    RPC_COUNT=$(wc -l <"$RPC_FILE")
-    # Nearest-rank p99: ceil(count * 99 / 100), clamped into [1, count].
-    # Integer arithmetic avoids both the floor bias of the old bc expression
-    # and the bc dependency, and the lower clamp keeps sed off line address 0
-    # (a file with no trailing newline makes wc -l report 0).
-    P99_INDEX=$(((RPC_COUNT * 99 + 99) / 100))
-    if [ "$P99_INDEX" -lt 1 ]; then
-        P99_INDEX=1
-    fi
-    if [ "$P99_INDEX" -gt "$RPC_COUNT" ]; then
-        P99_INDEX="$RPC_COUNT"
-    fi
-    RPC_P99=$(sort -n "$RPC_FILE" | sed -n "${P99_INDEX}p")
-    if [ -z "$RPC_P99" ]; then
-        warn "RPC latency file has no line $P99_INDEX; rpc_p99_ms is a 0 placeholder"
-        RPC_P99="0"
-        METRICS_COMPLETE=false
-    fi
-else
-    warn "No successful RPC probes; rpc_p99_ms is a 0 placeholder"
-    RPC_P99="0"
-    METRICS_COMPLETE=false
-fi
-
-# TPS calculation from ledger sequence advancement.
-FINAL_SEQ=0
-for port in "${RPC_PORTS[@]}"; do
-    seq=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$port" \
-        -d '{"method":"server_info"}' 2>/dev/null |
-        jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
-    if [ "$seq" -gt "$FINAL_SEQ" ]; then
-        FINAL_SEQ=$seq
-    fi
-done
-FINAL_TIME=$(date +%s)
-ELAPSED=$((FINAL_TIME - INITIAL_TIME))
-LEDGER_ADVANCE=$((FINAL_SEQ - INITIAL_SEQ))
-if [ "$ELAPSED" -gt 0 ] && [ "$LEDGER_ADVANCE" -gt 0 ]; then
-    # Rough TPS: assume ~avg_txs_per_ledger * ledgers / elapsed.
-    # Without tx count, use ledger close rate as proxy.
-    #
-    # awk rather than bc, because bc omits the leading zero: `scale=2` prints
-    # ".25", not "0.25", and a bare ".25" is not valid JSON. A value below 1 is
-    # the normal case here, not an edge case — ledgers close every few seconds,
-    # so advance/elapsed is well under 1 for any realistic window. jq happens
-    # to accept the malformed form, which is why it survived earlier checks,
-    # but a strict parser rejects the whole file. awk's %.2f always pads.
-    TPS=$(awk -v a="$LEDGER_ADVANCE" -v b="$ELAPSED" 'BEGIN { printf "%.2f", a / b }')
-else
-    # No ledger advance means the cluster produced nothing to divide, so 0 here
-    # is an absence of data, not a measured rate. Flagged like every other empty
-    # source so the run reports itself inconclusive instead of a real 0 TPS.
-    warn "Ledger seq did not advance (advance=$LEDGER_ADVANCE over ${ELAPSED}s); tps is a 0 placeholder"
-    TPS="0"
-    METRICS_COMPLETE=false
-fi
-
-# Mean inter-ledger interval in ms: ELAPSED / (distinct ledgers - 1) * 1000.
-#
-# ELAPSED, not DURATION: DURATION is what the caller asked for, while the loop
-# also spends time on its own probes, so it always runs longer. TPS above uses
-# ELAPSED for the same reason.
-#
-# This is a MEAN, not a percentile — the JSON key says so. It still cannot
-# resolve a close interval at or below SAMPLE_INTERVAL, because LEDGER_FILE
-# gets at most one sequence per sample. Read it as a coarse trend.
-if [ -s "$LEDGER_FILE" ]; then
-    UNIQUE_LEDGERS=$(sort -u "$LEDGER_FILE" | wc -l)
-    # The > 1 test also keeps the divisor below at 1 or more.
-    if [ "$UNIQUE_LEDGERS" -gt 1 ]; then
-        # awk rather than bc, for the same reason as the TPS calculation above.
-        # bc is an optional package, and the `|| echo "0"` this replaces turned
-        # a missing or failing bc into a 0 that read exactly like a measured
-        # value: no warn(), and METRICS_COMPLETE left true, so the run reported
-        # a complete measurement and exited 0. awk is already required by the
-        # sampling loop and the CPU average, so computing this with awk removes
-        # the failure path rather than reporting it. The divisor is >= 1 by the
-        # test above.
-        CONSENSUS_MEAN=$(awk -v d="$ELAPSED" -v u="$UNIQUE_LEDGERS" \
-            'BEGIN { printf "%.0f", d * 1000 / (u - 1) }')
-    else
-        warn "Ledger seq never advanced ($UNIQUE_LEDGERS distinct); consensus_round_mean_ms is a 0 placeholder"
-        CONSENSUS_MEAN="0"
-        METRICS_COMPLETE=false
-    fi
-else
-    warn "No ledger samples collected; consensus_round_mean_ms is a 0 placeholder"
-    CONSENSUS_MEAN="0"
-    METRICS_COMPLETE=false
-fi
-
-# ---------------------------------------------------------------------------
-# Write output JSON
-# ---------------------------------------------------------------------------
-cat >"$OUTPUT_FILE" <<EOF_JSON
-{
-  "cpu_pct_avg": $CPU_AVG,
-  "memory_rss_mb_peak": $MEM_PEAK,
-  "rpc_p99_ms": $RPC_P99,
-  "tps": $TPS,
-  "consensus_round_mean_ms": $CONSENSUS_MEAN,
-  "metrics_complete": $METRICS_COMPLETE,
-  "samples": $SAMPLES,
-  "duration_seconds": $DURATION,
-  "node_count": ${#RPC_PORTS[@]},
-  "initial_ledger_seq": $INITIAL_SEQ,
-  "final_ledger_seq": $FINAL_SEQ
-}
-EOF_JSON
-
-ok "Metrics written to $OUTPUT_FILE"
-cat "$OUTPUT_FILE"
-
-# The file is always written first so CI still has an artifact to publish.
-if [ "$METRICS_COMPLETE" != "true" ]; then
-    warn "metrics_complete=false — treat this run as inconclusive (exit 3)"
-    exit 3
-fi

docker/telemetry/workload/compare_to_baseline.py

@@ -1,529 +0,0 @@
-#!/usr/bin/env python3
-"""Compare captured OTel timings against a committed baseline.
-
-Operating modes (chosen automatically based on the baseline file contents):
-
-1. **No baseline** — if ``baseline-timings.json`` has an empty
-   ``metrics`` object (or is marked with ``"placeholder": true``), this
-   script is in "populate" mode. It prints the captured timings JSON in
-   the exact format expected for pasting into
-   ``baselines/baseline-timings.json``, then exits 0. No regression check.
-   An INCOMPLETE capture is refused here instead (exit 2): the timings file
-   states its own completeness in its ``capture`` block, and a capture that
-   fell short of ``--min-capture-ratio`` describes metrics that were never
-   measured. Pasted in, it would narrow the gate to whichever keys came back
-   with nothing reporting that it had narrowed. Refusing prints nothing on
-   stdout, so a caller redirecting stdout to the baseline file cannot end up
-   with a truncated one blessed by exit 0.
-
-2. **Populated baseline** — per-metric percentage AND absolute deltas are
-   computed against thresholds from ``regression-thresholds.json``. A
-   regression occurs when BOTH bounds are breached for the same quantile.
-   The one exception is a non-positive baseline, where the percentage is
-   undefined: there the absolute bound decides alone.
-   Prints a human-readable table and writes a full JSON report.
-   Exits 1 if any regression was detected, else 0.
-
-Inputs:
-    --timings     Captured timings JSON (from capture_timings.py)
-    --baseline    Committed baseline JSON
-    --thresholds  Threshold policy JSON
-    --report      Where to write regression-report.json (optional)
-
-Exit codes:
-    0 — No baseline (paste-me emitted), OR baseline populated and no regression
-    1 — Regression detected (at least one metric breached both bounds)
-    2 — Internal error (e.g. bad JSON, baseline/current key mismatch), OR the
-        baseline is a placeholder and the capture is too incomplete to seed one
-
-Note that an incomplete capture is refused only on the paste-me path. Against a
-POPULATED baseline the comparison still runs and still reports uncaptured keys
-as ``not captured in current run``, exactly as before: there the thin capture
-cannot corrupt anything, and reporting what is missing is more useful than
-refusing to look.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import logging
-import sys
-from dataclasses import dataclass, asdict
-from pathlib import Path
-from typing import Any
-
-logger = logging.getLogger("compare_to_baseline")
-
-
-@dataclass
-class MetricDelta:
-    """Single metric's baseline-vs-current comparison outcome.
-
-    Attributes:
-        key:                Flat metric key (e.g. span.tx.process.p99).
-        baseline:           Baseline value (may be None if unpopulated).
-        current:            Current run value (may be None if not captured).
-        delta:              current - baseline (None if either side None).
-        pct_change:         100 * delta / baseline (None if baseline ≤ 0).
-        unit:               Unit from baseline (preserved as-is).
-        threshold_pct:      Resolved per-metric pct threshold.
-        threshold_abs:      Resolved per-metric absolute threshold.
-        regressed:          True iff both bounds breached, or — when the
-                            baseline is not positive and pct_change is
-                            therefore None — iff the absolute bound breached.
-        note:               Human-readable classification of the outcome.
-    """
-
-    key: str
-    baseline: float | None
-    current: float | None
-    delta: float | None
-    pct_change: float | None
-    unit: str
-    threshold_pct: float | None
-    threshold_abs: float | None
-    regressed: bool
-    note: str
-
-
-def load_json(path: Path) -> dict:
-    with open(path) as f:
-        return json.load(f)
-
-
-def is_placeholder(baseline: dict) -> bool:
-    """A baseline is a placeholder if explicitly marked OR metrics are empty."""
-    if baseline.get("placeholder") is True:
-        return True
-    return not baseline.get("metrics")
-
-
-def capture_is_complete(timings: dict) -> bool:
-    """True only if the timings file states that its capture was complete.
-
-    The flag is written by ``capture_timings.py``, which computes it against
-    ``--min-capture-ratio``; this reads it rather than re-deriving the rule, so
-    the two cannot disagree. Anything other than boolean ``true`` — the block
-    absent because the artifact predates it, a truncated file, a string
-    ``"true"`` from a hand edit — is treated as not complete. Completeness has
-    to be proven, not assumed, because assuming it is how a thin capture
-    reaches a committed baseline.
-    """
-    capture = timings.get("capture")
-    if not isinstance(capture, dict):
-        return False
-    return capture.get("complete") is True
-
-
-def print_incomplete_capture(timings: dict) -> None:
-    """Explain why no paste-me block was printed, naming the shortfall.
-
-    Deliberately writes to stderr only. The paste-me path's stdout is the
-    baseline file's contents, so leaving stdout empty is what stops a
-    ``> baseline-timings.json`` redirect from producing a file that looks
-    captured.
-    """
-    capture = timings.get("capture")
-    if isinstance(capture, dict):
-        detail = (
-            f"only {capture.get('captured')} of {capture.get('declared')} declared "
-            f"metrics came back, below the minimum ratio "
-            f"{capture.get('min_ratio')}"
-        )
-    else:
-        detail = (
-            "the file carries no 'capture' block, so its completeness cannot be "
-            "established — recapture with the current capture_timings.py"
-        )
-
-    banner = "=" * 72
-    print(banner, file=sys.stderr)
-    print("  CAPTURE INCOMPLETE — refusing to print a baseline block", file=sys.stderr)
-    print(f"  {detail}.", file=sys.stderr)
-    print(
-        "  These timings may describe metrics that were never measured. Pasting\n"
-        "  them into baselines/baseline-timings.json would narrow the regression\n"
-        "  gate to whichever keys were captured, with nothing reporting that it\n"
-        "  had narrowed. Fix the capture and re-run.",
-        file=sys.stderr,
-    )
-    print(banner, file=sys.stderr)
-
-
-def print_paste_me(timings: dict) -> None:
-    """Print captured timings in the exact baseline-timings.json format.
-
-    The output between the two banner lines is the file contents to paste,
-    byte-for-byte — sorted keys, 2-space indent, trailing newline.
-    """
-    banner = "=" * 72
-    print(banner, file=sys.stderr)
-    print(
-        "  NO BASELINE FOUND — paste the JSON below into",
-        file=sys.stderr,
-    )
-    print(
-        "  docker/telemetry/workload/baselines/baseline-timings.json",
-        file=sys.stderr,
-    )
-    print(banner, file=sys.stderr)
-
-    print(json.dumps(timings, indent=2, sort_keys=True))
-
-    print(banner, file=sys.stderr)
-    print(
-        "  (End of paste-me JSON. Gate did NOT run — baseline is empty.)",
-        file=sys.stderr,
-    )
-    print(banner, file=sys.stderr)
-
-
-def resolve_thresholds(
-    key: str,
-    thresholds: dict,
-) -> tuple[float | None, float | None]:
-    """Return ``(pct_threshold, abs_threshold)`` for a metric key.
-
-    Per-metric overrides win over defaults. Returns ``(None, None)`` if no
-    threshold is defined for this category/quantile — such metrics are
-    captured but never gate the build.
-    """
-    parts = key.split(".")
-    if len(parts) < 3:
-        return (None, None)
-    category_key = parts[0]
-    quantile_key = parts[-1]
-
-    category_map = {
-        "span": "span",
-        "rpc": "rpc_method",
-        "job": "job_queue",
-    }
-    cat = category_map.get(category_key)
-    if cat is None:
-        return (None, None)
-
-    override_key = f"{category_key}.{'.'.join(parts[1:-1])}"
-    overrides = thresholds.get("overrides", {})
-    defaults = thresholds.get("defaults", {}).get(cat, {})
-
-    rule = overrides.get(override_key, {}).get(quantile_key)
-    if rule is None:
-        rule = defaults.get(quantile_key)
-    if rule is None:
-        return (None, None)
-
-    pct = rule.get("max_pct_increase")
-    abs_bound = rule.get("max_abs_increase_ms")
-    if abs_bound is None:
-        abs_bound = rule.get("max_abs_increase_us")
-    return (pct, abs_bound)
-
-
-def _skip_delta(
-    key: str,
-    baseline: float | None,
-    current: float | None,
-    unit: str,
-    thresholds: dict,
-    note: str,
-) -> MetricDelta:
-    """Build a MetricDelta for cases where comparison is not possible."""
-    pct_threshold, abs_threshold = resolve_thresholds(key, thresholds)
-    return MetricDelta(
-        key=key,
-        baseline=baseline,
-        current=current,
-        delta=None,
-        pct_change=None,
-        unit=unit,
-        threshold_pct=pct_threshold,
-        threshold_abs=abs_threshold,
-        regressed=False,
-        note=note,
-    )
-
-
-def _delta_note(regressed: bool, delta: float, pct_change: float | None) -> str:
-    """Classify one comparison outcome for the report and the table."""
-    if regressed:
-        note = "REGRESSION"
-    elif delta < 0:
-        note = "improved"
-    else:
-        note = "within bounds"
-    if pct_change is None:
-        note += " (absolute bound only; baseline not positive)"
-    return note
-
-
-# The regression rule, applied by compute_delta below.
-#
-# A regression normally requires BOTH bounds to be breached simultaneously.
-# That tolerates small-value noise: a 100% increase on a 0.5 ms metric (to
-# 1.0 ms) is not a regression under a 5 ms absolute bound.
-#
-# A non-positive baseline has no defined percentage change, so there the
-# absolute bound decides alone. Requiring both bounds in that case would make
-# the gate unreachable and let a 0 -> 500 ms jump pass as "within bounds".
-def compute_delta(
-    key: str,
-    baseline_entry: dict | None,
-    current_entry: dict | None,
-    thresholds: dict,
-) -> MetricDelta:
-    """Compute a MetricDelta for one metric key.
-
-    Follows the regression rule set out in the comment above, including the
-    non-positive-baseline exception.
-    """
-    baseline = baseline_entry.get("value") if baseline_entry else None
-    current = current_entry.get("value") if current_entry else None
-    unit = (baseline_entry or current_entry or {}).get("unit", "")
-
-    # A unit change makes the two numbers incomparable, so subtracting them is
-    # meaningless: us -> ms reads as a 99.9% improvement and the gate passes.
-    # Fail instead, and name both units so the baseline can be refreshed.
-    baseline_unit = (baseline_entry or {}).get("unit", "")
-    current_unit = (current_entry or {}).get("unit", "")
-    if baseline_unit and current_unit and baseline_unit != current_unit:
-        pct_threshold, abs_threshold = resolve_thresholds(key, thresholds)
-        return MetricDelta(
-            key=key,
-            baseline=baseline,
-            current=current,
-            delta=None,
-            pct_change=None,
-            unit=f"{baseline_unit}->{current_unit}",
-            threshold_pct=pct_threshold,
-            threshold_abs=abs_threshold,
-            regressed=True,
-            note=(
-                f"unit changed: baseline is {baseline_unit}, current run is "
-                f"{current_unit} -- refresh the baseline instead of comparing"
-            ),
-        )
-
-    if baseline is None and current is None:
-        return _skip_delta(
-            key, None, None, unit, thresholds, "no data (neither baseline nor current)"
-        )
-
-    if baseline is None:
-        return _skip_delta(
-            key, None, current, unit, thresholds, "new metric (not in baseline)"
-        )
-
-    if current is None:
-        return _skip_delta(
-            key, baseline, None, unit, thresholds, "not captured in current run"
-        )
-
-    pct_threshold, abs_threshold = resolve_thresholds(key, thresholds)
-    delta = current - baseline
-    pct_change = (delta / baseline * 100.0) if baseline > 0 else None
-
-    if pct_threshold is None or abs_threshold is None:
-        return MetricDelta(
-            key=key,
-            baseline=baseline,
-            current=current,
-            delta=delta,
-            pct_change=pct_change,
-            unit=unit,
-            threshold_pct=pct_threshold,
-            threshold_abs=abs_threshold,
-            regressed=False,
-            note="no threshold configured",
-        )
-
-    abs_breach = delta > abs_threshold
-    if pct_change is None:
-        # Baseline is not positive, so there is no percentage to compare.
-        # The absolute bound is the only usable signal here.
-        regressed = abs_breach
-    else:
-        regressed = pct_change > pct_threshold and abs_breach
-
-    return MetricDelta(
-        key=key,
-        baseline=baseline,
-        current=current,
-        delta=delta,
-        pct_change=pct_change,
-        unit=unit,
-        threshold_pct=pct_threshold,
-        threshold_abs=abs_threshold,
-        regressed=regressed,
-        note=_delta_note(regressed, delta, pct_change),
-    )
-
-
-def print_summary(deltas: list[MetricDelta]) -> None:
-    """Print a sorted, human-readable table of per-metric results."""
-    regressions = [d for d in deltas if d.regressed]
-    improvements = [
-        d
-        for d in deltas
-        if d.delta is not None and d.delta < 0 and d.baseline not in (None, 0)
-    ]
-    improvements.sort(key=lambda d: d.pct_change or 0)
-    regressions.sort(key=lambda d: -(d.pct_change or 0))
-
-    print("=" * 72)
-    print(f"  Regression check: {len(regressions)} regression(s) detected")
-    print("=" * 72)
-
-    if regressions:
-        print(
-            "\nRegressions (breached BOTH pct AND absolute bounds, or the "
-            "absolute bound alone where the baseline is not positive):"
-        )
-        _print_table(regressions)
-        # A regression can also be recorded with no delta at all -- a unit
-        # change makes the two numbers incomparable. That row prints as dashes,
-        # so name the reason here or the table looks like a bug.
-        for d in regressions:
-            if d.delta is None:
-                print(f"  {d.key}: {d.note}")
-
-    if improvements:
-        top = improvements[:5]
-        print("\nTop improvements:")
-        _print_table(top)
-
-    missing = [d for d in deltas if d.note == "not captured in current run"]
-    if missing:
-        print(f"\n{len(missing)} baseline metric(s) not captured in current run:")
-        for d in missing:
-            print(f"  {d.key}")
-
-
-def _print_table(rows: list[MetricDelta]) -> None:
-    """Print a fixed-width table for a list of deltas."""
-    header = f"  {'METRIC':<45} {'BASE':>10} {'CUR':>10} {'Δ':>10} {'%':>8} UNIT"
-    print(header)
-    print("  " + "-" * (len(header) - 2))
-    for d in rows:
-        base = f"{d.baseline:.2f}" if d.baseline is not None else "-"
-        cur = f"{d.current:.2f}" if d.current is not None else "-"
-        delta = f"{d.delta:+.2f}" if d.delta is not None else "-"
-        pct = f"{d.pct_change:+.1f}%" if d.pct_change is not None else "-"
-        print(f"  {d.key:<45} {base:>10} {cur:>10} {delta:>10} {pct:>8} {d.unit}")
-
-
-def write_report(
-    deltas: list[MetricDelta],
-    report_path: Path,
-    baseline: dict,
-    timings: dict,
-) -> None:
-    """Write regression-report.json — machine-readable artifact for CI."""
-    regressions = [d for d in deltas if d.regressed]
-    payload = {
-        "schema_version": 1,
-        "baseline_captured_at": baseline.get("captured_at"),
-        "baseline_git_sha": baseline.get("git_sha"),
-        "current_captured_at": timings.get("captured_at"),
-        "current_git_sha": timings.get("git_sha"),
-        "window": timings.get("window"),
-        "profile": timings.get("profile"),
-        "summary": {
-            # total is every key in the report, which is the UNION of the
-            # baseline and the current run -- not the baseline count. "compared"
-            # is the only number that says how much was actually gated: a delta
-            # exists only when both sides had a value.
-            "total": len(deltas),
-            "compared": sum(1 for d in deltas if d.delta is not None),
-            "regressions": len(regressions),
-            "improvements": sum(
-                1
-                for d in deltas
-                if d.delta is not None and d.delta < 0 and d.baseline not in (None, 0)
-            ),
-            "missing_in_current": sum(
-                1 for d in deltas if d.note == "not captured in current run"
-            ),
-        },
-        "metrics": [asdict(d) for d in deltas],
-    }
-    report_path.parent.mkdir(parents=True, exist_ok=True)
-    with open(report_path, "w") as f:
-        json.dump(payload, f, indent=2, sort_keys=True)
-        f.write("\n")
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument(
-        "--timings",
-        type=Path,
-        required=True,
-        help="Captured timings JSON (from capture_timings.py)",
-    )
-    parser.add_argument(
-        "--baseline",
-        type=Path,
-        required=True,
-        help="Committed baseline-timings.json",
-    )
-    parser.add_argument(
-        "--thresholds",
-        type=Path,
-        default=Path(__file__).parent / "regression-thresholds.json",
-        help="Threshold policy JSON",
-    )
-    parser.add_argument(
-        "--report",
-        type=Path,
-        default=None,
-        help="Where to write regression-report.json (optional)",
-    )
-    args = parser.parse_args()
-
-    logging.basicConfig(
-        level=logging.INFO,
-        format="%(levelname)s %(name)s: %(message)s",
-    )
-
-    try:
-        timings = load_json(args.timings)
-        baseline = load_json(args.baseline)
-        thresholds = load_json(args.thresholds)
-    except (OSError, json.JSONDecodeError) as exc:
-        logger.error("failed to load inputs: %s", exc)
-        return 2
-
-    if is_placeholder(baseline):
-        if not capture_is_complete(timings):
-            print_incomplete_capture(timings)
-            return 2
-        print_paste_me(timings)
-        return 0
-
-    baseline_metrics = baseline.get("metrics", {})
-    current_metrics = timings.get("metrics", {})
-
-    all_keys = sorted(set(baseline_metrics) | set(current_metrics))
-    deltas = [
-        compute_delta(
-            key,
-            baseline_metrics.get(key),
-            current_metrics.get(key),
-            thresholds,
-        )
-        for key in all_keys
-    ]
-
-    print_summary(deltas)
-
-    if args.report:
-        write_report(deltas, args.report, baseline, timings)
-        logger.info("wrote %s", args.report)
-
-    return 1 if any(d.regressed for d in deltas) else 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

docker/telemetry/workload/expected_metrics.json

@@ -1,438 +0,0 @@
-{
-  "description": "Expected metric inventory for xrpld telemetry validation. Metric names have no prefix. beast::insight metrics are lowercased by formatName. Keep every name in step with the code that emits it and with the panel queries under docker/telemetry/grafana/dashboards/. IMPORTANT: validate_telemetry.py has no notion of an optional metric — validate_metrics() iterates every group that has a \"metrics\" key and hard-fails any name with 0 Prometheus series after a 45 s poll. A metric is therefore listed only when the harness workload guarantees it will appear: observable gauges/counters whose callbacks Observe unconditionally (series exist at value 0), or push counters/histograms on a path every run exercises. A push counter that starts at 0 has a series on every run, so for it the existence check proves only that it exports, not that the path ran; such a counter is also listed in its group's \"min_values\". Workload-gated and defect-gated names are recorded in the \"not_asserted\" group, which intentionally has no \"metrics\" key so the validator skips it. The \"metrics\" checks test series existence only. A group may also declare \"min_values\": each entry names a selector from the same group's \"metrics\" list and a \"min\", and validate_metric_minimums() fails it unless sum(<selector>) reaches the minimum, per value of the \"by\" label when one is given. The only other value checks are the four bounds checks hardcoded in PARITY_VALUE_SANITY. Several not_asserted counters also start at 0, so their series exist on every run; their notes say when the value moves, and promoting one takes a min_values entry, not only a \"metrics\" entry. A group may additionally declare \"required_labels\": each label there becomes one check that at least one of the group's series carries it with a non-empty value, matched as <label>!=\"\" because Prometheus cannot tell an absent label from an empty one. The same guarantee rule applies — list a label only where the workload guarantees it. Two conventions apply to the consumer evidence cited in the group descriptions below. First, only tracked files count: dashboards under docker/telemetry/grafana/dashboards/ and rules under docker/telemetry/grafana/provisioning/alerting/. The docker/telemetry/grafanacloud/ tree is gitignored (sourceCode/.gitignore:27) and has zero tracked files, so a cloud copy cannot be cited as evidence on a clean checkout and is not counted here. Second, every rule in provisioning/alerting/rules.yaml is provisioned with isPaused: true (14 of 14), so a metric feeding a rule is a wiring dependency, not a live pager: losing it leaves the rule wired to a signal that no longer resolves, and it starts silent rather than becoming silent. Finally, the top-level \"accounted_patterns\" list is not a group and asserts nothing. It declares anchored regexes for the bulk families that cannot usefully be enumerated as literal names — the per-job-type gauges and histograms, the overlay per-category traffic cross product — plus the Prometheus scrape plumbing that is not xrpld telemetry at all. Its only consumer is the reverse coverage check in validate_telemetry.py, which walks the metric families Prometheus actually holds and names every one that neither a group's \"metrics\" entry, nor a \"metrics_excluded\" key, nor one of these patterns accounts for. That check warns and never fails: downstream branches legitimately add telemetry this contract has not seen yet, and a hard failure would redden all of them. Its value is visibility, not enforcement.",
-  "spanmetrics": {
-    "description": "SpanMetrics-derived RED metrics from the OTel Collector spanmetrics connector.",
-    "metrics": [
-      "span_calls_total",
-      "span_duration_milliseconds_bucket",
-      "span_duration_milliseconds_count",
-      "span_duration_milliseconds_sum"
-    ],
-    "required_labels": [
-      "span_name",
-      "status_code",
-      "service_name",
-      "span_kind"
-    ],
-    "dimension_labels": [
-      "command",
-      "rpc_status",
-      "consensus_mode",
-      "local",
-      "proposal_trusted",
-      "validation_trusted",
-      "tx_type",
-      "ter_result",
-      "stage",
-      "txq_status",
-      "close_time_correct",
-      "consensus_state"
-    ],
-    "_dimension_labels_note": "Bare label names as configured in otel-collector-config.yaml spanmetrics dimensions. Informational only (not asserted by the validator)."
-  },
-  "statsd_gauges": {
-    "description": "beast::insight gauges exported via OTLP/HTTP to the collector (server=otel). What guarantees these is the export mechanism, not the workload. On the OTel path a beast gauge is an Int64ObservableGauge (OTelCollector.cpp:703); every OTelGaugeImpl registers itself with the collector in its own constructor (:690); onCollectionReady() arms every registered gauge with no per-metric condition (:941-979); and the armed callback Observes currentValue() on each export cycle regardless of the value and regardless of whether set() was ever called (:719-727). Series therefore exist from the first export, at 0 if nothing has happened. The 'gauges only mark dirty on value changes' caveat in _poll_series_count is a property of the StatsD backend and does not apply here. The consequence is that a beast gauge whose object is constructed on an unconditional startup path is as safe to assert as an observable gauge from MetricsRegistry — provided the object is constructed before Application.cpp:1570, which is where onCollectionReady() runs. That precondition is load-bearing and is the one way a gauge on this path can still go missing. Registration and arming are separate steps: addGauge() (:927-931) only appends to gauges_ under the lock, and onCollectionReady() snapshots that vector and calls arm() on each entry, exactly once per process (Application.cpp:1570, on the setup path, with no second call site). A gauge whose owner is built after that line is registered, never armed, and never exported — no callback, no series, and no error. Every gauge asserted in this group clears the precondition: jobQueue_, nodeFamily_, ledgerMaster_ and networkOPs_ are all in ApplicationImp's member-init list (Application.cpp:377, :430, :439, :465), and overlay_ — which owns peerFinder_ in its own member-init list (OverlayImpl.cpp:185) and Stats::peerDisconnects (OverlayImpl.h:603) — is built at Application.cpp:1551, nineteen lines before the arming call. The comment at Application.cpp:1258-1272 states that ordering deliberately: overlay_ is the last service the callbacks read, which is why arming waits for it. This is also why the state_accounting family below is asserted whole rather than one member deep. The ten state_accounting_* names are one family with one guarantee. NetworkOPsImp::Stats creates all ten in the NetworkOPsImp constructor (NetworkOPs.cpp:1069-1080; makeGauge(\"State_Accounting\", \"Full_duration\") joins on '.' at Collector.h:151-155 and formatName lowercases it, giving state_accounting_full_duration), and NetworkOPsImp::collectMetrics() set()s all ten unconditionally in a single block (:5204-5231). No state has to be entered for its pair to exist: a node that never disconnects still publishes state_accounting_disconnected_duration and _transitions at 0. All ten appear on the node-health dashboard, and state_accounting_full_transitions is also the input to the NodeStateFlapping alert rule (provisioning/alerting/rules.yaml:645-649, uid xrpld-node-state-flapping — provisioned isPaused: true, so this is a wiring dependency and not a live pager). Asserting only full_duration left the other nine unchecked, the alert's own input among them, even though a regression could not plausibly drop one sibling without dropping full_duration too. node_family_full_below_cache_hit_rate and _size are the TaggedCache Stats pair (TaggedCache.h:282-283), created in the cache constructor (TaggedCache.ipp:63-66) and published by collectMetrics(), which reports a hit_rate of 0 rather than skipping the gauge when hits+misses is 0 (TaggedCache.ipp:751-765). The instance is NodeFamily's full-below cache, constructed with the real collector in NodeFamily's constructor (NodeFamily.cpp:28-35), which every node builds. Both are on node-health. overlay_peer_disconnects is the first member of OverlayImpl::Stats (OverlayImpl.h:603) and OverlayImpl::collectMetrics() assigns it getPeerDisconnect() unconditionally (:646), in the same hook that publishes the overlay_traffic group's gauges. Its only query consumer is network-traffic.json:148. It also appears once in log-derived-insights.json:1908, but that occurrence is inside a panel description which explicitly disclaims it ('overlay_peer_disconnects exists as a metric but carries no reason breakdown, which is what this panel adds') — that panel is Loki-derived and queries nothing from Prometheus, so it is not a consumer. The third occurrence, validate_dashboards.py:17, is a lint rule listing the name as a cumulative gauge that must be rate()-wrapped; also not a query consumer. It is listed here rather than in overlay_traffic because that group's contract is an explicit subset of the per-category traffic family, and a disconnect count is not one of those categories.",
-    "metrics": [
-      "ledgermaster_validated_ledger_age",
-      "ledgermaster_published_ledger_age",
-      "state_accounting_full_duration",
-      "peer_finder_active_inbound_peers",
-      "peer_finder_active_outbound_peers",
-      "jobq_job_count",
-      "state_accounting_connected_duration",
-      "state_accounting_connected_transitions",
-      "state_accounting_disconnected_duration",
-      "state_accounting_disconnected_transitions",
-      "state_accounting_full_transitions",
-      "state_accounting_syncing_duration",
-      "state_accounting_syncing_transitions",
-      "state_accounting_tracking_duration",
-      "state_accounting_tracking_transitions",
-      "node_family_full_below_cache_hit_rate",
-      "node_family_full_below_cache_size",
-      "overlay_peer_disconnects"
-    ]
-  },
-  "statsd_counters": {
-    "description": "beast::insight counters exported via OTLP/HTTP. The OTel Prometheus exporter appends _total to monotonic counters. Both counters start at 0 when their owner is built (ServerHandler and InboundLedgersImp, both in ApplicationImp's constructor), so each series exists before any event, and the existence check proves only that the counter exports. min_values checks the events. rpc_requests_total counts HTTP JSON-RPC requests only (ServerHandler::processRequest). The load generators use WebSocket, but run-full-validation.sh polls every node's HTTP port with server_info until all nodes are proposing, so each node counts at least one request, and the check is per node. ledger_fetches_total counts a new InboundLedger in InboundLedgersImp::acquire, the same event that opens the optional ledger.acquire span. A fresh cluster started with --start fetches while its nodes converge: NetworkOPsImp::checkLastClosedLedger, among other paths, acquires the closed ledger a node's peers prefer when the node lacks it. Which nodes fetch depends on start timing, so the check is cluster-wide: it asks only whether any node fetched a ledger, which is what its existence check asked before the counter started at 0.",
-    "metrics": ["rpc_requests_total", "ledger_fetches_total"],
-    "min_values": [
-      {
-        "metric": "rpc_requests_total",
-        "min": 1,
-        "by": "service_instance_id"
-      },
-      {
-        "metric": "ledger_fetches_total",
-        "min": 1
-      }
-    ]
-  },
-  "io_latency": {
-    "description": "io_context scheduling-latency histogram — a beast::insight Event exported over OTLP (made by makeEvent('ios_latency') in the ApplicationImp constructor, with the default millisecond unit). It is the one beast Event the node itself guarantees, which is why it is asserted while rpc_time_milliseconds and the jobq_* pairs sit in not_asserted: the sampler starts on the unconditional startup path (in ApplicationImp::start(), outside the 'if (withTimers)' guard), and its handler always emits the first sample whatever its value (IOLatencySampler::operator(), 'firstSample_.exchange(false) || lastSample >= 10ms', with a comment stating the point is to register the metric downstream). The OTel histogram is cumulative, so that one notify creates series that persist for the rest of the run. Named with the _bucket/_count/_sum suffixes the Prometheus exporter emits for a histogram — no bare ios_latency_milliseconds series exists, the same convention rpc_method_us and span_duration_milliseconds follow, and every consumer queries ios_latency_milliseconds_bucket: 2 distinct panels (ledger-data-sync 'I/O Scheduler Latency p95' and node-health 'I/O Latency') plus the IOEventLoopLatencyHigh alert rule (grafana/provisioning/alerting/rules.yaml). Only existence is asserted: past the first sample the handler reports only latencies >= 10 ms, so neither the sample count nor the value is predictable.",
-    "metrics": [
-      "ios_latency_milliseconds_bucket",
-      "ios_latency_milliseconds_count",
-      "ios_latency_milliseconds_sum"
-    ]
-  },
-  "overlay_traffic": {
-    "description": "Overlay traffic metrics (subset — full list has 45+ categories).",
-    "metrics": [
-      "total_bytes_in",
-      "total_bytes_out",
-      "total_messages_in",
-      "total_messages_out"
-    ]
-  },
-  "nodestore_io": {
-    "description": "NodeStore I/O observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label distinguishing sub-metrics. Every series also carries a constant 'backend' label naming the NodeStore backend ([node_db] type, as its factory spells it, e.g. NuDB); required_labels checks that it reaches Prometheus.",
-    "required_labels": ["backend"],
-    "metrics": ["nodestore_state"]
-  },
-  "cache_hit_rates": {
-    "description": "Cache hit rate observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
-    "metrics": ["cache_metrics"]
-  },
-  "transaction_queue": {
-    "description": "Transaction queue observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
-    "metrics": ["txq_metrics"]
-  },
-  "rpc_method_detail": {
-    "description": "Per-RPC-method counters and duration histogram, created in MetricsRegistry::initSyncInstruments(). rpc_method_errored_total is deliberately absent — see not_asserted below. rpc_method_us is a Histogram, so the Prometheus exporter emits only the _bucket/_count/_sum triple and there is no bare rpc_method_us series to match — same convention as span_duration_milliseconds in the spanmetrics group above.",
-    "metrics": [
-      "rpc_method_started_total",
-      "rpc_method_finished_total",
-      "rpc_method_us_bucket",
-      "rpc_method_us_count",
-      "rpc_method_us_sum"
-    ]
-  },
-  "job_queue": {
-    "description": "Job-queue counters and latency histograms, created in MetricsRegistry::initSyncInstruments(). Every xrpld job passes through these, so they populate under any workload. Both histograms are recorded in the same function bodies as job_started_total / job_finished_total, under the same guard and with the same labels, so their presence is equally guaranteed. They are named with the _bucket/_count/_sum suffixes the Prometheus exporter emits: regression-metrics.json and the job-queue dashboard both query job_queued_us_bucket / job_running_us_bucket, and no bare series exists. This group also carries the xrpl_node_id required_labels assertion. The xrpl.node.id resource attribute keeps Grafana Cloud trace ingest from folding distinct nodes into one. Ingest groups ResourceSpans but ignores service.instance.id. The assertion catches a regression that drops the attribute. It is asserted here rather than on a statsd_* group because these metrics are MetricsRegistry-backed and guaranteed by any workload. The beast::insight meter provider is built before the node key is known, so those metrics legitimately lack the label. Telemetry's makeMetricsResource() leaves it off rather than writing it blank.",
-    "required_labels": ["xrpl_node_id"],
-    "metrics": [
-      "job_queued_total",
-      "job_started_total",
-      "job_finished_total",
-      "job_queued_us_bucket",
-      "job_queued_us_count",
-      "job_queued_us_sum",
-      "job_running_us_bucket",
-      "job_running_us_count",
-      "job_running_us_sum"
-    ]
-  },
-  "job_queue_per_type_gauges": {
-    "description": "Per-job-type queue-depth gauges from the beast::insight \"jobq\" group, for the six job types the ledger-sync diagnostics single out. JobTypeData's constructor creates waiting/running/deferred for every non-special job type (JobTypeData.h:96-103), the JobQueue constructor eagerly constructs a JobTypeData for every type in JobTypes (JobQueue.cpp:45-53), and JobQueue::collect() assigns all three from one snapshot for every type on every hook cycle, with no per-type condition (JobQueue.cpp:104-106). The collector is the \"jobq\" group, so GroupImp::makeName() prefixes \"jobq.\" and formatName lowercases the job type: JtTxnData's \"fetchTxnData\" + \"_running\" exports as jobq_fetchtxndata_running (JobTypeData.h:29-37). Existence rests on the same ObservableGauge mechanism documented on statsd_gauges, so these series exist at 0 on an idle node. All six were present in the CI validation run's own emitted-metric list, which is the strongest evidence behind any entry in this file. All six job types are non-special: makeFetchPack (limit 1), manifest (maxLimit), ledgerRequest (3), ledgerData (3), updatePaths (1) and fetchTxnData (5) all declare a non-zero limit in JobTypes.h, so none is skipped by the special() guard in JobTypeData's constructor. The 11 special types (limit 0) get no gauges at all, which is why no jobq_pathfind_* or jobq_peercommand_* name can be asserted. Why these six and not all 105. The guarantee is identical for every one of the 35 non-special types x 3 states, so the discriminator here is consumer coverage rather than emission: these six are the only per-type names a dashboard or an alert queries literally, so losing one blanks a specific panel or leaves a rule wired to a signal that no longer resolves. jobq_manifest_waiting is the input to the ManifestJobQueueConvoy alert rule (provisioning/alerting/rules.yaml:796-800, uid xrpld-manifest-job-convoy — provisioned isPaused: true like every rule in that file, so the consequence is a broken wire rather than a silenced pager); the other five are per-type saturation panels on node-health.json:5267-5291. The rest of the family is reached only through two regex queries, both on ledger-data-sync: __name__=~\"jobq_.*_deferred\" at :1701 and __name__=~\"jobq_.*_waiting\" at :1708. Those are the only two jobq regexes anywhere in the tree — there is no jobq_.*_running regex — so the 30 unasserted _running gauges have zero consumers of any kind, and the unasserted _waiting and _deferred gauges are reached only by a topk that keeps working as long as the family exists and depends on no single job type. Consumer coverage is the whole argument and it stands alone. An earlier revision also cited poll bandwidth; that reason was wrong and has been removed, because METRIC_POLL_CONCURRENCY is 8 and the semaphore is released across each sleep, so 105 targets cost on the order of a second against the shared 45 s deadline. A bogus cost argument would be a bad reason to refuse a legitimate expansion later: if a panel or rule starts naming a seventh job type, add it.",
-    "metrics": [
-      "jobq_fetchtxndata_running",
-      "jobq_ledgerdata_running",
-      "jobq_ledgerrequest_running",
-      "jobq_makefetchpack_running",
-      "jobq_manifest_waiting",
-      "jobq_updatepaths_running"
-    ]
-  },
-  "rpc_in_flight": {
-    "description": "In-flight RPC gauge via the XRPL_METRIC_UPDOWN_ADD call-site macro (PerfLogImp.cpp, +1 rpcStart / -1 rpcEnd). UpDownCounter: no _total suffix.",
-    "metrics": ["rpc_in_flight_requests"]
-  },
-  "object_counts": {
-    "description": "Counted object instances observable gauge (MetricsRegistry via OTLP).",
-    "metrics": ["object_count"]
-  },
-  "load_factors": {
-    "description": "Fee escalation and load factor observable gauge (MetricsRegistry via OTLP).",
-    "metrics": ["load_factor_metrics"]
-  },
-  "parity_validation_agreement": {
-    "description": "External dashboard parity: validation agreement percentages (AppMetricGauges). Published only by a node with a validator key; every node run-full-validation.sh starts is a validator with a [validation_seed], so both series exist from the first export.",
-    "metrics": [
-      "validation_agreement{metric=\"agreement_pct_1h\"}",
-      "validation_agreement{metric=\"agreement_pct_24h\"}"
-    ]
-  },
-  "parity_validator_health": {
-    "description": "External dashboard parity: validator health indicators (MetricsRegistry).",
-    "metrics": [
-      "validator_health{metric=\"amendment_blocked\"}",
-      "validator_health{metric=\"unl_expiry_days\"}"
-    ]
-  },
-  "parity_peer_quality": {
-    "description": "External dashboard parity: peer quality metrics (MetricsRegistry).",
-    "metrics": [
-      "peer_quality{metric=\"peer_latency_p90_ms\"}",
-      "peer_quality{metric=\"peers_insane_count\"}"
-    ]
-  },
-  "peer_validation_load": {
-    "description": "Per-peer validation load (AppMetricGauges). An observable gauge whose labels come only from fixed lists: five metric values for each of two trust values, so every node has exactly these 10 series. All 10 are observed from the first collection, at 0 until a peer has been read twice, so no workload is needed.",
-    "metrics": [
-      "peer_validation_load{metric=\"top1_rate\",trust=\"trusted\"}",
-      "peer_validation_load{metric=\"top2_rate\",trust=\"trusted\"}",
-      "peer_validation_load{metric=\"top3_rate\",trust=\"trusted\"}",
-      "peer_validation_load{metric=\"top_share\",trust=\"trusted\"}",
-      "peer_validation_load{metric=\"peers_over_limit\",trust=\"trusted\"}",
-      "peer_validation_load{metric=\"top1_rate\",trust=\"untrusted\"}",
-      "peer_validation_load{metric=\"top2_rate\",trust=\"untrusted\"}",
-      "peer_validation_load{metric=\"top3_rate\",trust=\"untrusted\"}",
-      "peer_validation_load{metric=\"top_share\",trust=\"untrusted\"}",
-      "peer_validation_load{metric=\"peers_over_limit\",trust=\"untrusted\"}"
-    ]
-  },
-  "parity_ledger_economy": {
-    "description": "External dashboard parity: ledger economy metrics (AppMetricGauges.cpp). transaction_rate is observed on every export, in both branches of the ledger-age test. base_fee_drops is observed only inside the 'if (ledger)' guard on getValidatedLedger(), and that returns validLedger_ (LedgerMaster::getValidatedLedger), which stays null until a ledger validates — the same precondition complete_ledgers has. Both are asserted because run-full-validation.sh waits for a validated ledger before running the workload. base_fee_drops absent while transaction_rate is present is the signature of a cluster that never validated, not of a missing metric.",
-    "metrics": [
-      "ledger_economy{metric=\"base_fee_drops\"}",
-      "ledger_economy{metric=\"transaction_rate\"}"
-    ]
-  },
-  "parity_state_tracking": {
-    "description": "External dashboard parity: server state tracking (MetricsRegistry).",
-    "metrics": ["state_tracking{metric=\"state_value\"}"]
-  },
-  "parity_counters": {
-    "description": "External dashboard parity: monotonic counters (MetricsRegistry). All four start at 0 at startup, so each series exists before any event, and the existence check proves only that the counter exports. min_values checks that every node that exports the counter recorded the event. Every node closes ledgers (RCLConsensus::Adaptor::doAccept). Every node sends validations: run-full-validation.sh gives each one a [validation_seed] and waits until all of them are proposing. Every node checks the validations its peers send, counted at the top of NetworkOPsImp::recvValidation. And every node changes mode on its way from disconnected to proposing (NetworkOPsImp::setMode).",
-    "metrics": [
-      "ledgers_closed_total",
-      "validations_sent_total",
-      "validations_checked_total",
-      "state_changes_total"
-    ],
-    "min_values": [
-      {
-        "metric": "ledgers_closed_total",
-        "min": 1,
-        "by": "service_instance_id"
-      },
-      {
-        "metric": "validations_sent_total",
-        "min": 1,
-        "by": "service_instance_id"
-      },
-      {
-        "metric": "validations_checked_total",
-        "min": 1,
-        "by": "service_instance_id"
-      },
-      {
-        "metric": "state_changes_total",
-        "min": 1,
-        "by": "service_instance_id"
-      }
-    ]
-  },
-  "parity_storage": {
-    "description": "External dashboard parity: storage detail metrics (MetricsRegistry). Every series also carries a constant 'backend' label naming the NodeStore backend ([node_db] type, as its factory spells it, e.g. NuDB); required_labels checks that it reaches Prometheus.",
-    "required_labels": ["backend"],
-    "metrics": ["storage_detail{metric=\"stored_object_bytes\"}"]
-  },
-  "sync_diagnostics": {
-    "description": "Fresh-node sync diagnostics (native metrics). Bootstrap (Domain 0) and acquire-pipeline signals rendered by the ledger-sync-health dashboard. Names are appended one per signal. Histograms are listed by all three of their Prometheus series -- _bucket, _count and _sum -- because the bare instrument name is not a series and an exporter regression can drop one without dropping the others; this matches the rpc_method_us / job_queued_us convention in the groups above. Observable gauges carry an inline metric= selector so the specific sub-series is asserted, matching the parity_* groups. Signals this workload cannot produce are NOT listed here: they belong in the not_asserted group, whose metrics_excluded map is the single machine-readable record of every emitted-but-unasserted name. A counter that preRegisterMacroCounters() starts at 0 has a series before its first event, so its entry here proves only that it exports; each such counter the workload does trigger is also listed in min_values (see _min_values_note).",
-    "metrics": [
-      "dns_resolve_total",
-      "dns_resolve_latency_ms_bucket",
-      "dns_resolve_latency_ms_count",
-      "dns_resolve_latency_ms_sum",
-      "overlay_connect_total",
-      "overlay_dial_latency_ms_bucket",
-      "overlay_dial_latency_ms_count",
-      "overlay_dial_latency_ms_sum",
-      "unl_quorum{metric=\"trusted_keys\"}",
-      "unl_quorum{metric=\"quorum\"}",
-      "clock_close_offset_seconds{metric=\"offset\"}",
-      "sync_state{metric=\"initial_full_duration_us\"}",
-      "sync_state{metric=\"network_ledger_gate\"}",
-      "sync_state{metric=\"server_stall_seconds\"}",
-      "sync_state{metric=\"ledgers_behind\"}",
-      "server_stall_events_total",
-      "state_changes_total{from!=\"\",to!=\"\"}",
-      "sync_acquire{metric=\"missing_state_nodes_max\"}",
-      "sync_acquire{metric=\"missing_tx_nodes_max\"}",
-      "sync_acquire{metric=\"received_data_depth\"}",
-      "sync_acquire{metric=\"in_flight\"}",
-      "shamap_cache_hit_rate{metric=\"treenode\"}",
-      "jobq_saturation{metric=\"running_tasks\"}",
-      "jobq_saturation{metric=\"worker_threads\"}",
-      "jobq_saturation{metric=\"total_waiting\"}",
-      "peer_ledger_supply{metric=\"peers_reporting\"}",
-      "peer_ledger_supply{metric=\"peers_ahead\"}",
-      "peer_ledger_supply{metric=\"peers_serving_validated\"}",
-      "peer_ledger_supply{metric=\"peers_serving_next\"}",
-      "peer_ledger_supply{metric=\"supply_min_seq\"}",
-      "peer_ledger_supply{metric=\"supply_max_seq\"}",
-      "peerfinder_slot_census{metric=\"out_active\"}",
-      "peerfinder_slot_census{metric=\"out_max\"}",
-      "peerfinder_slot_census{metric=\"in_active\"}",
-      "peerfinder_slot_census{metric=\"in_max\"}",
-      "peerfinder_slot_census{metric=\"connecting\"}",
-      "peerfinder_slot_census{metric=\"fixed_configured\"}",
-      "peerfinder_slot_census{metric=\"fixed_active\"}",
-      "peerfinder_slot_census{metric=\"bootcache\"}",
-      "peerfinder_slot_census{metric=\"livecache\"}",
-      "amendment_block{metric=\"warned\"}",
-      "amendment_block{metric=\"seconds_to_block\"}",
-      "peer_accept_total",
-      "nodestore_state{metric=\"node_writes\"}",
-      "nodestore_state{metric=\"node_reads_total\"}",
-      "nodestore_state{metric=\"read_mean_us\"}",
-      "ledger_quorum_publish{metric=\"trusted_validation_tally\"}",
-      "ledger_quorum_publish{metric=\"quorum_target\"}",
-      "ledger_quorum_publish{metric=\"time_to_first_validated_us\"}",
-      "ledger_quorum_publish{metric=\"publish_lag\"}",
-      "ledger_quorum_shortfall_total{stage=\"pre_accept\"}",
-      "consensus_round_duration_ms_bucket",
-      "consensus_round_duration_ms_count",
-      "consensus_round_duration_ms_sum",
-      "nodestore_state{metric=\"node_writes_duration_us\"}",
-      "nodestore_state{metric=\"node_reads_duration_us\"}",
-      "unl_quorum{metric=\"quorum_disabled\"}",
-      "sweep_malloc_trim_us_bucket",
-      "sweep_malloc_trim_us_count",
-      "sweep_malloc_trim_us_sum",
-      "rotation_state{metric=\"in_flight\"}",
-      "rotation_state{metric=\"copy_forward\"}",
-      "cache_metrics{metric=\"treenode_lock_hold_peak_us\"}",
-      "cache_metrics{metric=\"fullbelow_lock_hold_peak_us\"}"
-    ],
-    "min_values": [
-      {
-        "metric": "dns_resolve_total",
-        "min": 1,
-        "by": "service_instance_id"
-      },
-      {
-        "metric": "overlay_connect_total",
-        "min": 1
-      },
-      {
-        "metric": "state_changes_total{from!=\"\",to!=\"\"}",
-        "min": 1,
-        "by": "service_instance_id"
-      },
-      {
-        "metric": "peer_accept_total",
-        "min": 1
-      },
-      {
-        "metric": "ledger_quorum_shortfall_total{stage=\"pre_accept\"}",
-        "min": 1
-      }
-    ],
-    "_min_values_note": "preRegisterMacroCounters() starts dns_resolve_total, overlay_connect_total, state_changes_total, peer_accept_total and ledger_quorum_shortfall_total at 0 on every label set, so their entries in metrics pass with no event. min_values checks the events. dns_resolve_total and state_changes_total are checked per node: every node resolves its [ips] names in OverlayImpl::start, and every node changes mode on its way from disconnected to proposing. overlay_connect_total, peer_accept_total and the pre_accept shortfall are checked cluster-wide, because which node dials, which node accepts and which node's first quorum check falls short all depend on timing the harness does not control.",
-    "_acquire_note": "The four sync_acquire sub-series and shamap_cache_hit_rate are unconditional: both are observable gauges whose callbacks observe every series on each collection tick, so each is present even when the value is 0 (an idle node reports in_flight=0 and missing_state_nodes_max=0, and a cold cache reports a 0.0 hit rate). Absence, not a zero, is the regression. The three InboundLedger acquire counters (sync_acquire_source_total, sync_addnode_total, sync_acquire_no_progress_total) are deliberately NOT asserted here: all three are emitted only from InboundLedger, which runs only when a node must fetch a ledger it lacks. expected_spans.json already marks the ledger.acquire span optional for exactly this reason (\"A healthy local cluster rarely back-fills history\"), and the metric validator has no per-metric optional flag. preRegisterMacroCounters() starts all three at 0, so a metrics entry would pass without an acquire and prove nothing, and a min_values entry would need an acquire on every run, which the harness does not force. They are covered by exact-value unit tests in src/tests/xrpld/telemetry/MetricMacros.cpp and by the ledger-sync-health panels; add them here only alongside a harness step that forces a real acquire (e.g. starting a node against an existing ledger history).",
-    "_jobq_note": "The three jobq_saturation series are unconditional: it is an observable gauge whose callback observes all three fields on every collection tick, so each series exists even when the value is 0, and absence rather than a zero is the regression. worker_threads is asserted because it is the denominator of the dashboard saturation ratio, and it is always at least 1 (the JobQueue ctor gives standalone mode exactly one worker), so a zero or missing reading there means the accessor regressed rather than the node being idle. The per-job-type waiting/running/deferred counts are published separately by JobQueue::collect() as the beast::insight gauges jobq_<type>_waiting / _running / _deferred, which the collector translates; they are covered by the StatsD-derived groups, not here.",
-    "_conditional_note": "handshake_negotiation_fail_total and unl_fetch_total used to be asserted here, with a note saying to move them out if a run ever reported 0 series. Both have now moved to not_asserted.metrics_excluded on CODE REACHABILITY rather than on any run: unl_fetch_total is emitted only from a site indexed out of ValidatorSite's sites_, which this harness leaves empty because it writes a static [validators] file and configures no list site at all. That argument does not depend on observing a run, which matters, because for a stretch of this branch's history no run could have observed anything here -- see _gate_history_note. Anything added to this group must be reachable under run-full-validation.sh's own cluster and workload, not merely emitted somewhere in the code.",
-    "_gate_history_note": "Why this group's contents were never validated by a run, recorded because it explains why several notes here rest on code reading rather than on observed output, and because the same failure mode can recur. assert_sync_diagnostics_metrics arrived on 2026-07-24 (96914b9f40) calling _check_prometheus_metric with five positional arguments, which was CORRECT then: the function took (session, prometheus_url, metric_name, category, report), returned None and recorded the result itself. On 2026-08-14 phase-10 gave that function a shared deadline and a semaphore (d059f21bf3), updating its own caller; 96914b9f40 is not an ancestor of that commit, so it never saw this one. The merge that first contained both, 7c70e142e9 on 2026-08-21, was CLEAN -- the two edits sit in different regions of the file -- and produced a caller that no longer matches the signature it calls. From then until the fix, every invocation raised TypeError before the first query, so the whole group asserted nothing and took the validation phases ordered after it down with it. Nothing could have caught this: it is a semantic conflict, invisible to git because neither side's text overlaps, and Python has no compile step to reject the arity. The general lesson for this harness is that a merge reported as clean is not evidence that a cross-branch caller still matches its callee -- after any merge that touches this file, re-check the call sites, not just the diff.",
-    "_sync_state_note": "The four sync_state sub-series are unconditional: the gauge observes all four on every collection tick, so each is present as a series even when its value is 0 (a node that never reached FULL reports initial_full_duration_us=0, and a healthy node reports server_stall_seconds=0). The check asserts series presence, not a non-zero value, which is exactly right here — a zero is a meaningful reading for these signals, and absence is the regression. server_stall_events_total is likewise always present because the observable counter reports the tally (0 or more) every tick. state_changes_total is listed here with a from!=\"\",to!=\"\" selector rather than bare (parity_counters already lists the bare name). preRegisterMacroCounters() starts every (from, to) pair of distinct modes at 0, so the selector matches from startup, and its metrics entry proves only that the labelled counter exports. Its min_values entry proves the rest: every node must count at least one labelled transition, which any node makes on its way from disconnected to proposing, and a recording site that stopped writing the labels would leave the labelled sums at 0.",
-    "_a7_note": "The peer-supply, slot-census and amendment-countdown signals are three observable gauges and four counters. The 17 gauge sub-series (peer_ledger_supply, peerfinder_slot_census, amendment_block) are unconditional and asserted individually: each callback in AppMetricGauges.cpp calls observe() for every field on every collection tick with no early return between them, so the series exists whatever the value. That includes the two sentinel readings — a node whose peers have advertised nothing reports peer_ledger_supply{metric=\"supply_min_seq\"} = 0 meaning unknown, and a node with no pending amendment reports amendment_block{metric=\"seconds_to_block\"} = -1 meaning healthy. Absence, not the sentinel, is the regression. Of the four counters only peer_accept_total is asserted: run-full-validation.sh gives every node a [port_peer] on 0.0.0.0 and lists the other four nodes in [ips], so the nodes dial each other, and every connection has an accepting side where OverlayImpl::onHandoff reports an outcome. It is asserted bare rather than with an outcome= selector, and cluster-wide in min_values rather than per node, because which node accepts and which outcome it records depend on dial ordering, which the harness does not control. The other three counters are deliberately NOT asserted. peer_disconnect_total is emitted only from PeerImp::close, and a healthy 5-node localhost cluster holds its peers for the whole run: the timer-driven reasons need maxUnknownTime (600 s) or maxDivergedTime (300 s) to elapse (Config.h) while the full-validation profile totals well under that, and the shutdown reasons only fire during teardown, which happens in run-full-validation.sh after Step 5 has already scraped. serve_refused_total needs a peer to ask this node for a ledger, tx set or object it cannot serve — on a cluster where every node has the same complete history from genesis, getLedger()/getTxSet() succeed and the send queues never approach Tuning::kDropSendQueue. ledger_jump_total needs NetworkOPsImp::switchLastClosedLedger, reached only when consensus reports an LCL this node did not build on; a healthy 5-node cluster agrees every round, so it never jumps. preRegisterMacroCounters() starts all three at 0, so a metrics entry would pass with no event and prove nothing, and a min_values entry would need an event the harness does not guarantee — the same reasoning _acquire_note applies to the InboundLedger acquire counters. All four counters are covered by exact-value unit tests in src/tests/xrpld/telemetry/MetricMacros.cpp and rendered by the ledger-sync-health panels Peer Disconnects (Count By Reason & Direction), Ledger/Object Serve Refusals and Byzantine Ledger Jumps, and by the peer-quality panels Peer Disconnect Rate and Peer Disconnects By Reason & Direction. To make them assertable the harness would need a fault-injection step: kill one node mid-run and re-scrape before teardown (peer_disconnect_total, reason=read_error), request a ledger sequence outside the cluster's history or drive a node past its send-queue limit (serve_refused_total), and start a node on a divergent chain tip or partition the cluster and heal it (ledger_jump_total).",
-    "_a6_note": "These node-store readings are sub-series of nodestore_state; there is no separate latency gauge. The five entries above are node_writes, node_reads_total, node_writes_duration_us, node_reads_duration_us and read_mean_us. All four totals are unconditional: AppMetricGauges::observeNodeStoreTotals observes them on every collection tick with no early return before them. So a series exists whatever the value, and a node that has written nothing reports node_writes=0 rather than dropping the series. read_mean_us is safe because any node that has opened a ledger has already fetched objects, so the fetch count is non-zero. write_mean_us is NOT asserted: scaledMean() omits a mean whose count is zero, and this validator has no per-metric optional flag. All three concrete store paths time themselves through Database::recordStoreDuration() (DatabaseNodeImp::store, DatabaseRotatingImp::store and Database::importInternal). So write_mean_us appears on any node that has stored at least one object. It can be promoted to an assertion once a harness run confirms it present. The two replay counters (ledger_replay_fallback_total, ledger_replay_outcome_total) are likewise NOT asserted, for the same reason _acquire_note gives for the InboundLedger acquire counters: both are emitted only from the ledger-replay path, which requires the [ledger_replay] config stanza AND a real historical back-fill against peers that support the LedgerReplay protocol feature. run-full-validation.sh starts a fresh local cluster with no history to back-fill, so no replay task is ever created and neither counter can move off 0. preRegisterMacroCounters() starts both at 0, so their series exist on every run. Both are covered by exact-value unit tests in src/tests/xrpld/telemetry/MetricMacros.cpp and rendered by the ledger-sync-health panels Replay Fallback To Full Acquire and Replay Outcomes. To make them assertable the harness would need to enable [ledger_replay] and start a node against an existing ledger history so it back-fills through the replay path.",
-    "_a5_note": "The quorum-and-publish signals are one observable gauge (ledger_quorum_publish) and one counter (ledger_quorum_shortfall_total). All four gauge sub-series are asserted and are unconditional: registerLedgerQuorumPublishGauge's callback in AppMetricGauges.cpp calls observe() for every field on every collection tick with no early return between them, and each accessor reads relaxed atomics and always returns a value, so the series exists whatever the reading. That deliberately includes the three diagnostic zeros: a node that has never had a gate evaluated reports trusted_validation_tally=0 and quorum_target=0, one that has never fully validated reports time_to_first_validated_us=0, and one that is caught up reports publish_lag=0. Absence, not the zero, is the regression -- the same reasoning _sync_state_note gives for initial_full_duration_us. Note the sentinel: when the trusted list disables quorum entirely, getNeededValidations() returns SIZE_MAX and LedgerMaster reports quorum_target as int64 max rather than letting the cast wrap to -1, so the target reads far above any tally instead of inverting the comparison (the same fix as the unl_quorum gauge). ledger_quorum_shortfall_total IS asserted, which differs from the acquire, replay and peer counters, and the reason is that this counter does not need a fault to fire. RCLConsensus::Adaptor::doAccept issues this node's own validation and then calls ledgerMaster_.consensusBuilt immediately (RCLConsensus.cpp), which calls checkAccept on the freshly built ledger (LedgerMaster.cpp), usually before most peers' validations for that same ledger have arrived. With the harness's 5 validators the quorum is max(ceil(5*0.8), ceil(5*0.6)) = 4 (ValidatorList::calculateQuorum), so that first evaluation of a round usually tallies short of 4 and takes the shortfall early return; the gate is then re-entered from RCLValidations handleNewValidation as each trusted validation arrives and eventually passes. A HEALTHY 5-node cluster therefore emits this counter in most rounds, which is why it is safe to assert on a clean run -- unlike peer_disconnect_total or ledger_replay_fallback_total, it needs no fault injection, no [ledger_replay] stanza and no historical back-fill. preRegisterMacroCounters() starts stage=\"pre_accept\" at 0, so the selector matches from startup and the metrics entry proves only that the labelled counter exports. The min_values entry checks the shortfall itself. It is cluster-wide, because whether a given node's first check of a round falls short depends on when the peers' validations arrive. Consequence for readers of the panels: a non-zero rate on Pre-Accept Quorum Shortfall Rate is NOT by itself a fault, and the panel description says so; the fault signature is that rate climbing well above the ledger-close rate while the tally on Trusted Validations vs Quorum Target stays flat below its target. If a future harness change makes the cluster single-node or standalone this assertion must move to a note: standalone_ short-circuits consensusBuilt before checkAccept, and getNeededValidations() returns 0 in standalone mode, so the gate can never report a shortfall.",
-    "_b5_sweep_note": "The per-sweep malloc_trim signals are one histogram and two counters. Only the histogram is asserted, by all three of its Prometheus series -- _bucket, _count and _sum -- because the bare instrument name is not a series. It is unconditional on any running node: ApplicationImp::start arms the sweep timer before the workload begins, the interval is SizedItem::SweepInterval (Config.cpp: 10 s at nodeSize 0 through 120 s at nodeSize 4), and the full-validation profile runs 270 s of workload before Step 5 scrapes -- so at least two sweeps complete even in the slowest case, and each one records exactly one sample. The measurement itself is now unconditional too: it used to sit inside `if (journal.debug())` in MallocTrim.cpp, so a node at ordinary log level measured nothing; that gate now covers only the JLOG. The two counters are deliberately NOT asserted. sweep_malloc_trim_minor_faults_total is incremented only when the trim's minor-fault delta is greater than zero, and a trim on the small heap of a fresh localhost node routinely faults zero times. Where the trim is measured (kMallocTrimSupported), preRegisterMacroCounters() starts both counters at 0, so their series exist and a 0 means no fault or no reclaim was counted; where it is not measured, neither counter is created. sweep_malloc_trim_reclaimed_kb_total is emitted only when resident memory actually FELL across the trim, which on a node whose caches are still filling frequently does not happen (glibc has nothing above the top of the heap to release, and mmap-backed allocations are returned on free regardless of trimming). So a metrics entry for either would pass with no event, and a min_values entry would fail on any run whose trims neither fault nor reclaim. Both are covered by exact-value unit tests in src/tests/xrpld/telemetry/MetricMacros.cpp -- including the skip paths -- and rendered by the ledger-sync-health panel Sweep Heap-Trim Faults & Reclaim Rate. To make them assertable the harness would need a node with a large enough resident heap for a trim to reclaim, e.g. starting against an existing populated database rather than from genesis.",
-    "_b5_rotation_note": "The online_delete rotation-write signals are one observable gauge (rotation_state, sub-series in_flight and copy_forward) and one counter (rotation_copy_node_restore_total). BOTH gauge sub-series ARE asserted; only the counter is excluded. A series is published even though no rotation runs. On a running node, registerRotationStateGauge's callback observes both fields once dynamic_cast<DatabaseRotating*> succeeds. The harness's xrpld-validator.cfg.template and run-full-validation.sh both set online_delete=256. So SHAMapStoreImp::makeNodeStore takes its deleteInterval_ != 0 branch and builds a DatabaseRotatingImp. The cast then succeeds, and both observe() calls run on every collection tick with no early return between them. So each series exists at value 0 (in_flight=0 meaning no rotation in progress, copy_forward=0 meaning none has yet copied anything), and absence rather than the zero is the regression, exactly as for the sync_state and peer_ledger_supply gauges. What the harness genuinely cannot produce is a rotation ACTUALLY RUNNING; that bounds the values, not the series. There are two independent reasons. Note that the mechanism sentence further down describes the callback's early return, not the instrument. AppMetricGauges::registerRotationStateGauge() creates the instrument eagerly and unconditionally, so it always exists. First, no rotation ever runs: xrpld-validator.cfg.template sets online_delete=256 and does not set advisory_delete, so SHAMapStoreImp's gate is validatedSeq >= lastRotated + 256 (SHAMapStoreImp.cpp), which needs 256 validated ledgers; at the network's several-seconds-per-ledger close rate that is on the order of 15-20 minutes, while the full-validation profile totals 270 s of workload before Step 5 scrapes. Second, even the in_flight flag needs a rotation to have started, and copy_forward additionally needs an ARCHIVE holding data that a fetch actually reads during the rotation window -- which requires a populated, already-rotated database, exactly the condition the hypothesis says is why this slowdown never appears on a fresh node. rotation_copy_node_restore_total is narrower still: it fires only for a clean tree node reachable from the validated state map whose sole on-disk copy was removed by an EARLIER rotation, so it needs at least two rotations plus real prior data loss. rotation_state publishes no series at all when online_delete is not configured, by design. The observe callback in AppMetricGauges::registerRotationStateGauge() dynamic_casts the node store to DatabaseRotating and returns early on failure. So an absent series means 'rotation is not configured', not the false 'rotation is free' that a zero would report. The instrument is still created, so this is a per-tick decision not to observe, not a missing instrument. All three signals are covered by exact-value unit tests in src/tests/xrpld/telemetry/MetricMacros.cpp (including the not-configured and between-rotations cases) and rendered by the ledger-sync-health panels Online-Delete Rotation Window & Copy-Forward Writes and Rotation Node Re-Stores. To make rotation_copy_node_restore_total assertable -- and to give the two asserted gauge sub-series a NON-ZERO reading rather than merely a present one -- the harness would need a step that starts a node against a pre-populated database that has already rotated at least once, or that lowers online_delete and advisory_delete far enough to force a rotation inside the run window and then re-scrapes before teardown.",
-    "_round_histogram_note": "consensus_round_duration_ms is a native OTel histogram recorded once per consensus round in RCLConsensus, so it needs no collector configuration -- it rides the existing OTLP -> Prometheus path. It is asserted by all three of its Prometheus series -- _bucket, _count and _sum -- because the bare instrument name is not a series. All three are unconditional on any running cluster: every node closes ledgers continuously, so a round completes within the harness window and the histogram is populated. Absence means the record site or the explicit-bucket view regressed, not that the node was idle. The instrument carries NO labels, so exactly one series exists per node and per bucket boundary.",
-    "_overlay_dial_series_note": "The three overlay_dial_latency_ms series report 4 series per run where dns_resolve_latency_ms reports 5, one per node. overlay_connect_total and dns_resolve_total do not show this, because preRegisterMacroCounters() gives every node a series at 0 for every outcome. That asymmetry is expected and benign, and it is recorded here so nobody investigates it again. A node records a dial outcome and latency only for a dial PeerFinder hands out. PeerFinder's autoconnect skips every address it already holds a connection to, by IP and ignoring the port, and every address in the harness's [ips] lists is 127.0.0.1, so a node that already holds a connection does not dial another harness node; the rest of its peers connect in. A node that gets an inbound connection before its first dial therefore makes no dial at all. Which node that is depends on startup and dial ordering, which the harness does not control, so the identity of the missing node is not stable across runs either. dns_resolve_latency_ms reports all 5 because reportDnsResolve fires inside the [ips] resolver handler in OverlayImpl::start, before any dial. This is why the three dial-latency entries assert series PRESENCE and not a series count, and why overlay_connect_total's min_values entry is cluster-wide. Asserting a count of 4 would bake today's start ordering into the contract: it would break the moment NUM_NODES or the peer seeding changes, neither of which is a telemetry defect. If a future run reports no dial series at all, that IS worth investigating: no node dialled.",
-    "_lock_hold_peak_note": "The two cache_metrics{metric=\"*_lock_hold_peak_us\"} series are unconditional: the cache_hit_rates callback observes both on every collection tick via TaggedCache::takeLockHoldPeak(), so each exists at value 0 on an idle node. A multi-second value in either is a mutex hold long enough to stall every job that reads that cache. Neither is asserted for VALUE, only existence; the values are the whole point but they only appear on a node that actually rotates."
-  },
-  "node_health_gauges": {
-    "description": "Node-health observable gauges (AppMetricGauges.cpp: registerServerInfoGauge, registerBuildInfoGauge, registerCompleteLedgersGauge, registerDbMetricsGauge). server_info, build_info and db_metrics Observe unconditionally on every periodic export (build_info observes a literal 1; server_info and db_metrics read live services), so their series exist regardless of workload shape. complete_ledgers is the exception and is asserted on a narrower guarantee: its callback returns without observing when the range is empty, and skips only the segments that MetricsRegistry::parseLedgerRange() cannot read. A one-sequence range reads as one ledger, so one complete ledger is enough. On a harness node, completeLedgers_ grows from the publish path in LedgerMaster::doAdvance, or from LedgerMaster::fetchForHistory when the node back-fills history. So closing a ledger is not enough; it has to validate. run-full-validation.sh waits for that before the workload starts, so on a healthy cluster the series always exists — a 5-node run yields 10 series, one start and one end per node. If this check ever fails, read the Step 3 output first: a run that logged 'No validated ledger' cannot produce this series and the cluster, not the exporter, is what broke.",
-    "metrics": ["server_info", "build_info", "complete_ledgers", "db_metrics"]
-  },
-  "overlay_reduce_relay": {
-    "description": "Transaction reduce-relay efficiency gauge (AppMetricGauges::registerReduceRelayGauge, peer-network dashboard). Backed by Overlay::txMetrics(); TxMetrics::json() emits txr_selected_cnt / txr_suppressed_cnt / txr_not_enabled_cnt unconditionally (TxMetrics.cpp:121-127), so the gauge always reports at least the selected_peers series.",
-    "metrics": ["reduce_relay_metrics"]
-  },
-  "overlay_overflow": {
-    "description": "Job-queue transaction overflow total (AppMetricGauges::registerJqTransOverflowCounter, job-queue dashboard). An ObservableCounter that reads Overlay::getJqTransOverflow() and Observes unconditionally, so the series exists at value 0 even when no overflow occurs.",
-    "metrics": ["jq_trans_overflow_total"]
-  },
-  "validation_lifetime_counters": {
-    "description": "Lifetime validation agreement/miss ObservableCounters (AppMetricGauges.cpp, validator-health dashboard). Both callbacks reconcile the tracker and then Observe only on a node with a validator key, so a node without one publishes neither series. Every node run-full-validation.sh starts is a validator with a [validation_seed], so both series exist from the first export, at 0 before any ledger is decided. Only existence is asserted, never the value.",
-    "metrics": ["validation_agreements_total", "validation_missed_total"]
-  },
-  "not_asserted": {
-    "description": "Emitted-and-dashboarded metrics deliberately left unasserted because they are workload-gated or defect-gated: the harness workload cannot guarantee their events, and a check that fails on a healthy run is worse than no check. Some of them start at 0, so their series exist on every run and only a value above 0 would show the event. This group has no \"metrics\" key, so validate_telemetry.py skips it (validate_metrics iterates category_data.get(\"metrics\", [])). Promote an entry into an asserted group only after the workload is changed to guarantee it. This map is the ONLY machine-readable record of an emitted-but-unasserted name, so every such name belongs here rather than in a prose note inside an asserted group -- prose cannot be linted. Concretely, phase-10 adds an _unaccounted_metric_names pass that harvests this map's keys as accounted names, so a name recorded only in a prose note is reported as unaccounted once that lands. It is warning-only and cannot fail CI, and it accepts a third source as well (an accounted_patterns regex list), so this map is the right home but not the only possible one. Caveat when adding: check_otel_naming.py Rule K harvests names only from `metrics` LISTS and from metric/name keys, so these dict KEYS are never validated against the MetricNames.h constants -- a typo here is silent, and must be checked by eye against the emit site cited in its own reason string.",
-    "metrics_excluded": {
-      "rpc_method_errored_total": "A push counter created in MetricsRegistry::initSyncInstruments(). 'Errored' here means a thrown C++ exception, not an error status in the JSON reply. Its one increment path is PerfLogImp::rpcError() -> PerfLogImp::rpcEnd() with finish false -> MetricsRegistry::recordRpcErrored(). PerfLog::rpcError() has one call site: the catch (std::exception&) handler in callMethod() (RPCHandler.cpp). An RPC that returns an error status still takes the rpcFinish() path in callMethod() and increments rpc_method_finished_total. A refused call is a normal return, not a throw. For example, with pathfinding off, as on every harness node, doRipplePathFind returns RpcNotSupported (see pathfind_fast_milliseconds below), and the counter does not move. The harness issues no path-finding command at all. Nothing in rpc_load_generator.py's server_info / fee / account / ledger / tx / DEX mix is expected to throw either. So no series may ever be created.",
-      "ledger_history_mismatch_total": "The beast::insight counter LedgerHistory builds with makeCounter('ledger.history', 'mismatch'). It is incremented at the top of LedgerHistory::handleMismatch on every built-vs-validated ledger mismatch, with no label; ledger_hash_mismatch_total below counts the same event by reason. It starts at 0 when LedgerMaster builds its LedgerHistory, so the series exists on every run. On a healthy run the value stays 0, and asserting a count would mean asserting a defect.",
-      "ledger_hash_mismatch_total": "MetricsRegistry counter with a reason label, recorded through MetricsRegistry::incrementLedgerHistoryMismatch() once LedgerHistory::handleMismatch has classified the mismatch: the same event as ledger_history_mismatch_total above. initSyncInstruments() starts every value in ledger_mismatch_reason::all at 0, so those series exist on every run. On a healthy run each stays 0, so it is not asserted for the same reason.",
-      "txq_expired_total": "Incremented only through MetricsRegistry::incrementTxqExpired(), from TxQ::processClosedLedger when a queued tx expires past its LastLedgerSequence. CI does run a txq-burst phase (workload-profiles.json, 30 s of single-type Payment at 60 TPS), but that does not guarantee sustained fee escalation followed by expiry. initSyncInstruments() starts it at 0, so the series exists on every run, and only a value above 0 would show an expiry.",
-      "txq_dropped_total": "Incremented only through MetricsRegistry::incrementTxqDropped(), from the two queue-full refusals in TxQ::apply, with reason=queue_full, the one value in txq_drop_reason::all. initSyncInstruments() starts that series at 0. Same reason as txq_expired_total.",
-      "getobject_rejected_total": "kGetObjectRejectedTotal (GetObjectMetricNames.h), recorded by PeerImp only for a TMGetObjectByHash message refused as oversize or malformed_ledgerhash. A cooperating cluster never sends one. preRegisterGetObjectCounters() (PreRegisteredCounters.h), called from ApplicationImp::setup(), starts both reasons at 0, so the series exist on every run, and only a value above 0 would show a refusal.",
-      "getobject_request_objects": "kGetObjectRequestObjects (GetObjectMetricNames.h), recorded by PeerImp only while serving an inbound TMGetObjectByHash. It is a histogram, and the recording macro creates the instrument on its first record, with no startup zero, so no series exists until a peer actually requests objects by hash — which a 5-node cluster started at genesis and already in sync may never do.",
-      "getobject_lookup_us": "GetObjectMetricNames.h:95, PeerImp.cpp:2929. Same lazy-creation and same inbound-request gate as getobject_request_objects.",
-      "getobject_lookups_total": "GetObjectMetricNames.h:100, PeerImp.cpp:2949/:2956. Same gate.",
-      "getobject_charge": "GetObjectMetricNames.h:105, PeerImp.cpp:2931. Same gate.",
-      "unl_fetch_total": "ValidatorSite.cpp:435, emitted only from reportFetchOutcome, which indexes sites_[siteIdx]. sites_ is populated from [validator_list_sites]; run-full-validation.sh writes a static [validators] file and no [validator_list_sites] exists anywhere under docker/telemetry/workload/, so sites_ is empty and no fetch outcome is ever reported. Needs a harness that serves a UNL over HTTP.",
-      "handshake_negotiation_fail_total": "Handshake.cpp:265, emitted only when a handshake is REJECTED. Every node in the cluster shares one network id and one build, dials 127.0.0.1 and has no clock skew, so no reject path was found to be reachable. Not proven unreachable. preRegisterMacroCounters() starts every reason at 0, so the series exists on every run; promote it to min_values only if a run ever shows a value above 0.",
-      "sync_acquire_source_total": "InboundLedger.cpp:165, emitted from InboundLedger::init(), the same function that opens the ledger.acquire span. expected_spans.json already marks that span optional (\"A healthy local cluster rarely back-fills history\"), so the counter inherits exactly that condition. Best first candidate to promote if the harness ever forces an acquire: unlike its siblings it fires even when the store was already complete, via source=local.",
-      "sync_acquire_no_progress_total": "InboundLedger.cpp:536. Strictly narrower than sync_acquire_source_total: needs an acquire AND a timer tick that made no progress.",
-      "sync_addnode_total": "InboundLedger.cpp:1636 (recordBatchOutcome). Needs an acquire that actually receives SHAMap nodes over the wire; the emit site returns early when the batch count is <= 0. The series exist anyway, because preRegisterMacroCounters() starts every outcome at 0.",
-      "ledger_replay_fallback_total": "SkipListAcquire.cpp:113 / LedgerDeltaAcquire.cpp:117. Structurally impossible here: reachable only from a LedgerReplayTask, created only via LedgerReplayer::replay(), whose sole production caller is LedgerMaster.cpp:1459 behind `if (app_.config().ledgerReplay)`. Config::ledgerReplay defaults false and is set only from [ledger_replay], which the harness never writes.",
-      "ledger_replay_outcome_total": "LedgerReplayTask.cpp:237. Same [ledger_replay] gate as ledger_replay_fallback_total, plus a real historical back-fill against peers advertising the LedgerReplay feature.",
-      "ledger_jump_total": "NetworkOPs.cpp:2307, unconditional inside switchLastClosedLedger, which is reached from checkLastClosedLedger -- called every round at :2489. Excluded as NOT DELIBERATELY PROVOKED rather than as impossible: the harness injects no divergence, but a node that falls behind can be told the network's LCL is not the one it built on and follow it, and the rpc-burst / tx-flood / txq-burst phases exist precisely to create load spikes (the same group already asserts server_stall_events_total on that basis). So this may fire on some healthy runs and not others, which is a flaky check either way. Cheap decisive test before ever promoting it: grep the five nodes' debug.log for 'JUMP last closed ledger' after a run.",
-      "peer_disconnect_total": "PeerImp.cpp:655, inside PeerImp::close(), immediately after the !socket_.is_open() early return that de-duplicates repeat closes -- so it does fire on the first genuine close, for any of its 13 reason values. Excluded because a healthy run produces no disconnect CAUSE, not because there is too little time: the cluster's 5 nodes hold each other for the whole run, the timer-driven reasons require the peer to actually be diverged or unknown rather than merely for maxDivergedTime (300 s) / maxUnknownTime (600 s) to pass, and the shutdown reasons fire during --cleanup teardown, after Step 5 has scraped. Do not rest this on the window length: the profile's 270 s of workload plus the 60 s propagation wait, node startup and up to 45 s of metric polling exceeds 300 s. What keeps it out of the gate is that a mutual-dial race CAN produce one non-deterministically -- asserting a signal that appears on some healthy runs and not others is a flaky check either way.",
-      "serve_refused_total": "PeerImp.cpp:669 (reportServeRefusal), reached from the notFound paths at :3759 (tx set) and :3793 (ledger). Excluded because nothing in the harness ASKS: a refusal requires an inbound TMGetLedger, and that message is originated only from InboundLedger.cpp:904 and TransactionAcquire.cpp:304/340 -- the ledger.acquire and txset.acquire paths, both of which expected_spans.json marks optional because a healthy cluster does not back-fill. Note the reason is the absence of requests, NOT that a request would succeed: PeerSetImpl::addPeers uses hasItem(peer) only as a sort SCORE and then adds peers in score order up to its limit (PeerSet.cpp:77-95), so peers that lack the item are asked too and would be refused. It follows that this counter and the acquire spans stand or fall together, and if a harness change ever makes an acquire reliable this should be promoted alongside them.",
-      "peer_tx_rejected_total": "PeerImp::handleTransaction, one reason per path: duplicate when a peer relays a transaction this node saw within the HashRouter window, known_bad when that transaction is marked bad, and inner_batch for a relayed tfInnerBatchTxn. preRegisterMacroCounters() starts all three reasons at 0, so the series exist on every run. Not asserted: whether the harness's relay pattern yields a duplicate on every run is not established, and known_bad and inner_batch need a misbehaving peer.",
-      "sweep_malloc_trim_minor_faults_total": "ApplicationImp::trimHeapAndRecord (Application.cpp), incremented only when the trim's minor-fault delta is > 0. A trim on the small heap of a fresh localhost node routinely faults zero times. Where the trim is measured (kMallocTrimSupported), preRegisterMacroCounters() starts it at 0, so the series exists and a 0 means no fault was counted; where the trim is not measured, the counter is not created.",
-      "sweep_malloc_trim_reclaimed_kb_total": "ApplicationImp::trimHeapAndRecord (Application.cpp), incremented only when resident memory actually FELL across the trim. On a node whose caches are still filling this frequently does not happen: glibc has nothing above the top of the heap to release, and mmap-backed allocations are returned on free regardless of trimming. Pre-registered at 0 on the same terms as sweep_malloc_trim_minor_faults_total.",
-      "rotation_copy_node_restore_total": "SHAMapStoreImp.cpp:283. Fires only for a clean tree node reachable from the validated state map whose sole on-disk copy an EARLIER rotation removed, so it needs at least two rotations (512 validated ledgers, ~15-20 min at the cluster's close rate) plus real prior data loss. The run window is 270 s. Note that the rotation_state gauge sub-series ARE asserted -- see sync_diagnostics._b5_rotation_note for why a series exists while a rotation never runs.",
-      "rpc_size_bytes": "ServerHandler.cpp:191, group('rpc')->makeEvent('size', Unit::Bytes). The OTLP Prometheus exporter derives the metric-name suffix from the declared unit, so a byte unit yields rpc_size_bytes. The Unit::Bytes declaration itself landed earlier, in 76c9051203; what 24094e427b changed was the exporter finally consuming it, replacing a hardcoded CreateDoubleHistogram(name, 'Duration in ms', 'ms') with otelUnitDescription(unit)/otelUnitCode(unit), and that is what renamed the series off rpc_size_milliseconds and the millisecond bucket ladder. Neither name was ever recorded here, so the harness could confirm neither the rename nor a regression back onto that ladder. Notified from ServerHandler::processRequest:1133, the HTTP JSON-RPC path — it computes an HTTP status and appends a trailing newline — and the load generators are WebSocket-only, the same gate regression-metrics.json:4 records for rpc.process, so only the harness's handful of HTTP health polls reach it. Real coverage needs an HTTP JSON-RPC phase in rpc_load_generator.py; that is a workload change rather than a harness correction, and is deliberately out of scope here.",
-      "rpc_time_milliseconds": "ServerHandler.cpp:192, group('rpc')->makeEvent('time') with the default millisecond unit. Notified from ServerHandler::processRequest:1129, the same HTTP JSON-RPC call site as rpc_size_bytes and behind the same WebSocket-only gate.",
-      "pathfind_fast_milliseconds": "PathRequestManager.h:35, makeEvent('pathfind_fast') with the default millisecond unit, so the exported form is the pathfind_fast_milliseconds_bucket/_count/_sum triple and there is no bare series — the same convention io_latency and rpc_method_us follow, and rpc-pathfinding queries the _bucket. THE OPERATIVE BLOCKER IS THE CONFIG, NOT THE CALL GRAPH: pathfinding is disabled outright on every harness node, so no PathRequest is ever constructed and no pathfind_* histogram can exist. Config.cpp:725-726 sets pathSearchMax to 0 whenever a [validation_seed] or [validator_token] section is present ('By default, validators don't have pathfinding enabled'); run-full-validation.sh writes [validation_seed] into every generated node cfg and contains no [path_search], [path_search_fast] or [path_search_max] section to put it back (grep count 0 for path_search in that file — the only [path_search*] sections in docker/telemetry/ are in xrpld-telemetry.cfg and xrpld-telemetry-mainnet.cfg, neither of which the harness uses); and doRipplePathFind returns RpcNotSupported at RipplePathFind.cpp:59-60, before context.loadType is set and before any branch on the ledger parameter. That config gate alone would keep the metric absent even if the generator did issue the command, because every such call is refused at the front door; and the generator issues no path-finding RPC in the first place. Two independent reasons. Read that first: the structural argument below is correct and matters if pathfinding is ever enabled, but it is not why the metric is missing today. STRUCTURAL ARGUMENT (verified, applies once pathSearchMax is non-zero): reportFast's only caller is PathRequest.cpp:852, inside the 'if (fast && quickReply_ == {})' branch of PathRequest::doUpdate. The only doUpdate call that passes fast=true is in PathRequest::doCreate (PathRequest.cpp:259), guarded by '!hasCompletion()'. Both ripple_path_find entry points construct the PathRequest with a completion function, so hasCompletion() (:161-164) is true and the fast pass is skipped: with no ledger specified doRipplePathFind goes to makeLegacyPathRequest, which passes the coroutine-post lambda (RipplePathFind.cpp:140-160); with a ledger specified it goes to doLegacyPathRequest, which passes an empty-body but non-null lambda (PathRequestManager.cpp:317). Only the path_find streaming subscription reaches reportFast, because makePathRequest builds the request from a subscriber with no completion (PathRequestManager.cpp:261). The load generator has never used path_find: it fires one request per send and awaits one reply, which a streaming subscription does not fit. Covering this metric therefore needs both a [path_search_max] override (or a non-validator node) in run-full-validation.sh and a path_find subscription phase in the generator. Both are harness/workload changes and out of scope here. Grafana Cloud shows zero series in 180 days on the devnet nodes, consistent with those nodes receiving no pathfinding RPC.",
-      "pathfind_full_milliseconds": "PathRequestManager.h:36, makeEvent('pathfind_full'); same histogram naming as pathfind_fast_milliseconds above. Notified from reportFull (:87-90) via PathRequest.cpp:857, the 'else if (!fast && fullReply_ == {})' branch. Blocked by exactly the same config gate as pathfind_fast_milliseconds, and the probability of emission under the harness workload is zero, not low: pathSearchMax is 0 on every harness node (Config.cpp:725-726 plus the [validation_seed] section run-full-validation.sh writes, with no [path_search*] override anywhere in that file), so doRipplePathFind returns RpcNotSupported at RipplePathFind.cpp:59-60 and no PathRequest object is ever constructed for reportFull to fire from. An earlier revision of this entry described the path as reachable-but-probabilistic — emitting one ledger close behind the request via PathRequestManager::updateAll's one-shot branch (PathRequestManager.cpp:160-166) — and prescribed sending an explicit ledger_index from the generator so doLegacyPathRequest would call doUpdate(cache, false) synchronously (PathRequestManager.cpp:321). Both halves were wrong. The probability is zero rather than merely unreliable, and the prescribed remedy cannot work at all, because the pathSearchMax guard fires before the ledger parameter is read: adding ledger_index changes nothing while pathfinding is off. The generator also issues no path-finding RPC, so covering this metric needs a [path_search_max] override (or a non-validator node) in run-full-validation.sh AND path-finding load added — a harness-topology plus workload change, out of scope here. The workload README section 'Pathfinding is not exercised' holds the recipe. Grafana Cloud shows zero series in 180 days on the devnet nodes, consistent with those nodes receiving no pathfinding RPC rather than with a broken exporter.",
-      "warn_total": "Logic's Stats builds makeMeter('warn') (include/xrpl/resource/detail/Logic.h). On the OTel path a meter is an OTelMeterImpl over CreateUInt64Counter (OTelCollector.cpp), so the Prometheus exporter appends _total; the meter is created on the bare collector with no group, hence the unprefixed name. Incremented only in Logic::warn(), inside its 'if (notify)' branch reached when a consumer's balance crosses kWarningThreshold. A cooperating 5-node cluster plus a rate-limited load generator never charges a consumer that far. OTelMeterImpl records 0 when it is built, and resource::Manager builds its Logic at startup, so the series exists on every run, and the rpc-pathfinding panel that queries it reads 0 rather than empty. Only a value above 0 would show a warning.",
-      "drop_total": "Logic's Stats builds makeMeter('drop') in the same file; same OTelMeterImpl mapping and same _total suffix as warn_total. Incremented only in Logic::disconnect(), when a consumer's balance is at or above kDropThreshold and the connection is dropped. It fires only on a genuinely abusive consumer, which the harness deliberately does not create, so it is condition-gated. Like warn_total it starts at 0 at startup, so the series exists on every run, and only a value above 0 would show a drop.",
-      "jobq_*_milliseconds, jobq_*_q_milliseconds": "This key is a pattern rather than a literal metric name — unlike every other entry in this map it stands for a whole family, one pair per job type. Created per job type in JobTypeData.h:97-98 from info.name() and info.name() + kSuffixQueued ('_q'), so the exported names are jobq_<jobtype>_milliseconds and jobq_<jobtype>_q_milliseconds with the job type lowercased by formatName. Which job types appear depends on which jobs a run happens to schedule, so no individual name is guaranteed. They are also rounded up to a whole millisecond at source (Event.h:47-51 applies ceil to a millisecond value type), which is why 6e2b2da772 moved the ledger-data-sync q-wait panels off jobq_<jobtype>_q_milliseconds_bucket onto job_queued_us_bucket — they are poor assertion targets regardless.",
-      "rotation_phase_duration_seconds": "Recorded only at the end of an online-delete rotation phase (SHAMapStoreImp::RotationPhase destructor). The harness never rotates; see the nodestore.rotate note in expected_spans.json. Carries stage and, on the freshen stage only, cache; the other stages emit cache=\"\", which Prometheus reads as absent.",
-      "rotation_freshen_keys_total": "Two Adds each time a rotation freshens a cache, from SHAMapStoreImp::recordFreshen, labelled cache (treenode | master_tx) and outcome (fetched | copied). preRegisterMacroCounters() creates all four series at 0 at startup, so they exist even though the harness never rotates (see sync_diagnostics._b5_rotation_note). Asserting it would check only that a series exists, so it would pass on those zeros alone and prove nothing about the recording site. Promote it once a harness step forces a rotation.",
-      "jobq_stall_total": "Incremented in MetricsRegistry::recordJobFinished only when a job ran >= kJobStallThresholdUs (1 s). A healthy 5-node localhost run has no such job, so the value stays 0. MetricsRegistry::initSyncInstruments() starts it at 0 for every job type that is not special(), so the series exist on every run; promote it to min_values only with a workload that forces a stall.",
-      "consensus_view_change_total": "Incremented only when RCLConsensus::Adaptor::getPrevLedger sees the network's preferred ledger differ from this node's and mode is not already WrongLedger. A healthy 5-node harness never diverges from the network view, so the value stays 0. preRegisterMacroCounters() starts every mode a view change can leave at 0, so the series exist on every run."
-    }
-  },
-  "accounted_patterns": [
-    {
-      "pattern": "^jobq_[a-z]+_(waiting|running|deferred)$",
-      "family": "Per-job-type queue-depth gauges: 35 non-special job types x 3 states = 105 families.",
-      "reason": "Cannot be enumerated usefully. JobTypeData's constructor creates waiting/running/deferred for every non-special job type (JobTypeData.h:96-103) and JobQueue eagerly constructs a JobTypeData for every type in JobTypes (JobQueue.cpp:45-53), so the family's membership is derived mechanically from the JobTypes.h table rather than chosen. Six members are asserted by literal name in job_queue_per_type_gauges on consumer-coverage grounds; this pattern accounts for the rest so the reverse coverage check does not report 99 names that the contract has already reasoned about. Narrow by construction: every one of the 35 non-special job type names lowercases to ^[a-z]+$ with no digits and no underscores (verified mechanically against JobTypes.h:52-86), and the state suffix is a closed set of three, so nothing outside this family can match. A new job type therefore stays covered without an edit here, which is correct — adding a job type is not a telemetry design change. A renamed state, a job type containing a digit or an underscore, or any other new jobq gauge shape would surface as unaccounted."
-    },
-    {
-      "pattern": "^jobq_[a-z]+(_q)?_milliseconds$",
-      "family": "Per-job-type job-latency and queue-wait histogram families: 35 job types x 2 = 70 families, each exported as a _bucket/_count/_sum triple.",
-      "reason": "The same mechanically derived family as the gauges above, in histogram form: JobTypeData.h:97-98 creates one Event from info.name() and one from info.name() + '_q'. Which job types actually record a sample depends on what a run schedules, which is why not_asserted.metrics_excluded carries these under a single pattern key rather than as literal names. The reverse coverage check matches this pattern against the base family name, after stripping the exporter's _bucket/_count/_sum suffix, so all three series of every triple are accounted for by this one entry. Same narrowness argument as above: the job type part is ^[a-z]+$ and the optional _q plus the _milliseconds unit suffix close the shape."
-    },
-    {
-      "pattern": "^(getobject_account_state_node_get|getobject_account_state_node_share|getobject_cas_get|getobject_cas_share|getobject_fetch_pack_get|getobject_fetch_pack_share|getobject_get|getobject_ledger_get|getobject_ledger_share|getobject_share|getobject_transaction_get|getobject_transaction_node_get|getobject_transaction_node_share|getobject_transaction_share|getobject_transactions_get|have_transactions|ledger_account_state_node_get|ledger_account_state_node_share|ledger_data_account_state_node_get|ledger_data_account_state_node_share|ledger_data_get|ledger_data_share|ledger_data_transaction_node_get|ledger_data_transaction_node_share|ledger_data_transaction_set_candidate_get|ledger_data_transaction_set_candidate_share|ledger_get|ledger_share|ledger_transaction_node_get|ledger_transaction_node_share|ledger_transaction_set_candidate_get|ledger_transaction_set_candidate_share|overhead|overhead_cluster|overhead_manifest|overhead_overlay|proof_path_request|proof_path_response|proposals|proposals_duplicate|proposals_untrusted|replay_delta_request|replay_delta_response|requested_transactions|set_get|set_share|squelch|squelch_ignored|squelch_suppressed|total|transactions|transactions_duplicate|unknown|validations|validations_duplicate|validations_untrusted|validator_lists)_(bytes|messages)_(in|out)$",
-      "family": "Overlay per-category traffic gauges: 57 categories x {bytes,messages} x {in,out} = 228 families, of which the four total_* are asserted literally in overlay_traffic.",
-      "reason": "Also mechanically derived: OverlayImpl builds one TrafficGauges per entry of TrafficCount::counts_ (OverlayImpl.cpp:200-203), and TrafficGauges creates Bytes_In / Bytes_Out / Messages_In / Messages_Out for each (OverlayImpl.h:580-593), so the set is exactly the cross product of the category table with those four names. The 57 categories are the 56 entries of TrafficCount::toString's kCategoryMap (TrafficCount.h:239-294) plus 'unknown', which is what Category::Unknown falls through to because it has an entry in counts_ but none in the map. The categories are enumerated here rather than reduced to a shape, because unlike the job types they contain underscores: a structural pattern would have to be ^[a-z_]+_(bytes|messages)_(in|out)$, which would silently swallow any future non-overlay metric ending in _bytes_in. Enumerating means a new traffic category surfaces as unaccounted, which is the right outcome: overlay_traffic's description calls itself an explicit subset and should be revisited when the table grows. The category list was extracted mechanically from TrafficCount.h; regenerate it the same way rather than editing by hand."
-    },
-    {
-      "pattern": "^(target_info|up|scrape_[a-z_]+|promhttp_[a-z_]+)$",
-      "family": "Prometheus scrape metadata and exporter housekeeping series.",
-      "reason": "Not xrpld telemetry, and not emitted by anything in this repository, so there is nothing for the contract to assert. prometheus.yml defines exactly one scrape job against the collector's Prometheus exporter (otel-collector:8889), and Prometheus synthesises up and the scrape_* family per target on every scrape. target_info is written by the collector's Prometheus exporter to carry the OTLP resource attributes, and several dashboards read it as label_values(target_info, service_instance_id). promhttp_* is the metrics handler's own counter set, present only if the exporter's handler exposes it. Listed so the reverse coverage check reports genuine telemetry gaps instead of scrape plumbing. Deliberately does not cover otelcol_*: the collector's internal telemetry defaults to port 8888 and otel-collector-config.yaml declares no service::telemetry block, so those series are not scraped here — if they ever appear, that is a collector configuration change worth seeing."
-    }
-  ],
-  "grafana_dashboards": {
-    "description": "Every Grafana dashboard provisioned on disk under docker/telemetry/grafana/dashboards/ (UID == file stem for every one). validate_dashboards() checks that each UID resolves via GET /api/dashboards/uid/<uid> and reports its panel count — it verifies provisioning and loadability, not panel data. log-derived-insights is included on that basis even though its panels are Loki-backed: the dashboard itself must still provision cleanly. Its panel data is not asserted anywhere.",
-    "uids": [
-      "rpc-performance",
-      "transaction-overview",
-      "consensus-health",
-      "ledger-operations",
-      "peer-network",
-      "peer-quality",
-      "fee-market",
-      "job-queue",
-      "validator-health",
-      "node-health",
-      "network-traffic",
-      "rpc-pathfinding",
-      "overlay-traffic-detail",
-      "ledger-data-sync",
-      "ledger-sync-health",
-      "log-derived-insights"
-    ]
-  }
-}

docker/telemetry/workload/expected_spans.json

@@ -1,796 +0,0 @@
-{
-  "description": "Expected span inventory for xrpld telemetry validation. Attribute keys follow the 2026-05-13 span-attr naming redesign (bare/underscore form; dotted xrpl.* reserved for resource attributes). Sourced from the *SpanNames.h headers and verified against the emitting call sites. Spans marked \"optional\": true are conditional — they only fire under traffic the harness may not produce (e.g. gRPC client, path-finding RPC, missing-ledger fetch, mode transitions) and are not failed when absent. \"allowed_parents\" is asserted, by validate_span_parents in validate_telemetry.py: every emitted instance of the span must be parented to one of the names listed, so a span declared a root that is emitted as somebody's child now fails a check instead of passing silently. \"ROOT\" means \"no parent from this node\" — a genuine root, an explicit freshRoot, or a span whose parent was created on a different node, because a cross-node parent is the design for the receive spans and is never a violation. A span reachable by two call paths lists every parent either path can give it, and a listed parent may itself be a glob (pathfind.request is declared under rpc.command.*), matched the same way the span names in this file are. Each list is derived from the span's creation factory plus every call path that reaches it — README.md, section \"expected_spans.json Format\", has the rule — so a span note that lists fewer parents than the code can produce is a bug in this file, not a reason to widen the gate at the call site. Three spans list ROOT even though their factory inherits the ambient scope (ledger.acquire, pathfind.update_all, grpc.*): they are created at a boundary or outlive the creating scope, so an observed parent there is an ambient leak to fix in the C++, not an edge to add here. required_attributes lists only attributes set on EVERY code path that creates the span — attributes set after an early return are described in the span's note instead, because _validate_span_attributes_otlp samples a single trace and would fail on a legitimate short-circuit path. total_unique_attributes is the size of the union of all required_attributes; total_span_types is len(spans). Span EVENTS (consensus.round phase.*/outcome.*, consensus.update_positions dispute.resolve, consensus.accept.apply tx.included) are NOT represented: validate_telemetry.py reads only span name, attributes and timestamps from Tempo, so an \"events\" key would be silently ignored. They are documented in the relevant span notes until the validator gains event support.",
-  "spans": [
-    {
-      "name": "rpc.ws_message",
-      "category": "rpc",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["command"],
-      "config_flag": "trace_rpc",
-      "note": "WebSocket RPC root span. The load generator uses WS, so this is the RPC entry span (not rpc.http_request, which needs an HTTP/JSON-RPC client)."
-    },
-    {
-      "name": "rpc.ws_upgrade",
-      "category": "rpc",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": [],
-      "config_flag": "trace_rpc",
-      "optional": true,
-      "note": "WebSocket handshake span (ServerHandler::onHandoff, ServerHandler.cpp:272-273). A freshRoot with no attributes — only setOk() on success or recordException() on an upgrade failure. Fires once per WS connection, so the load generator produces only a handful of these at connect time; by the time validation runs after the propagation wait they may fall outside the Tempo search window. Optional for that reason, not because the code path is conditional."
-    },
-    {
-      "name": "rpc.process",
-      "category": "rpc",
-      "allowed_parents": ["rpc.http_request"],
-      "required_attributes": [],
-      "config_flag": "trace_rpc",
-      "optional": true,
-      "note": "HTTP-only. Created solely in ServerHandler::processRequest() (ServerHandler.cpp:718), which is reached only from processSession(Session, coro) (ServerHandler.cpp:646) — the HTTP/JSON-RPC path that roots rpc.http_request at ServerHandler.cpp:640-641. The WebSocket path (processSession(WSSession, coro, jv), ServerHandler.cpp:467) never calls processRequest, so this span never appears under a WebSocket request. It does still appear under this harness, 5 traces on a normal run, because run-full-validation.sh polls each node over HTTP with curl (:449, :502) and those requests take the HTTP path. Note that the harness does speak HTTP: a reader concluding this span is unreachable here would go looking for a way to add HTTP traffic that already exists."
-    },
-    {
-      "name": "rpc.startup",
-      "category": "rpc",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": [],
-      "config_flag": "trace_rpc",
-      "note": "Wraps the [rpc_startup] batch ApplicationImp::setup() runs, so the rpc.command.* spans those commands create have a parent. A freshRoot, created only when the section has lines. Required here because every harness node config writes one [rpc_startup] command. No attributes: the command names are on its children."
-    },
-    {
-      "name": "rpc.command.*",
-      "category": "rpc",
-      "allowed_parents": ["rpc.ws_message", "rpc.process", "rpc.startup"],
-      "required_attributes": ["command", "version", "rpc_role", "rpc_status"],
-      "config_flag": "trace_rpc",
-      "note": "Wildcard — matches rpc.command.server_info, rpc.command.ledger, etc. Created as a plain ScopedSpanGuard in rpc::callMethod (RPCHandler.cpp:204-205) and, on the fillHandler-rejection path, in rpc::doCommand (:340-341), so its parent is whichever transport span is ambient on the thread. Three call paths, all three exercised by this harness: rpc::doCommand from the WebSocket handler inside the rpc.ws_message scope (ServerHandler.cpp:589 under the freshRoot at :495); from ServerHandler::processRequest inside the rpc.process scope (:1037 under the guard at :734); and from ApplicationImp::setup's [rpc_startup] loop, inside the rpc.startup scope that loop opens. startTelemetry() runs well before that loop, so a startup command does emit a command span, and with rpc.startup as its parent every rpc.command span has one. run-full-validation.sh writes a [rpc_startup] log_level command for every node, so there are five of them per run."
-    },
-    {
-      "name": "rpc.http_request",
-      "category": "rpc",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["request_payload_size"],
-      "config_flag": "trace_rpc",
-      "optional": true,
-      "note": "HTTP/JSON-RPC root span. It DOES fire under this harness, 5 traces on a normal run -- one per node -- even though the load generator is WebSocket-only, because run-full-validation.sh polls each node's HTTP port with curl for readiness and validated-ledger progress (:449, :502). Kept optional rather than required because those polls are harness scaffolding rather than workload: a future change to how the script waits for a node could remove them without anything being wrong with the node."
-    },
-    {
-      "name": "tx.process",
-      "category": "transaction",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["tx_hash", "local", "path"],
-      "config_flag": "trace_transactions",
-      "note": "Also carries tx_type, fee, sequence, and one attribute per account-typed top-level field of the transaction: tx_account always, plus tx_destination, tx_owner, tx_issuer and so on when the transaction has them (keys in TxAccountSpanNames.h, raw r-addresses). These are set only when the STTx parses, so like tx_type they are not required attributes."
-    },
-    {
-      "name": "tx.receive",
-      "category": "transaction",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["tx_hash", "peer_id", "tx_status"],
-      "config_flag": "trace_transactions",
-      "note": "Cross-node span: parent context propagated from the sender's tx.process via protobuf. Also carries current_ledger_seq, and tx_type and peer_version when known. The span is created only after the node decides to process the transaction, so a relayed duplicate produces no span at all; how many were dropped is the transactions_duplicate traffic category. tx_status is set once on every path after that point: dropped_no_sync, dropped_queue_full, dropped_queue_stopping or queued_for_check."
-    },
-    {
-      "name": "tx.apply",
-      "category": "transaction",
-      "allowed_parents": ["ledger.build"],
-      "required_attributes": ["tx_count", "tx_failed"],
-      "config_flag": "trace_transactions",
-      "note": "Apply-step span inside BuildLedger. Sets its own ledger_seq next to tx_count and tx_failed (BuildLedger.cpp:197-199), so do not re-add the earlier claim that ledger_seq lives on the parent ledger.build span — it does not, and ledger.build sets it independently."
-    },
-    {
-      "name": "tx.preflight",
-      "category": "transaction",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["stage", "tx_type", "ter_result"],
-      "config_flag": "trace_transactions",
-      "note": "Apply-pipeline stage span (stage=preflight). Shares a deterministic trace_id (txID[0:16]) with tx.preclaim/tx.transactor."
-    },
-    {
-      "name": "tx.preclaim",
-      "category": "transaction",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["stage", "tx_type", "ter_result"],
-      "config_flag": "trace_transactions",
-      "note": "Apply-pipeline stage span (stage=preclaim)."
-    },
-    {
-      "name": "tx.transactor",
-      "category": "transaction",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["stage", "tx_type"],
-      "config_flag": "trace_transactions",
-      "note": "Apply-pipeline stage span (stage=apply). Also carries applied."
-    },
-    {
-      "name": "txq.enqueue",
-      "category": "transaction",
-      "allowed_parents": [
-        "tx.process",
-        "consensus.accept.apply",
-        "rpc.command.*",
-        "ROOT"
-      ],
-      "required_attributes": ["tx_hash", "tx_type", "txq_status"],
-      "config_flag": "trace_transactions",
-      "optional": true,
-      "note": "Fires on every TxQ::apply call: each submitted tx, each local tx replayed on an open-ledger rebuild, and simulate. txq_status names the outcome (rejected, applied_direct, failed, applied, queued); only queued means the tx entered the queue. tx_hash/tx_type/txq_status are set on every code path; fee_level_paid/required_fee_level are conditional (TxQ.cpp ~895-898, after the rejected and applied_direct early exits), so they are NOT guaranteed on every txq.enqueue span and cannot be required. Four parents, because TxQ::apply (TxQ.cpp:770) takes an OPTIONAL parent context (:790-795): with a context it is an explicit childSpan, without one it is a plain ScopedSpanGuard that inherits whatever is ambient. tx.process comes from the submission path, which passes the captured context (NetworkOPs.cpp:1803-1810). consensus.accept.apply comes from the open-ledger rebuild, which passes none: OpenLedger.cpp:121 replays local transactions, and OpenLedger::accept is called from inside doAccept (RCLConsensus.cpp:871) where the consensus span is ambient. ROOT comes from that same rebuild reached by the switchLastClosedLedger jump path instead (NetworkOPs.cpp:2313), which has no ambient scope, and from the submission path when the tx.process context is invalid. rpc.command.* comes from the simulate RPC (Simulate.cpp:249), which passes no context and runs inside the command span's scope; the harness never issues simulate, so that parent is unreachable here but is still lawful."
-    },
-    {
-      "name": "txq.apply_direct",
-      "category": "transaction",
-      "allowed_parents": ["txq.enqueue"],
-      "required_attributes": [],
-      "config_flag": "trace_transactions",
-      "optional": true,
-      "note": "Child of txq.enqueue when the tx applies directly without queueing."
-    },
-    {
-      "name": "txq.batch_clear",
-      "category": "transaction",
-      "allowed_parents": ["txq.enqueue"],
-      "required_attributes": [],
-      "config_flag": "trace_transactions",
-      "optional": true,
-      "note": "TxQ::tryClearAccountQueueUpThruTx (TxQ.cpp:571-572). num_cleared is NOT required: it is written only once the batch has actually cleared, inside the `if (txResult.applied)` branch at TxQ.cpp:646. Three earlier returns leave the span without it — a fee-level overflow (:589-590), a batch that did not pay enough (:598-599), and a queued transaction that failed to apply (:632-635) — as does the final doApply failing, which falls past the branch. The span is opened before any of those decisions, so a required num_cleared would fail the check on a legitimate fall-back-to-normal-processing path rather than on a defect."
-    },
-    {
-      "name": "txq.accept",
-      "category": "transaction",
-      "allowed_parents": ["consensus.accept.apply", "ROOT"],
-      "required_attributes": ["queue_size", "ledger_changed"],
-      "config_flag": "trace_transactions",
-      "optional": true,
-      "note": "Ledger-close accept loop (TxQ::accept, TxQ.cpp:1499). Only meaningful when the queue is non-empty. Two call paths, and they give it two different parents. On the consensus path (RCLConsensus.cpp:823, inside doAccept) consensus.accept.apply is the ambient scope for the whole of doAccept, so this span is its child. On the switchLastClosedLedger jump path (NetworkOPs.cpp:2150) no consensus span is open on that thread, so the span is a root. The span is a ScopedSpanGuard and adopts whatever OTel context is ambient, which is what makes the parent a property of the caller rather than of this span."
-    },
-    {
-      "name": "txq.accept_tx",
-      "category": "transaction",
-      "allowed_parents": ["txq.accept"],
-      "required_attributes": [
-        "tx_hash",
-        "ter_code",
-        "retries_remaining",
-        "txq_status"
-      ],
-      "config_flag": "trace_transactions",
-      "optional": true
-    },
-    {
-      "name": "txq.cleanup",
-      "category": "transaction",
-      "allowed_parents": ["consensus.accept.apply", "ROOT"],
-      "required_attributes": ["ledger_seq", "expired_count"],
-      "config_flag": "trace_transactions",
-      "optional": true,
-      "note": "TxQ::processClosedLedger (TxQ.cpp:1403). The same two call paths as txq.accept, with the same split: the consensus path (RCLConsensus.cpp:950) runs inside the consensus.accept.apply scope and nests under it, while the switchLastClosedLedger jump path (NetworkOPs.cpp:2121) has no consensus span open and leaves it a root."
-    },
-    {
-      "name": "consensus.round",
-      "category": "consensus",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": [
-        "consensus_ledger_id",
-        "ledger_seq",
-        "consensus_mode",
-        "consensus_round_id",
-        "consensus_phase"
-      ],
-      "config_flag": "trace_consensus",
-      "note": "Root consensus span created per round. Also carries trace_strategy, previous_ledger_seq, previous_proposers, previous_round_time_ms. Emits seven span EVENTS that this manifest cannot assert: phase.open, phase.recovery, phase.establish, phase.accepted, outcome.yes, outcome.moved_on, outcome.expired (declared ConsensusSpanNames.h:265-277; emitted RCLConsensus.cpp:1344 and via onPhaseEvent/onOutcomeEvent from Consensus.h:764, 793, 1047, 1517-1525, 1530, 1566), plus a `view.change` event emitted from RCLConsensus::Adaptor::getPrevLedger when the network's preferred ledger differs from this node's (carries prev_ledger_prefix and net_ledger_prefix). validate_telemetry.py reads only span name, attributes and start/end timestamps from the Tempo OTLP payload — it has no event assertion support — so adding an \"events\" key here would be silently ignored. Recorded as a note instead; asserting events needs validator support first."
-    },
-    {
-      "name": "consensus.phase.open",
-      "category": "consensus",
-      "allowed_parents": ["consensus.round"],
-      "required_attributes": [],
-      "config_flag": "trace_consensus"
-    },
-    {
-      "name": "consensus.proposal.send",
-      "category": "consensus",
-      "allowed_parents": ["consensus.round"],
-      "required_attributes": ["consensus_round"],
-      "config_flag": "trace_consensus",
-      "note": "Also carries is_bow_out."
-    },
-    {
-      "name": "consensus.ledger_close",
-      "category": "consensus",
-      "allowed_parents": ["consensus.round"],
-      "required_attributes": ["ledger_seq", "consensus_mode"],
-      "config_flag": "trace_consensus",
-      "note": "Also carries tx_count_open, close_time_resolution_ms."
-    },
-    {
-      "name": "consensus.establish",
-      "category": "consensus",
-      "allowed_parents": ["consensus.round"],
-      "required_attributes": [
-        "converge_percent",
-        "establish_count",
-        "proposers",
-        "disputes_count"
-      ],
-      "config_flag": "trace_consensus"
-    },
-    {
-      "name": "consensus.update_positions",
-      "category": "consensus",
-      "allowed_parents": ["consensus.establish"],
-      "required_attributes": [
-        "converge_percent",
-        "proposers",
-        "disputes_count"
-      ],
-      "config_flag": "trace_consensus",
-      "note": "childSpan of establishSpanContext_ (Consensus.h:1628), so the parent is consensus.establish — not consensus.round. Also emits a dispute.resolve span EVENT per resolved dispute (Consensus.h:1697-1698), which validate_telemetry.py cannot assert (no event support)."
-    },
-    {
-      "name": "consensus.check",
-      "category": "consensus",
-      "allowed_parents": ["consensus.establish"],
-      "required_attributes": [
-        "agree_count",
-        "disagree_count",
-        "threshold_percent",
-        "consensus_result"
-      ],
-      "config_flag": "trace_consensus",
-      "note": "childSpan of establishSpanContext_ (Consensus.h:1837), so the parent is consensus.establish — not consensus.round."
-    },
-    {
-      "name": "consensus.accept",
-      "category": "consensus",
-      "allowed_parents": ["consensus.round"],
-      "required_attributes": ["proposers", "round_time_ms", "quorum"],
-      "config_flag": "trace_consensus"
-    },
-    {
-      "name": "consensus.accept.apply",
-      "category": "consensus",
-      "allowed_parents": ["consensus.accept"],
-      "required_attributes": [
-        "ledger_seq",
-        "close_time_ripple_epoch_s",
-        "parent_close_time_ripple_epoch_s",
-        "close_time_self_ripple_epoch_s",
-        "close_time_vote_bins",
-        "resolution_direction"
-      ],
-      "config_flag": "trace_consensus",
-      "note": "Also carries close_time_correct, close_resolution_ms, consensus_state, proposing, round_time_ms, tx_count. Emits a tx.included span EVENT per transaction in the accepted set (RCLConsensus.cpp:720, with a tx_id attribute), which validate_telemetry.py cannot assert (no event support). One parent: a scoped childSpan of the captured accept context (RCLConsensus.cpp doAccept). The context is valid whenever the accept span is live; when it is not, the factory returns a null guard and no span is emitted at all. Scoped, so the txq spans doAccept creates after it are its children."
-    },
-    {
-      "name": "consensus.validation.send",
-      "category": "consensus",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": [
-        "ledger_seq",
-        "proposing",
-        "ledger_hash",
-        "full_validation"
-      ],
-      "config_flag": "trace_consensus",
-      "note": "follows-from consensus.accept. ledger_hash and full_validation are shared with peer.validation.receive (same keys, told apart by span name). Also carries validation_sign_time."
-    },
-    {
-      "name": "consensus.proposal.receive",
-      "category": "consensus",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": [],
-      "config_flag": "trace_consensus",
-      "note": "Context-propagated from the sending peer. No required local attributes."
-    },
-    {
-      "name": "consensus.validation.receive",
-      "category": "consensus",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": [],
-      "config_flag": "trace_consensus",
-      "note": "Context-propagated from the sending peer. No required local attributes."
-    },
-    {
-      "name": "consensus.validation.accept",
-      "category": "consensus",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": [
-        "ledger_hash",
-        "ledger_seq",
-        "validation_status",
-        "accept_gated",
-        "full_validation"
-      ],
-      "config_flag": "trace_ledger",
-      "note": "Emitted once per TRUSTED validation as it reaches the ledger-acceptance gate (handleNewValidation), so its rate is bounded by the UNL size per ledger close. Required, not optional: every node in the harness cluster validates and every other node receives those validations, so the span fires continuously. config_flag is trace_ledger, not trace_consensus, because the span is created through the ledger-hash trace join (TraceCategory::Ledger) -- it belongs to the ledger's trace, not the round's. Its trace id is derived from the VALIDATED ledger hash, the same key ledger.validate and ledger.store use, which is what makes one slow ledger read as one connected trace across the three threads that produce those spans. validation_status is one of the 6 ValStatus values and accept_gated says whether the validation actually reached the gate; both are spanmetrics dimensions. See trace_join_groups below."
-    },
-    {
-      "name": "consensus.mode_change",
-      "category": "consensus",
-      "allowed_parents": ["consensus.round"],
-      "required_attributes": ["mode_old", "mode_new"],
-      "config_flag": "trace_consensus",
-      "optional": true,
-      "note": "childSpan of roundSpanContext_ (RCLConsensus.cpp:1101), so the parent is consensus.round. Only fires on an operating-mode transition; a steady cluster rarely changes mode after warmup. Note the handler itself is called at the start of EVERY round, with the before and after mode equal on almost all of them, so the guard on before != after is the whole reason this span is conditional rather than per-round — without it the span records no change and span.mode_change.records_a_real_change fails. A mode change outside a round leaves roundSpanContext_ invalid, which yields a null (no-op) guard rather than a root span."
-    },
-    {
-      "name": "ledger.build",
-      "category": "ledger",
-      "allowed_parents": ["consensus.accept.apply", "ROOT"],
-      "required_attributes": [
-        "ledger_seq",
-        "close_time_ripple_epoch_s",
-        "close_time_correct",
-        "close_resolution_ms"
-      ],
-      "config_flag": "trace_ledger",
-      "note": "tx_count/tx_failed live on the child tx.apply span, not here. Two call paths give it two parents. The consensus path (RCLConsensus.cpp:983-985, inside doAccept) builds the ledger inside the consensus.accept.apply scope, so the span nests under it. The LedgerDeltaAcquire replay path (LedgerDeltaAcquire.cpp:208) rebuilds a historical ledger with no consensus span open on that thread, so the span is a root there."
-    },
-    {
-      "name": "ledger.validate",
-      "category": "ledger",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["ledger_hash", "ledger_seq", "validations"],
-      "config_flag": "trace_ledger",
-      "note": "Created in LedgerMaster::checkAccept (LedgerMaster.cpp:1002-1003) with the ambient-inheriting span() factory, so its parent is whatever scope is open on the calling thread. The consensus path reaches it inside doAccept — consensusBuilt (RCLConsensus.cpp:803) calls checkAccept at LedgerMaster.cpp:1149, and the node's own validation takes the same route through handleNewValidation (RCLConsensus.cpp:1115) — so there it nests under the consensus span. ROOT covers the two paths with no ambient scope: an inbound peer validation, which PeerImp defers to a JtValidationT/JtValidationUt job (PeerImp.cpp:2751-2757) carrying only a never-activated SpanGuard handle, and the InboundLedger completion path (InboundLedger.cpp:168, :583). allowed_parents is ROOT alone on this branch: the span is a hashSpan (a true root on the ledger-hash trace), so it never inherits the consensus.accept.apply scope the upstream branches document."
-    },
-    {
-      "name": "ledger.store",
-      "category": "ledger",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["ledger_hash", "ledger_seq"],
-      "config_flag": "trace_ledger",
-      "note": "Created in LedgerMaster::storeLedger (LedgerMaster.cpp:470) with the ambient-inheriting span() factory. The consensus path reaches it from RCLConsensus.cpp:1001, inside doAccept, so there it nests under the consensus span. ROOT covers every other caller, none of which has an ambient scope: InboundLedger.cpp:164 and :574, LedgerDeltaAcquire.cpp:243, and the genesis/startup stores at Application.cpp:2028 and :2037. allowed_parents is ROOT alone on this branch: the span is a hashSpan (a true root on the ledger-hash trace), so it never inherits the consensus.accept.apply scope the upstream branches document."
-    },
-    {
-      "name": "ledger.acquire",
-      "category": "ledger",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": [
-        "ledger_seq",
-        "acquire_reason",
-        "timeouts",
-        "outcome",
-        "ledger_hash"
-      ],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "Only fires when a node must fetch a missing ledger (InboundLedger). A healthy local cluster rarely back-fills history. outcome is one of complete|failed|abandoned and is stamped on every exit path, including the sweep/shutdown path where the fetch never finished (abandoned). ledger_hash identifies the target ledger from the first moment, since a by-hash acquire starts with ledger_seq 0. peer_count is not required: the destructor path deliberately skips the peer lookup to avoid taking the Overlay lock under the InboundLedgers collection lock. allowed_parents is ROOT alone on this branch: the span is a hashSpan (a true root on the ledger-hash trace), so it never inherits the consensus.accept.apply scope the upstream branches document."
-    },
-    {
-      "name": "ledger.acquire.header",
-      "category": "ledger",
-      "allowed_parents": ["ledger.acquire"],
-      "required_attributes": ["ledger_hash", "outcome", "timed_out"],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "Child of ledger.acquire, so it is optional for the same reason the parent is: it exists only while a node is fetching a missing ledger, and a healthy 5-node cluster that has agreed from genesis rarely back-fills. Covers the wait for the ledger header, which gates both tree phases -- until it arrives the account-state and transaction root hashes are unknown, so nothing else can be requested. missing_nodes is deliberately absent: a header is a single object, not a tree."
-    },
-    {
-      "name": "ledger.acquire.astree",
-      "category": "ledger",
-      "allowed_parents": ["ledger.acquire"],
-      "required_attributes": [
-        "ledger_hash",
-        "outcome",
-        "timed_out",
-        "missing_nodes"
-      ],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "Child of ledger.acquire (optional for the same reason). The account-state SHAMap phase, which is nearly all of the work in a real fresh sync -- the flat parent span could not separate it from the small transaction tree. outcome=timeout with a non-zero missing_nodes is the 'peers are not serving this tree' signature. missing_nodes is read from the count getMissingNodes() already produced during its sweep; no extra tree walk."
-    },
-    {
-      "name": "ledger.acquire.txtree",
-      "category": "ledger",
-      "allowed_parents": ["ledger.acquire"],
-      "required_attributes": [
-        "ledger_hash",
-        "outcome",
-        "timed_out",
-        "missing_nodes"
-      ],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "Child of ledger.acquire (optional for the same reason). The transaction SHAMap phase. Usually completes long before the astree phase, and that asymmetry is the point of splitting them: the parent span's duration is the state tree's, not this one's."
-    },
-    {
-      "name": "ledger.serve",
-      "category": "ledger",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": [
-        "object_type",
-        "peer_id",
-        "served_nodes",
-        "outcome"
-      ],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "The supply side: this node answering a peer's TMGetLedger request on the JtLedgerReq worker. In practice every node in the harness cluster is listed in the others' [ips], so they exchange ledger and tx-set requests continuously throughout the run -- but optional even so, because the supply side cannot be required while every demand-side span is optional. A TMGetLedger request is constructed in exactly two places, InboundLedger.cpp and TransactionAcquire.cpp, and their ledger.acquire and txset.acquire spans are both optional; requiring this one would rest a mandatory check on an optional cause. Promote it together with them if a future harness step forces a real fetch. A fresh trace root -- the request arrives from the wire on a shared worker, so it must not inherit an unrelated span active there. object_type is header|tx|as|txset and outcome is complete|partial|refused, both derived by shared rules in LedgerSpanNames.h so the eight exits of processLedgerRequest cannot disagree. ledger_seq is present only once getLedger() succeeded, so it is not required."
-    },
-    {
-      "name": "txset.acquire",
-      "category": "ledger",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": [
-        "txset_hash",
-        "outcome",
-        "timeouts",
-        "duration_ms",
-        "peer_count"
-      ],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "One attempt to fetch a transaction set a consensus proposal referenced (TransactionAcquire, whose only telemetry is this span). Optional because it only fires when a node does NOT already hold a proposed set: in the harness cluster every node sees the same relayed transactions and builds the same set locally, so InboundTransactions::getSet finds it in its map and never constructs a TransactionAcquire. It is the sibling of ledger.acquire -- same TimeoutCounter base, same trigger/onTimer/takeNodes shape -- and shares the trace_ledger flag so the two halves of a stuck sync cannot be enabled apart. outcome is complete|failed|timeout|abandoned, stamped on all three exits (done(); abandonAcquireSpan(), when InboundTransactions stops pursuing the fetch -- giveSet, the newRound sweep, stop, or ~InboundTransactionsImp destroying the container; and cancel(), when the TimeoutCounter base marks the task failed without reaching done()); the destructor only asserts one of them already ran. A fetch dropped in the window before init() runs emits NO span at all rather than an abandoned one, so the outcome set describes the spans that exist, not every TransactionAcquire ever constructed. The requesting round(s) are carried by repeated round.request EVENTS (current_ledger_hash + current_ledger_seq), not by a parent or a link: one round starts many fetches and one fetch is wanted by many rounds, so no round owns the span -- which is why there is deliberately no parent_child_relationships entry for it. Events are not asserted by this validator; they are read in Tempo."
-    },
-    {
-      "name": "peer.dial",
-      "category": "peer",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["remote_endpoint"],
-      "config_flag": "trace_peer",
-      "note": "One outbound connect attempt (ConnectAttempt), a fresh trace root because a dial is the first thing a starting node does and there is nothing to parent it to. Required: run-full-validation.sh lists the other four nodes in each node's [ips], so every node dials and the span always fires. Telemetry is live in time to catch it -- ApplicationImp::setup() calls startTelemetry() before start() calls overlay_->start(). outcome carries the same six values as the overlay_connect_total counter (connected|tcp_fail|tls_fail|self_connection|upgrade_fail|timeout) and is set from the same reportOutcome() funnel, so span and counter cannot disagree. remote_endpoint is the span-only dimension the counter cannot carry, since one series per peer address would be unbounded cardinality. Among its attributes only remote_endpoint is required. outcome and duration_ms are set on every terminal path, both inside reportOutcome(), but NOT on the teardown path: an attempt destroyed during overlay shutdown, or one whose connect was aborted, ends its span in ~ConnectAttempt with neither, deliberately -- the destructor's own comment records that a span ending with no outcome is the honest record of a dial that never concluded. Because peer.dial is a freshRoot, each dial is its own trace holding exactly one instance of the span, and the validator inspects only the most recent trace; a single newly-aborted dial would therefore fail CI on healthy behaviour. That single-trace sampling of a one-instance span is what makes requiring them unsafe -- not any doubt about the terminal paths setting them."
-    },
-    {
-      "name": "peer.proposal.receive",
-      "category": "peer",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["peer_id"],
-      "config_flag": "trace_peer",
-      "note": "peer_id is set immediately after the freshRoot (PeerImp.cpp:1925) and is the only unconditional attribute. proposal_trusted is set at PeerImp.cpp:1953, after several early returns (stale/duplicate/self-originated proposal checks), so a single rejected proposal in the sampled trace would fail the check — it is therefore not required."
-    },
-    {
-      "name": "peer.validation.receive",
-      "category": "peer",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["peer_id", "ledger_hash", "full_validation"],
-      "config_flag": "trace_peer",
-      "note": "ledger_hash and full_validation are shared with consensus.validation.send (same keys, told apart by span name). Both are set at PeerImp.cpp:2573-2574, BEFORE the isCurrent() gate, so only a too-small or unparseable validation skips them — they stay required (and validate_telemetry.py's PARITY_SPAN_ATTRS already asserts them independently). validation_trusted is set at PeerImp.cpp:2591, after the isCurrent() early return at :2576-2584, so a single not-current validation in the sampled trace would fail the check — it is therefore not required."
-    },
-    {
-      "name": "pathfind.request",
-      "category": "pathfind",
-      "allowed_parents": ["rpc.command.*"],
-      "required_attributes": [
-        "pathfind_source_account",
-        "pathfind_dest_account"
-      ],
-      "config_flag": "trace_rpc",
-      "optional": true,
-      "note": "Fires on ripple_path_find / path_find RPC. Optional because the harness issues neither: rpc_load_generator.py's DEFAULT_WEIGHTS carries no ripple_path_find entry, and no workload-profiles.json phase names a path-finding command in a weights override, so no such RPC reaches a node at all. Created as an ambient (scoped) child inside the RPC command handler (RipplePathFind.cpp:35-36, PathFind.cpp:26-27), so its parent is the enclosing rpc.command.* span — RipplePathFind.cpp:30 states this explicitly. Note the span does NOT need pathfinding to be ENABLED, only the RPC to be issued: the ScopedSpanGuard is constructed at RipplePathFind.cpp:35, above the 'if (pathSearchMax == 0) return rpcError(RpcNotSupported)' guard at :48-49, so even a refused call opens and closes it. That positional accident means the load alone would satisfy this entry while pathfind.compute and pathfind.discover below stayed unasserted, and it is why such refusals would also drive a steady ~3% STATUS_CODE_ERROR floor in span_calls_total. Adding the load alone would make this span required and introduce that error floor; covering the whole family needs pathfinding actually enabled (a [path_search_max] override in run-full-validation.sh — see the pathfind.compute entry below). The workload README section 'Pathfinding is not exercised' carries the full recipe for enabling it."
-    },
-    {
-      "name": "pathfind.compute",
-      "category": "pathfind",
-      "allowed_parents": ["pathfind.request", "pathfind.update_all"],
-      "required_attributes": ["pathfind_fast"],
-      "config_flag": "trace_rpc",
-      "optional": true,
-      "note": "Created by PathRequest::doUpdate (PathRequest.cpp:749-750), which the harness never reaches: pathfinding is disabled on every harness node, so no PathRequest is ever constructed. Config.cpp:725-726 sets pathSearchMax to 0 when a [validation_seed] or [validator_token] section is present, run-full-validation.sh writes [validation_seed] for every node and has no [path_search*] override, and doRipplePathFind then returns RpcNotSupported at RipplePathFind.cpp:59-60 — above the request-construction branches and below the pathfind.request span guard at :35. Liquidity is not the reason and never was: the call is refused before any path search is attempted, so the outcome does not depend on what the ledger holds. There is a second, independent reason: the harness sends no path-finding RPC at all, since rpc_load_generator.py carries no ripple_path_find weight. Enabling this span therefore needs BOTH a [path_search_max] override (or a non-validator node) in run-full-validation.sh AND the load restored — see the workload README section 'Pathfinding is not exercised'. Two parents: doUpdate's plain ScopedSpanGuard (PathRequest.cpp:788-789) inherits the ambient scope, and doUpdate has callers under both of them — under pathfind.request via PathRequest.cpp:272 and PathRequestManager.cpp:334, and under pathfind.update_all via PathRequestManager.cpp:161 and :176, where updateAll's own scoped guard is open."
-    },
-    {
-      "name": "pathfind.discover",
-      "category": "pathfind",
-      "allowed_parents": ["pathfind.compute"],
-      "required_attributes": ["pathfind_search_level", "pathfind_num_paths"],
-      "config_flag": "trace_rpc",
-      "optional": true,
-      "note": "Graph exploration; only fires under pathfind.compute, which the harness never reaches because pathfinding is disabled on every node (see the pathfind.compute entry above for the config chain). Liquidity is not the reason: the RPC is refused before any search runs."
-    },
-    {
-      "name": "pathfind.update_all",
-      "category": "pathfind",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["pathfind_ledger_index", "pathfind_num_requests"],
-      "config_flag": "trace_rpc",
-      "optional": true,
-      "note": "Async recomputation at ledger close. PathRequestManager::updateAll emits the span only when requests_ is non-empty (PathRequestManager.cpp:88-95), so it needs a live path_find subscription. On the harness requests_ can never become non-empty at all: the only two insertPathRequest call sites are makePathRequest (:268) and makeLegacyPathRequest (:296), and both handlers return RpcNotSupported first because pathfinding is disabled on every node (PathFind.cpp:50-51, RipplePathFind.cpp:59; see the pathfind.compute entry above for the config chain). ROOT despite the ambient-inheriting ScopedSpanGuard, because the one caller is LedgerMaster's advance thread (LedgerMaster.cpp:1491) with no scope open. An observed parent is an ambient leak into that thread, not a lawful edge."
-    },
-    {
-      "name": "grpc.*",
-      "category": "grpc",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["method", "grpc_role", "grpc_status"],
-      "config_flag": "trace_rpc",
-      "optional": true,
-      "note": "Wildcard — grpc.<MethodName>. The harness has no gRPC client, so these do not fire. Tracked for completeness. ROOT despite the ambient-inheriting span() factory (GRPCServer.cpp:174), because the span is created at the inbound boundary on a gRPC handler thread that runs no other instrumented work. An observed parent is an ambient leak onto that thread, not a lawful edge."
-    },
-    {
-      "name": "nodestore.rotate",
-      "category": "ledger",
-      "allowed_parents": ["ROOT"],
-      "required_attributes": ["ledger_seq", "last_rotated", "outcome"],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "One per online-delete rotation (SHAMapStoreImp::run). Optional: the harness cluster has no online_delete interval it can reach in a run, so no node rotates. A fresh trace root on the SHAMapStore thread; every nodestore.rotate.* span is its child through the thread's ambient scope. outcome is complete|expired|stopping|missing_node, stamped by a RotationOutcome finalizer on every exit of the rotation block. The root closes when the rotation ends (13-17 min on mainnet), so it reaches Tempo after its children."
-    },
-    {
-      "name": "nodestore.rotate.clear_prior",
-      "category": "ledger",
-      "allowed_parents": ["nodestore.rotate"],
-      "required_attributes": [],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "SQL deletion of ledgers below lastRotated."
-    },
-    {
-      "name": "nodestore.rotate.copy",
-      "category": "ledger",
-      "allowed_parents": ["nodestore.rotate"],
-      "required_attributes": ["node_count", "nodes_copied"],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "The whole-state-map walk; ten minutes or more on a populated node. nodes_copied is how many of the walked nodes were found only in the archive and had to be written to the writable backend (DatabaseRotating::duplicateCopyForwardTotal delta across the walk)."
-    },
-    {
-      "name": "nodestore.rotate.freshen.fetch",
-      "category": "ledger",
-      "allowed_parents": ["nodestore.rotate"],
-      "required_attributes": ["key_count", "cache", "keys_copied"],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "The per-key re-fetch loop. Keys are copied one cache partition at a time (TaggedCache::forEachKeyPartition) so the tree-node cache mutex is never held across the whole cache; the per-partition hold shows up on cache_metrics{metric=\"treenode_lock_hold_peak_us\"}. A separate freshen.keys child once covered a single getKeys() hold that lasted seconds on a large cache and dropped the node out of sync on every rotation. Two instances per rotation, told apart by cache (treenode | master_tx). keys_copied is the freshen's yield: keys that were only in the archive (duplicateCopyForwardTotal delta)."
-    },
-    {
-      "name": "nodestore.rotate.new_backend",
-      "category": "ledger",
-      "allowed_parents": ["nodestore.rotate"],
-      "required_attributes": [],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "Creating the new NuDB/RocksDB backend."
-    },
-    {
-      "name": "nodestore.rotate.clear_caches",
-      "category": "ledger",
-      "allowed_parents": ["nodestore.rotate"],
-      "required_attributes": [],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "LedgerMaster::clearLedgerCachePrior + FullBelowCache::clear."
-    },
-    {
-      "name": "nodestore.rotate.swap",
-      "category": "ledger",
-      "allowed_parents": ["nodestore.rotate"],
-      "required_attributes": ["copy_forwards"],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "DatabaseRotatingImp::rotate, including the second clearCaches inside its callback. copy_forwards is DatabaseRotating::copyForwardTotal() at swap time."
-    },
-    {
-      "name": "nodestore.rotate.health_wait",
-      "category": "ledger",
-      "allowed_parents": [
-        "nodestore.rotate",
-        "nodestore.rotate.clear_prior",
-        "nodestore.rotate.copy",
-        "nodestore.rotate.freshen.fetch",
-        "nodestore.rotate.new_backend",
-        "nodestore.rotate.clear_caches",
-        "nodestore.rotate.swap"
-      ],
-      "required_attributes": ["server_mode", "missing_ledgers"],
-      "config_flag": "trace_ledger",
-      "optional": true,
-      "note": "Emitted only when healthWait() actually waits inside a running rotation; may repeat if the loop iterates. The pre-rotation healthWait() at the top of run() does not open this span."
-    }
-  ],
-  "parent_child_relationships": [
-    {
-      "parent": "rpc.ws_message",
-      "child": "rpc.process",
-      "description": "WebSocket message contains processing span",
-      "skip": true,
-      "skip_reason": "This relationship does not exist in the code: rpc.process is created only in ServerHandler::processRequest() (ServerHandler.cpp:718), reached only from processSession(Session, coro) (ServerHandler.cpp:646) — the HTTP/JSON-RPC path. The WebSocket path (processSession(WSSession, coro, jv), ServerHandler.cpp:467) never calls processRequest, so rpc.process is never emitted at all under the WebSocket-only harness. The earlier diagnosis (cross-thread context loss needing a C++ fix) was wrong: rpc.ws_message is a deliberate freshRoot (ServerHandler.cpp:473-474) so each WS message is its own trace rather than nesting under a span leaked on a reused coroutine worker. Nothing to fix."
-    },
-    {
-      "parent": "rpc.ws_message",
-      "child": "rpc.command.*",
-      "description": "WebSocket message contains the per-command span — the real relationship on the harness WS path (rpc::doCommand at RPCHandler.cpp:271 creates an ambient child of the rpc.ws_message scope inside the same coroutine). Not skipped, because the validator globs the wildcard child: _span_name_matches() matches via fnmatch.fnmatchcase, so any rpc.command.<anything> under the parent satisfies the contract and the command mix does not matter. A validator that instead collapsed the wildcard to one literal name -- child_name.replace(\"*\", \"server_info\") -- would make this check depend on which command the sampled traces happened to carry: server_info is 25/100 of rpc_load_generator.py's DEFAULT_WEIGHTS, so a healthy run could sample three non-server_info traces and fail."
-    },
-    {
-      "parent": "rpc.process",
-      "child": "rpc.command.*",
-      "description": "Processing span contains the per-command span, on the HTTP/JSON-RPC path. Not skipped, for the same reason as the WebSocket pair above: the validator globs a wildcard child via _span_name_matches(), so no single literal command has to be sampled. Both ends emit: rpc.process reports 5 traces on a normal run, not from the WebSocket-only load generator but because run-full-validation.sh polls each node's HTTP port with curl for readiness and validated-ledger progress (:449, :502), and every such request runs a command."
-    },
-    {
-      "parent": "rpc.startup",
-      "child": "rpc.command.*",
-      "description": "The startup batch contains the per-command spans it runs. Asserted rather than skipped: the parent is a literal name, so the Tempo query that selects candidate traces resolves, and the wildcard child is globbed by _span_name_matches as it is for the two transport parents."
-    },
-    {
-      "parent": "ledger.build",
-      "child": "tx.apply",
-      "description": "Ledger build contains transaction application"
-    },
-    {
-      "parent": "consensus.round",
-      "child": "consensus.accept",
-      "description": "Consensus round contains the accept sub-span"
-    },
-    {
-      "parent": "consensus.accept",
-      "child": "consensus.accept.apply",
-      "description": "Accept contains the ledger-apply sub-span"
-    },
-    {
-      "parent": "pathfind.request",
-      "child": "pathfind.compute",
-      "description": "Pathfind request contains the compute sub-span",
-      "skip": true,
-      "skip_reason": "Real relationship (pathfind.compute is created inside PathRequest::doUpdate at PathRequest.cpp:749-750, under the pathfind.request scope), but the child never exists on the harness because pathfinding is disabled on every node: Config.cpp:725-726 zeroes pathSearchMax whenever a [validation_seed] or [validator_token] section is present, run-full-validation.sh writes [validation_seed] for all five nodes with no [path_search*] override, and doRipplePathFind returns RpcNotSupported at RipplePathFind.cpp:59-60 before constructing a PathRequest. The parent would appear anyway if the RPC were issued, because its ScopedSpanGuard is created at RipplePathFind.cpp:35, above that guard; but rpc_load_generator.py carries no ripple_path_find weight, so not even the parent appears and both ends of this relationship are absent. Liquidity has nothing to do with it — the earlier 'no liquidity, returns before computing' reason was wrong, because no path search is attempted at all. Asserting this relationship needs the load restored AND a [path_search_max] override (or a non-validator node) in run-full-validation.sh."
-    },
-    {
-      "parent": "ledger.acquire",
-      "child": "ledger.acquire.astree",
-      "description": "Ledger acquire contains the account-state tree fetch phase. Un-skipped 2026-08-26. The old skip reason said the parent is optional because a healthy 5-node cluster agreeing from genesis rarely back-fills, so the check would fail on a parent with no traces. Run 32969481032 refutes the premise: ledger.acquire reported 5 traces and so did each of its three phase children, one per node. The parenting was never the doubtful part -- beginPhaseSpan() parents through the acquire span's own captured SpanContext rather than the ambient thread context, so it holds on whichever worker opens a phase."
-    },
-    {
-      "parent": "ledger.acquire",
-      "child": "ledger.acquire.header",
-      "description": "Ledger acquire contains the header fetch phase, which gates the two tree fetches. Added 2026-08-26: all three phases are opened identically through the parent's captured SpanContext and all three emitted 5 traces on run 32969481032, but only astree had ever been declared here, so two thirds of the phase hierarchy went unchecked."
-    },
-    {
-      "parent": "ledger.acquire",
-      "child": "ledger.acquire.txtree",
-      "description": "Ledger acquire contains the transaction-tree fetch phase. Un-skipped once the hierarchy check stopped sampling only the newest parent traces. This phase is the conditional one of the three: header opens on !haveHeader_ (InboundLedger.cpp:672) and astree in the else of haveState_ (:689), both of which hold on essentially every acquire, while txtree opens only in the else of haveTransactions_ (:698) -- and a node in this cluster usually already holds the transaction set, because every node sees the same relayed transactions and builds the same set. So the phase fires on a minority of acquires, which is precisely the case newest-N sampling of the parent got wrong; the child was always emitting, just not in the three most recent acquires."
-    },
-    {
-      "parent": "rpc.command.*",
-      "child": "pathfind.request",
-      "description": "The RPC command span contains the path-finding request span.",
-      "skip": true,
-      "skip_reason": "Real relationship, and the one skip here caused by a WILDCARD PARENT rather than by a missing span. _validate_parent_child builds its Tempo query as name=\"<parent>\" with the contract string inserted literally (validate_telemetry.py:801), so a parent of rpc.command.* searches for a span literally named that and finds nothing. Note the asymmetry: the ancestry check globs both sides through _span_name_matches, which is why rpc.ws_message -> rpc.command.* is asserted, but the Tempo query that selects the candidate traces is literal on the parent, so no trace is ever fetched to run it on. Asserting this needs the parent query to accept a glob -- a TraceQL name=~ regex, or resolving the glob to the concrete names Tempo reports first. Independently of that, both ends are absent today anyway: the harness issues no path-finding RPC, see the pathfind.compute entry above.",
-      "added": "Closes the declared-but-unlisted gap"
-    },
-    {
-      "parent": "pathfind.compute",
-      "child": "pathfind.discover",
-      "description": "The path computation contains the discovery pass.",
-      "skip": true,
-      "skip_reason": "Real relationship with BOTH ends absent, for the reason given in full on the pathfind.compute entry above: pathfinding is disabled on every harness node because Config.cpp:725-726 zeroes pathSearchMax whenever a [validation_seed] section is present, run-full-validation.sh writes one for all five nodes with no [path_search_max] override, so doRipplePathFind returns RpcNotSupported before any PathRequest is constructed -- and the harness sends no path-finding RPC at all either. Asserting this needs both blockers lifted, which is a workload and node-config change rather than a harness one. Listed here so that the pathfinding family is fully accounted for rather than partly silent.",
-      "added": "Closes the declared-but-unlisted gap"
-    },
-    {
-      "parent": "rpc.http_request",
-      "child": "rpc.process",
-      "description": "The HTTP request span contains the request-processing span. ServerHandler::processRequest() opens rpc.process as a scoped child and its own comment names rpc.http_request as the parent; processRequest has exactly one caller. Both ends emit on every run: the harness polls each node's HTTP port with curl, so this is the one rpc hierarchy that is assertable without a wildcard."
-    },
-    {
-      "parent": "tx.process",
-      "child": "txq.enqueue",
-      "description": "Transaction processing contains the queue-admission span."
-    },
-    {
-      "parent": "txq.enqueue",
-      "child": "txq.apply_direct",
-      "description": "Queue admission contains the direct-apply path taken when the transaction is applied straight to the open ledger instead of being queued."
-    },
-    {
-      "parent": "txq.enqueue",
-      "child": "txq.batch_clear",
-      "description": "Queue admission contains the batch-clear pass that drops an account's superseded queued transactions.",
-      "skip": true,
-      "skip_reason": "Real relationship, skipped for one reason: the child never fires at all under this workload -- the run reports \"span.txq.batch_clear: optional span not emitted under this workload\" -- because it is created in TxQ::tryClearAccountQueueUpThruTx (TxQ.cpp:550), which needs one account holding several queued transactions AND an arriving transaction that supersedes the whole batch. Nothing in txq-burst arranges that shape. Sampling is not a second reason here: the hierarchy check queries Tempo for traces containing both parent and child, so it would find the pair wherever it occurred. This is purely a workload gap, and un-skipping it needs the workload to produce a supersedable batch -- nothing further from the validator."
-    },
-    {
-      "parent": "txq.accept",
-      "child": "txq.accept_tx",
-      "description": "The queue's accept pass contains the per-transaction accept span. Not skipped, because the hierarchy check asks Tempo for traces containing both spans rather than sampling the newest parent traces. That matters here: the child is created inside the loop over queued transactions and behind `if (feeLevelPaid >= requiredFeeLevel)` (TxQ.cpp:1530), so it exists only for a close whose queue held a fee-clearing transaction, while the parent fires on every close (:1499) -- exactly the shape newest-N sampling gets wrong. Searching for co-occurrence finds it wherever it happened, not only in the most recent closes."
-    },
-    {
-      "parent": "consensus.round",
-      "child": "consensus.phase.open",
-      "description": "A consensus round contains its open phase."
-    },
-    {
-      "parent": "consensus.round",
-      "child": "consensus.establish",
-      "description": "A consensus round contains its establish phase."
-    },
-    {
-      "parent": "consensus.round",
-      "child": "consensus.proposal.send",
-      "description": "A consensus round contains the proposals this node sends during it."
-    },
-    {
-      "parent": "consensus.round",
-      "child": "consensus.ledger_close",
-      "description": "A consensus round contains the ledger close it performs."
-    },
-    {
-      "parent": "consensus.round",
-      "child": "consensus.mode_change",
-      "description": "A consensus round contains any mode transition that happens inside it."
-    },
-    {
-      "parent": "consensus.establish",
-      "child": "consensus.update_positions",
-      "description": "The establish phase contains each position-update pass."
-    },
-    {
-      "parent": "consensus.establish",
-      "child": "consensus.check",
-      "description": "The establish phase contains each consensus-reached check."
-    },
-    {
-      "parent": "nodestore.rotate",
-      "child": "nodestore.rotate.clear_prior",
-      "description": "phase child of the rotation root; both optional, so the check is a no-op until a harness step forces a rotation",
-      "skip": true,
-      "skip_reason": "nodestore.rotate never fires under the harness; see the span note."
-    },
-    {
-      "parent": "nodestore.rotate",
-      "child": "nodestore.rotate.copy",
-      "description": "phase child of the rotation root; both optional",
-      "skip": true,
-      "skip_reason": "nodestore.rotate never fires under the harness; see the span note."
-    },
-    {
-      "parent": "nodestore.rotate",
-      "child": "nodestore.rotate.freshen.fetch",
-      "description": "phase child of the rotation root; both optional",
-      "skip": true,
-      "skip_reason": "nodestore.rotate never fires under the harness; see the span note."
-    },
-    {
-      "parent": "nodestore.rotate",
-      "child": "nodestore.rotate.new_backend",
-      "description": "phase child of the rotation root; both optional",
-      "skip": true,
-      "skip_reason": "nodestore.rotate never fires under the harness; see the span note."
-    },
-    {
-      "parent": "nodestore.rotate",
-      "child": "nodestore.rotate.clear_caches",
-      "description": "phase child of the rotation root; both optional",
-      "skip": true,
-      "skip_reason": "nodestore.rotate never fires under the harness; see the span note."
-    },
-    {
-      "parent": "nodestore.rotate",
-      "child": "nodestore.rotate.swap",
-      "description": "phase child of the rotation root; both optional",
-      "skip": true,
-      "skip_reason": "nodestore.rotate never fires under the harness; see the span note."
-    },
-    {
-      "parent": "nodestore.rotate",
-      "child": "nodestore.rotate.health_wait",
-      "description": "phase child, emitted only when a rotation waits",
-      "skip": true,
-      "skip_reason": "nodestore.rotate never fires under the harness; see the span note."
-    },
-    {
-      "parent": "tx.process",
-      "child": "tx.receive",
-      "description": "CROSS-NODE. The submitting node's tx.process span is the parent of the tx.receive span on every peer the transaction is relayed to. NetworkOPs::apply() injects the tx.process span's own context into the relayed TMTransaction (NetworkOPs.cpp:1979, the span created at :1630 by txProcessSpan), and PeerImp extracts it into a child span. Both ends emit on every run: tx_submitter.py submits to one node and the cluster relays."
-    },
-    {
-      "parent": "consensus.proposal.send",
-      "child": "consensus.proposal.receive",
-      "description": "CROSS-NODE. RCLConsensus::Adaptor::propose() injects the send span's own context into the TMProposeSet, and the receiving peer parents consensus.proposal.receive to it (ConsensusReceiveTracing.h, the has_trace_context() branch). Asserted rather than joined on a hash, because this pair really is parent and child. Until the send side was fixed it injected the AMBIENT context, which is never set on that thread, so every receiver silently took its freshRoot fallback and this relationship held for nobody."
-    },
-    {
-      "parent": "consensus.validation.send",
-      "child": "consensus.validation.receive",
-      "description": "CROSS-NODE. The same pairing for validations, injected by RCLConsensus::Adaptor::validate(). Declared separately from the proposal pair because the two send sites are independent, and a regression in one says nothing about the other."
-    }
-  ],
-  "trace_join_groups": {
-    "description": "Groups of spans that share ONE trace id without any parent/child link between them, because each derives its trace id deterministically from the same hash (SpanGuard::hashSpan). This is how spans produced on unrelated threads are joined: no context is propagated, so there is no parent to assert -- the assertion is that the spans co-occur in a single trace. Checked by assert_trace_join_groups() in validate_telemetry.py, which searches for the anchor span and requires at least one of its traces to also contain every member listed in required_members.",
-    "groups": [
-      {
-        "name": "per_ledger",
-        "join_key": "ledger_hash",
-        "anchor": "ledger.validate",
-        "required_members": ["ledger.store"],
-        "optional_members": ["consensus.validation.accept", "ledger.acquire"],
-        "note": "One ledger's spans, keyed on its own 32-byte hash: the acceptance decision (ledger.validate, LedgerMaster::checkAccept), the persist (ledger.store), the trusted validation that drove acceptance (consensus.validation.accept) and the network fetch (ledger.acquire). Each runs on a different thread, so before the join each was its own single-span trace and a slow ledger could not be read as one unit. ledger.validate is the anchor because it fires for every validated ledger. ledger.store is required with it: both run for every ledger the node accepts, and checkAccept reaches storeLedger on the acquire path while consensus reaches it via buildLCL, so at least one ordering always produces both. consensus.validation.accept is optional here only because a ledger this node built itself is accepted via switchLCL without a peer validation arriving first, so the two need not land in the same trace on every ledger. ledger.acquire is optional for the reason its own entry gives: a healthy cluster agreeing from genesis rarely back-fills."
-      }
-    ]
-  },
-  "_conditional_attributes_note": "Five attributes documented in the 'Fresh-node sync diagnostics' table of docs/telemetry-glossary.md are deliberately absent from required_attributes above, because each is emitted only when its value is known and _validate_span_attributes_otlp() has no per-attribute optional flag -- listing one would fail CI red on a healthy run. ledger.acquire/peer_count is set only when finalizeAcquireSpan() is passed a peer count (InboundLedger.cpp), which the sweep and shutdown paths cannot supply. ledger_seq on the three ledger.acquire.header/.astree/.txtree phase spans is set only when seq_ != 0 (InboundLedger.cpp startPhaseSpan), and a by-hash acquire starts with seq_ == 0 and learns the sequence only when the header arrives -- so a phase that opens before the header legitimately carries no sequence. ledger_seq on ledger.serve is set only when the reply carries one (PeerImp.cpp), which an object-by-hash request does not. All five ARE indexed in the glossary table and rendered by the Ledger Sync Health board; the honest encoding is to document them here rather than assert a conditional attribute as required.",
-  "total_span_types": 57,
-  "total_unique_attributes": 82
-}

docker/telemetry/workload/generate-validator-keys.sh

@@ -1,179 +0,0 @@
-#!/usr/bin/env bash
-# generate-validator-keys.sh — Generate validator key pairs for the workload harness.
-#
-# Uses a temporary standalone xrpld instance to call `validation_create` RPC
-# for each node. Outputs a JSON file mapping node index to seed + public key.
-#
-# Production does this differently, and deliberately not the way this script
-# does. There, `validator-keys-tool` runs `create_keys` once to make a master
-# key that never leaves the operator's custody, then `create_token` to mint a
-# revocable `[validator_token]` for the node; rotating a node means minting a
-# new token, not moving the master key. This harness uses `validation_create`
-# instead, which puts the seed itself in `[validation_seed]` on the node.
-#
-# That is safe here only because the cluster is disposable: the keys are made on
-# the same machine that runs the nodes, into a temp workdir that the next run
-# deletes, so there is no custody boundary for the two-step split to protect.
-# The tool is also not present on this path -- it is built only with
-# `-Dvalidator_keys=ON`, which the telemetry CI job does not pass. Do not copy
-# this script's approach to a node holding a key you care about.
-#
-# Usage:
-#   ./generate-validator-keys.sh <xrpld_binary> <num_nodes> <output_dir>
-#
-# Output:
-#   <output_dir>/validator-keys.json — JSON array of {index, seed, public_key}
-#   <output_dir>/validators.txt     — [validators] section for xrpld.cfg
-
-set -euo pipefail
-
-# ---------------------------------------------------------------------------
-# Colored output helpers
-# ---------------------------------------------------------------------------
-log() { printf "\033[1;34m[KEYGEN]\033[0m %s\n" "$*"; }
-ok() { printf "\033[1;32m[KEYGEN]\033[0m %s\n" "$*"; }
-die() {
-    printf "\033[1;31m[KEYGEN]\033[0m %s\n" "$*" >&2
-    exit 1
-}
-
-# ---------------------------------------------------------------------------
-# Argument parsing
-# ---------------------------------------------------------------------------
-usage() {
-    echo "Usage: $0 <xrpld_binary> <num_nodes> <output_dir>"
-    echo ""
-    echo "Arguments:"
-    echo "  xrpld_binary  Path to xrpld binary (built with telemetry=ON)"
-    echo "  num_nodes     Number of validator key pairs to generate (1-20)"
-    echo "  output_dir    Directory to write validator-keys.json and validators.txt"
-    exit 1
-}
-
-if [ $# -lt 3 ]; then
-    usage
-fi
-
-XRPLD="$1"
-NUM_NODES="$2"
-OUTPUT_DIR="$3"
-
-# Validate arguments
-[ -x "$XRPLD" ] || die "xrpld binary not found or not executable: $XRPLD"
-[[ "$NUM_NODES" =~ ^[0-9]+$ ]] || die "num_nodes must be a positive integer"
-[ "$NUM_NODES" -ge 1 ] && [ "$NUM_NODES" -le 20 ] || die "num_nodes must be between 1 and 20"
-
-mkdir -p "$OUTPUT_DIR"
-
-# ---------------------------------------------------------------------------
-# Start a temporary standalone xrpld for key generation
-# ---------------------------------------------------------------------------
-TEMP_DIR="$(mktemp -d)"
-TEMP_PORT=5099
-TEMP_CFG="$TEMP_DIR/xrpld.cfg"
-
-# Hard ceiling on every RPC probe below. curl applies no overall timeout of its
-# own, so a node that accepts the connection and then stops answering parks the
-# poll loop for the rest of the run. The loops here count attempts, not seconds,
-# so without this their stated timeouts are not bounds at all.
-CURL_MAX_TIME="${CURL_MAX_TIME:-5}"
-
-log "Starting temporary xrpld for key generation (port $TEMP_PORT)..."
-
-cat >"$TEMP_CFG" <<EOCFG
-[server]
-port_rpc_keygen
-
-[port_rpc_keygen]
-port = $TEMP_PORT
-ip = 127.0.0.1
-admin = 127.0.0.1
-protocol = http
-
-[node_db]
-type=NuDB
-path=$TEMP_DIR/nudb
-online_delete=256
-
-[database_path]
-$TEMP_DIR/db
-
-[debug_logfile]
-$TEMP_DIR/debug.log
-
-[ssl_verify]
-0
-EOCFG
-
-"$XRPLD" --conf "$TEMP_CFG" -a --start >"$TEMP_DIR/stdout.log" 2>&1 &
-TEMP_PID=$!
-
-# Ensure cleanup on exit
-cleanup_temp() {
-    kill "$TEMP_PID" 2>/dev/null || true
-    wait "$TEMP_PID" 2>/dev/null || true
-    rm -rf "$TEMP_DIR"
-}
-trap cleanup_temp EXIT
-
-# Wait for RPC to become available
-for attempt in $(seq 1 30); do
-    if curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$TEMP_PORT" \
-        -d '{"method":"server_info"}' >/dev/null 2>&1; then
-        log "Temporary xrpld RPC ready (attempt $attempt)."
-        break
-    fi
-    if [ "$attempt" -eq 30 ]; then
-        die "Temporary xrpld RPC not ready after 30s"
-    fi
-    sleep 1
-done
-
-# ---------------------------------------------------------------------------
-# Generate key pairs
-# ---------------------------------------------------------------------------
-log "Generating $NUM_NODES validator key pairs..."
-
-KEYS_JSON="["
-VALIDATORS_TXT="[validators]"
-
-for i in $(seq 1 "$NUM_NODES"); do
-    result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$TEMP_PORT" \
-        -d '{"method":"validation_create"}')
-    seed=$(echo "$result" | jq -r '.result.validation_seed')
-    pubkey=$(echo "$result" | jq -r '.result.validation_public_key')
-
-    # Both fields must be present. jq -r prints the literal string "null" for
-    # a missing field, so an unvalidated pubkey would be written verbatim into
-    # validators.txt and xrpld would reject the file at startup.
-    if [ -z "$seed" ] || [ "$seed" = "null" ]; then
-        die "Failed to generate key pair for node $i: no validation_seed in response"
-    fi
-    if [ -z "$pubkey" ] || [ "$pubkey" = "null" ]; then
-        die "Failed to generate key pair for node $i: no validation_public_key in response"
-    fi
-
-    log "  Node $i: ${pubkey:0:20}..."
-
-    # Build JSON entry
-    entry="{\"index\": $i, \"seed\": \"$seed\", \"public_key\": \"$pubkey\"}"
-    if [ "$i" -gt 1 ]; then
-        KEYS_JSON="$KEYS_JSON,"
-    fi
-    KEYS_JSON="$KEYS_JSON$entry"
-
-    VALIDATORS_TXT="$VALIDATORS_TXT
-$pubkey"
-done
-
-KEYS_JSON="$KEYS_JSON]"
-
-# ---------------------------------------------------------------------------
-# Write output files
-# ---------------------------------------------------------------------------
-echo "$KEYS_JSON" | jq '.' >"$OUTPUT_DIR/validator-keys.json"
-echo "$VALIDATORS_TXT" >"$OUTPUT_DIR/validators.txt"
-
-ok "Generated $NUM_NODES key pairs:"
-ok "  Keys:       $OUTPUT_DIR/validator-keys.json"
-ok "  Validators: $OUTPUT_DIR/validators.txt"

docker/telemetry/workload/prom_queries.py

@@ -1,278 +0,0 @@
-#!/usr/bin/env python3
-"""Shared Prometheus query helpers for the regression gate.
-
-Single source of truth for how regression metrics are computed. Both
-``capture_timings.py`` and any future tooling consume this module so metric
-name → PromQL expression stays consistent.
-
-Design:
-- Every captured metric has a key in the form ``{category}.{name}.p{quantile}``
-  (e.g. ``span.tx.process.p99``). Keys are flat strings so JSON diffing is
-  trivial.
-- Quantile queries go through ``histogram_quantile`` over the standard
-  ``_bucket`` series. The rate window is a parameter (defaults to the
-  capture window, not Prometheus's default 5m) so short CI runs are usable.
-- The catalogue of what to capture lives in ``regression-metrics.json`` —
-  this module only knows how to translate that JSON into HTTP queries.
-
-Usage::
-
-    import asyncio, aiohttp
-    from prom_queries import build_query_plan, run_query_plan
-
-    plan = build_query_plan("regression-metrics.json", window="3m")
-    async with aiohttp.ClientSession() as s:
-        timings = await run_query_plan(s, "http://localhost:9090", plan)
-    # timings = {"span.tx.process.p99": {"value": 12.4, "unit": "ms"}, ...}
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import logging
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-
-import aiohttp
-
-logger = logging.getLogger("prom_queries")
-
-# Instant queries run in parallel, but not all at once: a single-node
-# Prometheus is easily saturated by a burst of the whole plan. With this cap
-# the worst case (every query hitting the 30 s timeout) is
-# ceil(len(plan) / 8) * 30 s rather than len(plan) * 30 s, which for a
-# ~30-entry plan is ~2 min instead of ~15 min of a 30 min CI job.
-MAX_CONCURRENT_QUERIES = 8
-
-
-@dataclass(frozen=True)
-class QueryEntry:
-    """One metric to capture from Prometheus.
-
-    Attributes:
-        key:      Flat output key, e.g. ``span.tx.process.p99``.
-        promql:   The PromQL expression to send to /api/v1/query.
-        unit:     Unit of the returned value, e.g. ``ms`` or ``us``.
-                  Baseline JSON preserves this so the comparator can
-                  sanity-check unit drift.
-    """
-
-    key: str
-    promql: str
-    unit: str
-
-
-def _build_simple_entries(
-    cfg: dict,
-    prefix: str,
-    window: str,
-    excluded: frozenset[str] = frozenset(),
-) -> list[QueryEntry]:
-    """Build QueryEntry list for a single-template category (spans, rpc).
-
-    ``excluded`` holds flat keys the surface declares but does not gate (see
-    ``excluded_keys`` in regression-metrics.json). They are dropped here rather
-    than filtered later, so a key that no longer gates is never queried and
-    never appears in ``timings.json`` — a captured key with no baseline reports
-    as "new metric (not in baseline)", which reads as coverage while being
-    unable to gate.
-    """
-    tmpl = cfg.get("_query_template", "")
-    unit = cfg.get("_unit", "ms")
-    entries: list[QueryEntry] = []
-    for name in cfg.get("names", []):
-        for q in cfg.get("_quantiles", []):
-            if f"{prefix}.{name}.p{_quantile_label(q)}" in excluded:
-                continue
-            expr = (
-                tmpl.replace("{quantile}", _format_quantile(q))
-                .replace("{name}", name)
-                .replace("{window}", window)
-            )
-            entries.append(
-                QueryEntry(
-                    key=f"{prefix}.{name}.p{_quantile_label(q)}",
-                    promql=expr,
-                    unit=unit,
-                )
-            )
-    return entries
-
-
-def _build_job_entries(
-    cfg: dict, window: str, excluded: frozenset[str] = frozenset()
-) -> list[QueryEntry]:
-    """Build QueryEntry list for the job_queue category (multi-phase).
-
-    ``excluded`` is applied for the same reason as in _build_simple_entries; the
-    key format is flat, so one exclusion list covers both categories.
-    """
-    unit = cfg.get("_unit", "us")
-    phases = cfg.get("_phases", ["queued", "running"])
-    tmpl_map = {
-        "queued": cfg.get("_queued_template", ""),
-        "running": cfg.get("_running_template", ""),
-    }
-    entries: list[QueryEntry] = []
-    for name in cfg.get("names", []):
-        for phase in phases:
-            tmpl = tmpl_map.get(phase, "")
-            if not tmpl:
-                continue
-            for q in cfg.get("_quantiles", []):
-                if f"job.{name}.{phase}.p{_quantile_label(q)}" in excluded:
-                    continue
-                expr = (
-                    tmpl.replace("{quantile}", _format_quantile(q))
-                    .replace("{name}", name)
-                    .replace("{window}", window)
-                )
-                entries.append(
-                    QueryEntry(
-                        key=f"job.{name}.{phase}.p{_quantile_label(q)}",
-                        promql=expr,
-                        unit=unit,
-                    )
-                )
-    return entries
-
-
-def build_query_plan(metrics_path: str | Path, window: str = "3m") -> list[QueryEntry]:
-    """Translate regression-metrics.json into a list of PromQL queries.
-
-    Args:
-        metrics_path: Path to ``regression-metrics.json``.
-        window:       Rate window passed to ``rate()``. For short CI runs
-                      keep this close to the test duration so the bucket
-                      counts are meaningful. Default 3m matches the
-                      ``regression`` workload profile.
-
-    Returns:
-        A list of ``QueryEntry`` values, one per (metric × quantile), less any
-        key listed in the config's ``excluded_keys`` map.
-    """
-    with open(metrics_path) as f:
-        cfg = json.load(f)
-
-    excluded = frozenset(cfg.get("excluded_keys", {}))
-    plan: list[QueryEntry] = []
-    plan.extend(_build_simple_entries(cfg.get("spans", {}), "span", window, excluded))
-    plan.extend(
-        _build_simple_entries(cfg.get("rpc_methods", {}), "rpc", window, excluded)
-    )
-    plan.extend(_build_job_entries(cfg.get("job_queue", {}), window, excluded))
-    return plan
-
-
-async def run_query_plan(
-    session: aiohttp.ClientSession,
-    prom_url: str,
-    plan: list[QueryEntry],
-) -> dict[str, dict[str, Any]]:
-    """Execute a query plan and return a flat ``key → {value, unit}`` map.
-
-    Queries that return no data (NaN, empty result) are still included in
-    the output with ``value: null`` — the comparator treats missing values
-    as "not yet observed" rather than as a regression. This keeps the
-    baseline schema stable across runs with different load levels.
-
-    Queries run concurrently, at most MAX_CONCURRENT_QUERIES at a time. Keys
-    are emitted in plan order regardless of which query answers first.
-
-    ``return_exceptions=True`` is what keeps the fan-out self-contained.
-    Without it the first escaping exception returns from ``gather`` while its
-    siblings keep running against a session the caller is about to close, so
-    the capture ends with dozens of orphaned queries and warnings from a
-    closed session. With it, every query is awaited before this returns, and
-    an unexpected failure is logged and recorded as no data -- the same
-    outcome _instant_query already produces for a query that fails.
-
-    Args:
-        session:  Shared aiohttp session.
-        prom_url: Base URL of Prometheus (e.g. ``http://localhost:9090``).
-        plan:     Output of :func:`build_query_plan`.
-
-    Returns:
-        Mapping from metric key to ``{"value": float|None, "unit": str}``.
-    """
-    gate = asyncio.Semaphore(MAX_CONCURRENT_QUERIES)
-
-    async def fetch(entry: QueryEntry) -> float | None:
-        """Run one plan entry once a query slot is free."""
-        async with gate:
-            return await _instant_query(session, prom_url, entry.promql)
-
-    results = await asyncio.gather(
-        *(fetch(entry) for entry in plan), return_exceptions=True
-    )
-
-    captured: dict[str, dict[str, Any]] = {}
-    for entry, result in zip(plan, results, strict=True):
-        value: float | None
-        if isinstance(result, BaseException):
-            logger.error(
-                "query for %s raised %s: %s",
-                entry.key,
-                type(result).__name__,
-                result,
-            )
-            value = None
-        else:
-            value = result
-        captured[entry.key] = {"value": value, "unit": entry.unit}
-    return captured
-
-
-async def _instant_query(
-    session: aiohttp.ClientSession,
-    prom_url: str,
-    promql: str,
-) -> float | None:
-    """POST an instant query to Prometheus; return the scalar value or None.
-
-    None is returned for NaN, empty results, or HTTP errors — every call
-    site treats None identically ("no data captured").
-    """
-    url = f"{prom_url.rstrip('/')}/api/v1/query"
-    try:
-        async with session.post(
-            url, data={"query": promql}, timeout=aiohttp.ClientTimeout(total=30)
-        ) as resp:
-            if resp.status != 200:
-                logger.warning("query HTTP %d: %s", resp.status, promql)
-                return None
-            body = await resp.json()
-    # JSONDecodeError covers a 200 response whose body is not JSON: without it
-    # one malformed reply aborts the whole capture, since no caller of
-    # run_query_plan wraps it.
-    except (aiohttp.ClientError, TimeoutError, json.JSONDecodeError) as exc:
-        logger.warning("query failed: %s — %s", promql, exc)
-        return None
-
-    if body.get("status") != "success":
-        logger.warning("query status=%s: %s", body.get("status"), promql)
-        return None
-
-    result = body.get("data", {}).get("result", [])
-    if not result:
-        return None
-
-    raw = result[0].get("value", [None, None])[1]
-    if raw is None or raw in ("NaN", "+Inf", "-Inf"):
-        return None
-    try:
-        return float(raw)
-    except (TypeError, ValueError):
-        return None
-
-
-def _format_quantile(q: float) -> str:
-    """Format a quantile for PromQL (``0.99`` → ``"0.99"``)."""
-    return f"{q:g}"
-
-
-def _quantile_label(q: float) -> str:
-    """Format a quantile for the output key (``0.95`` → ``"95"``)."""
-    return str(int(round(q * 100)))

docker/telemetry/workload/regression-metrics.json

@@ -1,38 +0,0 @@
-{
-  "_description": "Metric surface for the OTel-driven regression gate. Each entry names a metric, the quantiles to capture, and how to query Prometheus. The comparator compares current run against baseline-timings.json under these exact keys.",
-  "_excluded_ledger_store": "ledger.store is deliberately absent from spans.names too, for a different reason: it is below the ladder's resolution. The 2026-08-24 capture returned p50/p95/p99 of exactly 0.005/0.0095/0.0099 ms, which is 0.5/0.95/0.99 x the ladder's first edge of 0.01 ms \u2014 the signature of every sample landing in the first bucket, so the numbers are interpolation arithmetic on the bucket floor rather than latencies. That is physically plausible: LedgerMaster.cpp:470 wraps an in-memory ledgerHistory_.insert, which completes in single-digit microseconds. While all mass stays under 10 us the reported quantile cannot move materially, so NO absolute bound can gate it \u2014 every ledger.store slowing from 2 us to 9 us, 4.5x, leaves the reported value unchanged. Three keys that read as covered but cannot fire are worse than no keys (the same argument that excluded rpc.process), so they were removed rather than left in with a bound that looks derived. Restoring the key needs sub-10us edges on the collector's spanmetrics ladder (for example 0.001ms and 0.005ms) plus the matching entries in HistogramBuckets.h \u2014 that is the ladder's branch, not this file. ledger.store presence is still asserted by expected_spans.json and docker/telemetry/integration-test.sh, and its rate is still on the ledger-operations dashboard; only the latency gate drops it.",
-  "_excluded_quantiles": "A THIRD KIND OF EXCLUSION, and the only one that deleting a name cannot express. spans.names lists span NAMES while _quantiles is shared across all of them, so the declared surface is the names x quantiles product and dropping ONE quantile of ONE span needs a subtraction. excluded_keys below is that subtraction: a flat {category}.{name}.p{quantile} key, exactly as _key_format defines it, mapped to the reason it is not gated. It can only ever remove a key, never add one, so a typo cannot silently start gating something new -- and check_regression_bounds.py rule F rejects an entry that would not otherwise be declared, an entry with an empty reason, and an entry that still carries a threshold override or a baseline value, so the exclusion cannot rot into dead config. Both prom_queries.py (which builds the capture plan) and check_regression_bounds.py (rule A) subtract it, so an excluded key is not queried, never reaches timings.json, and is not expected in the baseline. NOTHING ELSE CHANGES: the quantile is still computable from Prometheus with the _query_template above, the span is still asserted by expected_spans.json, and its rate is still on the ledger-operations dashboard. Only the latency gate drops it.",
-  "_excluded_shape": "ALL SIX ENTRIES BELOW SHARE ONE SHAPE, and it is worth naming because it will recur: the observed maximum across CI runs exceeds (baseline + bound), so an ordinary run clears the trip point with nothing having regressed. Two mechanisms produce that, and both are visible here. (1) A baseline that lands in the ladder's LOW buckets gets a tiny derived bound, because the bound IS the distance to the next edge up -- span.tx.apply.p50 at 0.0060 ms sits in the first bucket (0, 0.01] and gets 0.0440 ms of headroom, against a metric that has been measured at 2.3378 ms. (2) A spread so large that no bucket of headroom could absorb it -- span.ledger.validate.p99's 66.8x range reaches 25.8750 ms against a 10 ms trip point even though its bound is a comparatively generous 8.94 ms. The first mechanism is the one that excludes four of the six keys here, and it is a property of WHERE THE CAPTURED RUN LANDED rather than of the metric: the same span.tx.apply.p50 has read 0.7917 ms, mid-distribution, where the identical rule produces a 4.21 ms bound that absorbs the whole range. Whether the gate functioned was therefore decided by luck of the draw. THE FOLLOW-UP THAT WOULD RESTORE COVERAGE, stated so it is not left implied: a baseline captured from a SINGLE run cannot support these keys, because one sample carries no information about spread and the bound is derived from that one sample alone. What would let them be gated again is a multi-run baseline -- or a spread measurement captured alongside the baseline, so a bound can be sized against observed variance instead of against the ladder only. That is not implemented; it is the design change these six exclusions are waiting on. Until then, do NOT re-gate any of them by re-baselining until a run happens to land favourably, which is the failure this note exists to prevent.",
-  "_excluded_spans": "rpc.process is deliberately absent from spans.names. It is created only in ServerHandler::processRequest() on the HTTP/JSON-RPC path, which the workload load generators, being WebSocket-only, never reach, so its quantiles were captured as null every run and could never gate. (The harness shell scripts do issue a few HTTP JSON-RPC health polls, far too few to produce a meaningful quantile.) See baselines/README.md.",
-  "_key_format": "{category}.{name}.p{quantile}  (e.g. span.tx.process.p99, job.transaction.queued.p95). Only the categories defined below are captured; there is no rpc_methods group, so no rpc.* key is produced or gated (FU-4).",
-  "excluded_keys": {
-    "span.consensus.ledger_close.p50": "Run-to-run variance exceeds the bound this ladder can derive, the same limit as the ledger.validate pair and the same mechanism as the two sibling p50 keys excluded alongside it. Baseline 0.0387 ms sits in the low bucket (0.01, 0.05], so hi_next is 0.1 ms and the derived bound is 0.0613 ms -- a 2.58x trip point. Measured across three CI runs the value spans 0.0387 to 0.2377 ms, a 6.1x spread (5.9x over four runs), and run 32867433073 read 0.2377 ms, 2.38x the trip point, on the SAME post-path-finding-removal workload as this baseline. So a healthy run reddens CI. This is a variance limit, not a defect and not a missing bound: widening is unavailable, because a bound tolerating 0.2377 ms would reach past the 0.25 ms edge and gate almost nothing. Do NOT re-gate by widening, and do NOT re-baseline until a run lands higher -- see _excluded_shape.",
-    "span.ledger.build.p50": "The same mechanism as span.consensus.ledger_close.p50, one bucket up. Baseline 0.1151 ms sits in (0.1, 0.25], so hi_next is 0.5 ms and the bound is 0.3849 ms -- a 4.34x trip point. Across three CI runs the value spans 0.1151 to 2.3826 ms, a 20.7x spread (25.3x over four runs), and the observed maximum is 4.77x the trip point. Note what the previous baseline hid: at 1.0612 ms the same rule gave a 8.94 ms bound and a 10 ms trip point, which absorbed the entire range, so this key read as gated purely because that capture landed mid-distribution. Ledger construction is the hot path this gate most wants to guard, which makes the loss real and worth fixing properly -- with a baseline that carries spread information, not with a wider bound.",
-    "span.ledger.build.p99": "Run-to-run variance exceeds the bound this ladder can derive, measured on the first three runs after the account-funding race was fixed. Baseline 8.944 ms (median of CI runs 34495527952, 34505215266, 34507425933) sits in (5, 10], so hi_next is 25 ms, the bound is 16.056 ms and the trip point is 25 ms. Across those three runs the value read 29.000, 7.060 and 8.944 ms -- a 4.11x spread, and the maximum is 1.16x the trip point, so a healthy run reddens CI. Run 34495527952 is that run: it turned the workload gate red on this key alone while all 200 span and metric checks passed and every phase reported 0 errors. Two corroborating details. First, its 29.000 ms reading is shared to the last digit with span.consensus.accept.p99 in the same run, which is the signature of histogram_quantile interpolating a thin tail inside one bucket rather than of ledger construction slowing down. Second, the previous baseline hid this: at 9.109 ms the same rule also gave a 25 ms trip point, and the key read as gated only because both the capture and the comparison runs happened to land low -- the 2026-08-26 capture predated the funding fix, so its phases submitted little or no traffic. Widening is not available: a bound tolerating 29.000 ms would be 20.06 ms and reach into the (25, 50] bucket, restoring exactly the single-crossing false positive the derivation rule exists to remove. Ledger construction is not left unguarded -- span.ledger.build.p95 stays gated and sits at 0.48 of its trip point. Do NOT re-gate this key by widening the bound; it needs a spread-aware baseline, or a finer ladder edge between 10 ms and 25 ms.",
-    "span.ledger.validate.p95": "Run-to-run variance is larger than the bound this ladder can derive. Measured across four CI runs the value spans 0.1281 to 0.7500 ms, a 5.9x spread, against a baseline of 0.2404 ms whose trip point is the next ladder edge at 0.5 ms -- so an ordinary run clears the trip point with nothing having regressed. Run 32867433073 read 0.7500 ms, +212%, and turned CI red. The derived bound models QUANTIZATION noise only (hi_next - baseline is one bucket of headroom); the dominant noise term for this span is peer-validation arrival timing in a 5-node cluster, and that term was never measured before the key was gated. Widening is not available: a bound that tolerated 0.7500 ms would reach past the 1 ms edge and leave the key gating nothing. This is a variance limit, not a defect and not a missing bound -- do NOT re-gate it by widening.",
-    "span.ledger.validate.p99": "The same mechanism as p95, two orders of magnitude worse. Across the same four runs the value spans 0.3875 to 25.8750 ms, a 66.8x spread, against a baseline of 1.0600 ms and a 10 ms trip point; run 32862589645 read 25.8750 ms, +2341%. The span opens only once a quorum-completing validation arrives (LedgerMaster.cpp:1003, inside checkAccept, past the tvc < minVal early return) and wraps the promotion work that follows -- setValidated, setFull, setValidLedger, pendSaveValidated -- so its duration tracks peer-validation arrival timing and what promotion then triggers. One slow consensus round therefore dominates the tail of a 3m rate window, and which round that is differs every run. A bound tolerating 25.8750 ms would be ~24.8 ms against a 1.0600 ms baseline, which gates nothing at all. Note that the two CI failures landed on DIFFERENT quantiles in different runs while the other quantile stayed well inside its bound in the same run: that asymmetry is the signature of variance, not of a regression.",
-    "span.tx.apply.p50": "The most extreme case of the low-bucket mechanism, and the clearest evidence that a single-run baseline cannot size a bound for these keys. Baseline 0.00597 ms lands in the ladder's FIRST bucket (0, 0.01], so hi_next is 0.05 ms and the bound is 0.0440 ms. Across three CI runs the value spans 0.00597 to 2.3378 ms, a 391.8x spread (364x over four runs), putting the observed maximum at 46.76x its trip point -- by far the worst of the six. The previous baseline read 0.7917 ms for the same key on the same workload, a 132x difference between two runs, and at that value the identical rule produced a 4.21 ms bound whose 5 ms trip point absorbed the full range. Nothing about the metric changed between those two captures; only where the sampled run fell in its own distribution did. Separately, a baseline inside the first bucket means the reported figure is interpolation across that bucket and tracks the FRACTION of applies finishing under 10 us rather than a latency, which is the ledger.store problem in embryo -- so restoring this key needs a finer low-end ladder as well as a spread-aware baseline. Rule E does not flag it because the value is not quantile x first_edge exactly."
-  },
-  "job_queue": {
-    "_phases": ["queued", "running"],
-    "_quantiles": [0.95],
-    "_queued_template": "histogram_quantile({quantile}, sum by (le) (rate(job_queued_us_bucket{job_type=\"{name}\"}[{window}])))",
-    "_running_template": "histogram_quantile({quantile}, sum by (le) (rate(job_running_us_bucket{job_type=\"{name}\"}[{window}])))",
-    "_unit": "us",
-    "names": ["transaction", "acceptLedger"]
-  },
-  "spans": {
-    "_quantiles": [0.5, 0.95, 0.99],
-    "_query_template": "histogram_quantile({quantile}, sum by (le) (rate(span_duration_milliseconds_bucket{span_name=\"{name}\"}[{window}])))",
-    "_unit": "ms",
-    "names": [
-      "rpc.ws_message",
-      "tx.process",
-      "tx.apply",
-      "ledger.build",
-      "ledger.validate",
-      "consensus.ledger_close",
-      "consensus.accept"
-    ]
-  }
-}

docker/telemetry/workload/requirements.txt

@@ -1,786 +0,0 @@
-# Python dependencies for the telemetry workload tools.
-#
-# cspell:ignore aiohappyeyeballs
-#
-# Install: pip install -r requirements.txt
-#
-# Pinned with hashes so every install resolves to exactly these releases.
-# Direct dependencies are websockets and aiohttp; the rest is aiohttp's
-# transitive closure, which pip requires once any hash is present.
-#
-# Floors to keep when bumping: aiohttp >= 3.10.11 (earliest release carrying
-# every 2024 aiohttp fix), websockets >= 14 (rejects a concurrent recv() with
-# ConcurrencyError instead of a bare RuntimeError).
-#
-# Regenerate (python >= 3.11, the repo minimum, matching the universal floor):
-#    printf 'websockets>=17.0\naiohttp>=3.14.3\n' > requirements.in
-#    uv pip compile requirements.in --generate-hashes --universal \
-#        --python-version 3.11 --output-file requirements.txt
-aiohappyeyeballs==2.7.1 \
-    --hash=sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d \
-    --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472
-    # via aiohttp
-aiohttp==3.14.3 \
-    --hash=sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39 \
-    --hash=sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043 \
-    --hash=sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b \
-    --hash=sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d \
-    --hash=sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf \
-    --hash=sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f \
-    --hash=sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7 \
-    --hash=sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc \
-    --hash=sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559 \
-    --hash=sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f \
-    --hash=sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929 \
-    --hash=sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147 \
-    --hash=sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9 \
-    --hash=sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8 \
-    --hash=sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf \
-    --hash=sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7 \
-    --hash=sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8 \
-    --hash=sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85 \
-    --hash=sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30 \
-    --hash=sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553 \
-    --hash=sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7 \
-    --hash=sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86 \
-    --hash=sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e \
-    --hash=sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a \
-    --hash=sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c \
-    --hash=sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da \
-    --hash=sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5 \
-    --hash=sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d \
-    --hash=sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100 \
-    --hash=sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71 \
-    --hash=sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22 \
-    --hash=sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1 \
-    --hash=sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479 \
-    --hash=sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb \
-    --hash=sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062 \
-    --hash=sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661 \
-    --hash=sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427 \
-    --hash=sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32 \
-    --hash=sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a \
-    --hash=sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db \
-    --hash=sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42 \
-    --hash=sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a \
-    --hash=sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd \
-    --hash=sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06 \
-    --hash=sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8 \
-    --hash=sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228 \
-    --hash=sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0 \
-    --hash=sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919 \
-    --hash=sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee \
-    --hash=sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2 \
-    --hash=sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f \
-    --hash=sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43 \
-    --hash=sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098 \
-    --hash=sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c \
-    --hash=sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371 \
-    --hash=sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b \
-    --hash=sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0 \
-    --hash=sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f \
-    --hash=sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100 \
-    --hash=sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529 \
-    --hash=sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc \
-    --hash=sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c \
-    --hash=sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41 \
-    --hash=sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716 \
-    --hash=sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33 \
-    --hash=sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f \
-    --hash=sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e \
-    --hash=sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa \
-    --hash=sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b \
-    --hash=sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80 \
-    --hash=sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646 \
-    --hash=sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e \
-    --hash=sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b \
-    --hash=sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c \
-    --hash=sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963 \
-    --hash=sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae \
-    --hash=sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25 \
-    --hash=sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c \
-    --hash=sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f \
-    --hash=sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807 \
-    --hash=sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a \
-    --hash=sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f \
-    --hash=sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d \
-    --hash=sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82 \
-    --hash=sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15 \
-    --hash=sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0 \
-    --hash=sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d \
-    --hash=sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9 \
-    --hash=sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19 \
-    --hash=sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239 \
-    --hash=sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0 \
-    --hash=sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c \
-    --hash=sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5 \
-    --hash=sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b \
-    --hash=sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4 \
-    --hash=sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2 \
-    --hash=sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9 \
-    --hash=sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0 \
-    --hash=sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883 \
-    --hash=sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d \
-    --hash=sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d \
-    --hash=sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6 \
-    --hash=sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3 \
-    --hash=sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924 \
-    --hash=sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde \
-    --hash=sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787 \
-    --hash=sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb \
-    --hash=sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b \
-    --hash=sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0 \
-    --hash=sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910 \
-    --hash=sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9 \
-    --hash=sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627 \
-    --hash=sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48 \
-    --hash=sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b \
-    --hash=sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce \
-    --hash=sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a \
-    --hash=sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0 \
-    --hash=sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24 \
-    --hash=sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5
-    # via -r requirements.in
-aiosignal==1.4.0 \
-    --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \
-    --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7
-    # via aiohttp
-attrs==26.1.0 \
-    --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \
-    --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32
-    # via aiohttp
-frozenlist==1.8.0 \
-    --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \
-    --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \
-    --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \
-    --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \
-    --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \
-    --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \
-    --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \
-    --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \
-    --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \
-    --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \
-    --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \
-    --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \
-    --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \
-    --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \
-    --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \
-    --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \
-    --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \
-    --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \
-    --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \
-    --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \
-    --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \
-    --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \
-    --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \
-    --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \
-    --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \
-    --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \
-    --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \
-    --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \
-    --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \
-    --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \
-    --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \
-    --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \
-    --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \
-    --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \
-    --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \
-    --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \
-    --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \
-    --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \
-    --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \
-    --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \
-    --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \
-    --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \
-    --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \
-    --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \
-    --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \
-    --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \
-    --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \
-    --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \
-    --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \
-    --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \
-    --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \
-    --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \
-    --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \
-    --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \
-    --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \
-    --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \
-    --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \
-    --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \
-    --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \
-    --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \
-    --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \
-    --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \
-    --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \
-    --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \
-    --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \
-    --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \
-    --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \
-    --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \
-    --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \
-    --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \
-    --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \
-    --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \
-    --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \
-    --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \
-    --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \
-    --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \
-    --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \
-    --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \
-    --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \
-    --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \
-    --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \
-    --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \
-    --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \
-    --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \
-    --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \
-    --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \
-    --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \
-    --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \
-    --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \
-    --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \
-    --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \
-    --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \
-    --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \
-    --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \
-    --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \
-    --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \
-    --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \
-    --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \
-    --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \
-    --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \
-    --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \
-    --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \
-    --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \
-    --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \
-    --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \
-    --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \
-    --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \
-    --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \
-    --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \
-    --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \
-    --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \
-    --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \
-    --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \
-    --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \
-    --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \
-    --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \
-    --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \
-    --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \
-    --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \
-    --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \
-    --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \
-    --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \
-    --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \
-    --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \
-    --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \
-    --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \
-    --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \
-    --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \
-    --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \
-    --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd
-    # via
-    #   aiohttp
-    #   aiosignal
-idna==3.18 \
-    --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \
-    --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848
-    # via yarl
-multidict==6.7.1 \
-    --hash=sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0 \
-    --hash=sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9 \
-    --hash=sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581 \
-    --hash=sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2 \
-    --hash=sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941 \
-    --hash=sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3 \
-    --hash=sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43 \
-    --hash=sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962 \
-    --hash=sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1 \
-    --hash=sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f \
-    --hash=sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c \
-    --hash=sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8 \
-    --hash=sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa \
-    --hash=sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6 \
-    --hash=sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c \
-    --hash=sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991 \
-    --hash=sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262 \
-    --hash=sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd \
-    --hash=sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d \
-    --hash=sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d \
-    --hash=sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5 \
-    --hash=sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3 \
-    --hash=sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601 \
-    --hash=sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505 \
-    --hash=sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0 \
-    --hash=sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292 \
-    --hash=sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed \
-    --hash=sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362 \
-    --hash=sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511 \
-    --hash=sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23 \
-    --hash=sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2 \
-    --hash=sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb \
-    --hash=sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e \
-    --hash=sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582 \
-    --hash=sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0 \
-    --hash=sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2 \
-    --hash=sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e \
-    --hash=sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d \
-    --hash=sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65 \
-    --hash=sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a \
-    --hash=sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd \
-    --hash=sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d \
-    --hash=sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108 \
-    --hash=sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177 \
-    --hash=sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144 \
-    --hash=sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5 \
-    --hash=sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd \
-    --hash=sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5 \
-    --hash=sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060 \
-    --hash=sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37 \
-    --hash=sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56 \
-    --hash=sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df \
-    --hash=sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963 \
-    --hash=sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568 \
-    --hash=sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db \
-    --hash=sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118 \
-    --hash=sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84 \
-    --hash=sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f \
-    --hash=sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889 \
-    --hash=sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71 \
-    --hash=sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f \
-    --hash=sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0 \
-    --hash=sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7 \
-    --hash=sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048 \
-    --hash=sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8 \
-    --hash=sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49 \
-    --hash=sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0 \
-    --hash=sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9 \
-    --hash=sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59 \
-    --hash=sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190 \
-    --hash=sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709 \
-    --hash=sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d \
-    --hash=sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c \
-    --hash=sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e \
-    --hash=sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2 \
-    --hash=sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40 \
-    --hash=sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3 \
-    --hash=sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee \
-    --hash=sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609 \
-    --hash=sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c \
-    --hash=sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445 \
-    --hash=sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1 \
-    --hash=sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a \
-    --hash=sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5 \
-    --hash=sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31 \
-    --hash=sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8 \
-    --hash=sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33 \
-    --hash=sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7 \
-    --hash=sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca \
-    --hash=sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8 \
-    --hash=sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92 \
-    --hash=sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733 \
-    --hash=sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429 \
-    --hash=sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9 \
-    --hash=sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4 \
-    --hash=sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6 \
-    --hash=sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2 \
-    --hash=sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172 \
-    --hash=sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981 \
-    --hash=sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5 \
-    --hash=sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de \
-    --hash=sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52 \
-    --hash=sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7 \
-    --hash=sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c \
-    --hash=sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2 \
-    --hash=sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6 \
-    --hash=sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf \
-    --hash=sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f \
-    --hash=sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b \
-    --hash=sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961 \
-    --hash=sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a \
-    --hash=sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3 \
-    --hash=sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b \
-    --hash=sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358 \
-    --hash=sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6 \
-    --hash=sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e \
-    --hash=sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1 \
-    --hash=sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c \
-    --hash=sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5 \
-    --hash=sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53 \
-    --hash=sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872 \
-    --hash=sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e \
-    --hash=sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df \
-    --hash=sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03 \
-    --hash=sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8 \
-    --hash=sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a \
-    --hash=sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122 \
-    --hash=sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a \
-    --hash=sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee \
-    --hash=sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32 \
-    --hash=sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3 \
-    --hash=sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489 \
-    --hash=sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23 \
-    --hash=sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34 \
-    --hash=sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75 \
-    --hash=sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8 \
-    --hash=sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a \
-    --hash=sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d \
-    --hash=sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855 \
-    --hash=sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b \
-    --hash=sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4 \
-    --hash=sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4 \
-    --hash=sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d \
-    --hash=sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0 \
-    --hash=sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba \
-    --hash=sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19
-    # via
-    #   aiohttp
-    #   yarl
-propcache==0.5.2 \
-    --hash=sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427 \
-    --hash=sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5 \
-    --hash=sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa \
-    --hash=sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7 \
-    --hash=sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a \
-    --hash=sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0 \
-    --hash=sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660 \
-    --hash=sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94 \
-    --hash=sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917 \
-    --hash=sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42 \
-    --hash=sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3 \
-    --hash=sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa \
-    --hash=sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d \
-    --hash=sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33 \
-    --hash=sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a \
-    --hash=sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511 \
-    --hash=sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0 \
-    --hash=sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84 \
-    --hash=sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c \
-    --hash=sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66 \
-    --hash=sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821 \
-    --hash=sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb \
-    --hash=sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e \
-    --hash=sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853 \
-    --hash=sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56 \
-    --hash=sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55 \
-    --hash=sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6 \
-    --hash=sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704 \
-    --hash=sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82 \
-    --hash=sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f \
-    --hash=sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64 \
-    --hash=sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999 \
-    --hash=sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b \
-    --hash=sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb \
-    --hash=sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d \
-    --hash=sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4 \
-    --hash=sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab \
-    --hash=sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f \
-    --hash=sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03 \
-    --hash=sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5 \
-    --hash=sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba \
-    --hash=sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979 \
-    --hash=sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b \
-    --hash=sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144 \
-    --hash=sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d \
-    --hash=sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e \
-    --hash=sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67 \
-    --hash=sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117 \
-    --hash=sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa \
-    --hash=sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb \
-    --hash=sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96 \
-    --hash=sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5 \
-    --hash=sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476 \
-    --hash=sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191 \
-    --hash=sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78 \
-    --hash=sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078 \
-    --hash=sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837 \
-    --hash=sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a \
-    --hash=sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba \
-    --hash=sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe \
-    --hash=sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c \
-    --hash=sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf \
-    --hash=sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c \
-    --hash=sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9 \
-    --hash=sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8 \
-    --hash=sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe \
-    --hash=sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031 \
-    --hash=sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913 \
-    --hash=sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d \
-    --hash=sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf \
-    --hash=sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f \
-    --hash=sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539 \
-    --hash=sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b \
-    --hash=sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285 \
-    --hash=sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959 \
-    --hash=sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d \
-    --hash=sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4 \
-    --hash=sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f \
-    --hash=sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836 \
-    --hash=sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274 \
-    --hash=sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d \
-    --hash=sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f \
-    --hash=sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e \
-    --hash=sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe \
-    --hash=sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1 \
-    --hash=sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a \
-    --hash=sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39 \
-    --hash=sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7 \
-    --hash=sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a \
-    --hash=sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164 \
-    --hash=sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e \
-    --hash=sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2 \
-    --hash=sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0 \
-    --hash=sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0 \
-    --hash=sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335 \
-    --hash=sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568 \
-    --hash=sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4 \
-    --hash=sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80 \
-    --hash=sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2 \
-    --hash=sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370 \
-    --hash=sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4 \
-    --hash=sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b \
-    --hash=sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42 \
-    --hash=sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a \
-    --hash=sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e \
-    --hash=sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757 \
-    --hash=sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825 \
-    --hash=sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0 \
-    --hash=sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27 \
-    --hash=sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf \
-    --hash=sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f \
-    --hash=sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d \
-    --hash=sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366 \
-    --hash=sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc \
-    --hash=sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c \
-    --hash=sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7 \
-    --hash=sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702 \
-    --hash=sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098 \
-    --hash=sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751 \
-    --hash=sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e \
-    --hash=sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6
-    # via
-    #   aiohttp
-    #   yarl
-typing-extensions==4.16.0 ; python_full_version < '3.13' \
-    --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
-    --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
-    # via
-    #   aiohttp
-    #   aiosignal
-websockets==17.0.1 \
-    --hash=sha256:02ed63bf26dda9fa27df730a41f6664586c4ee05972c8fb667ce1725b3fd13d3 \
-    --hash=sha256:02f0b037a737d0cb0c33866c97bcd1a0b73170dfbf42d69d8fb86f51002fd5ae \
-    --hash=sha256:038cfad5d5417f8bb09295abe986029a26d22f34bda622ccc79b670efd4dab56 \
-    --hash=sha256:07abc3bd196a48af476a82fd47f3f79a6a3f70937a9f930cef703cfa0c9d83b6 \
-    --hash=sha256:07d78a509c3333f5908c83d7f78144ea68a6c9ec28110f5c54d81d8fcdc262c4 \
-    --hash=sha256:0b52c76b8a870b141b7ca0705289452183ce7a523101954ccfe29a25986a673f \
-    --hash=sha256:10b1587c599fa0f2c89154587c80e0fda98ade6c9fa8c0260a2823fb1800b685 \
-    --hash=sha256:10f461191125c63902ea7394ae9e752b1b5785641850c1d365bb30b0f88bc53f \
-    --hash=sha256:15920057a6b723f84734f0641403bca163a4b176e5af809ee4f0c4a1e75e9fed \
-    --hash=sha256:17ac37716c0244e82c9e384c41653c090b1864c6610224ca3857e7f7b58fce10 \
-    --hash=sha256:18ded646ce98cdd3c0235825b3252f1df55765ba49b616bb10282f758667b4d0 \
-    --hash=sha256:1b363bfd72a52c0658a3154a4cff219f15a474b35a235057d38853bf151acce7 \
-    --hash=sha256:1bdd8c4be420905dd732e00dcd669852d8128cc723efa585a0c0e51adb00a28a \
-    --hash=sha256:1d4cf7e8e5b8b1fa40758ac7524843a00237b124ab217e227542cafcfeb7a946 \
-    --hash=sha256:1df81d174c1561292de9e40b141cafc04f69077272f6c352afe1d743e20810df \
-    --hash=sha256:20a92f78ac8250984ed459faa9ca48c285adbfc0038ddc3fdac6046990a9c9ed \
-    --hash=sha256:22bd00f8bae2bccdb5dbe41e20f58ba44ca9fff0b4b561aaf39099c35da762ed \
-    --hash=sha256:2437d4ca208cc0f246d3a2297ae7474b4ba18261aaf5b9c79c84c031ecf348e1 \
-    --hash=sha256:246927ae9ae06ca0d42a483a4bdb80d4862e1ee5b4cab37c354a5e1ad8356448 \
-    --hash=sha256:2503c7e2a5049a12d5dac917a46d5d52591283a766165b8176bb167560421b38 \
-    --hash=sha256:2604de7228506b13a44a256a9d223943340c0e725af5d367dc068e192b027761 \
-    --hash=sha256:28012a54510fe8301bb893ef143cec30a2780a2d3bc20b7bbdf4379d7a63945d \
-    --hash=sha256:2a855b6dfe21c4d3420be265ae031829ba8ba0be0ea350d9f7c3ef30ae63ebe2 \
-    --hash=sha256:2abb1ba0a5133b7d2ef3c1c9f4b0c1e8a101012dce0b594ab2b2888d9a64820e \
-    --hash=sha256:2b3f3020171202b135ca078e20434977c6b2b02af647130d6980c9e39b9462e3 \
-    --hash=sha256:2bc14b481e05e331811108daa1aeb41a5e237a5564ef2f02ec5a356a0f102f78 \
-    --hash=sha256:2fa2cb465a131c347ba6717a78c887746e73edb1c131d01c982d6ef0d68b82e0 \
-    --hash=sha256:409d93efcaa14f7a99592c5baaef5ec6ca94fba0f5aec1a86f693977c69c9c1c \
-    --hash=sha256:41d6aa06b5ab832aee72fedf47a149535b121ac900b6bb4d3fe14712afac9a79 \
-    --hash=sha256:49266e4488309b38783257293a38298942b9a03aa106fcb45195377a77c0c1e2 \
-    --hash=sha256:4d1d99db29b5444e3982f1ce2ba8a833508ad44b2f1fbd0bd99e81d825c0b461 \
-    --hash=sha256:4d41c0a1d47a478bc432b3b9068097bee1ce0c5b19327ea6f75c2ab34ab1f2fb \
-    --hash=sha256:5033ffe6804dd53afafa7d08e8c3eef2d2431f34d58ca30507a8442dd04a033a \
-    --hash=sha256:53b90c00bc6201ab6695c7ff51a04d0e425514c37515e9eeecd2c1b978ac6c0e \
-    --hash=sha256:54cdcaa56f5d3eafd57058f0fa4a3de93a310b43a3c4699f06efc4c0bd054a5a \
-    --hash=sha256:5508f38c98ac29def9e747b87543b008a58b075df6da70b2cf2e0b47073d33bb \
-    --hash=sha256:55383d8177b3c99fd873ee5db0e0193f4c1dd4a3feaccf1a4a03c1b7cf539cac \
-    --hash=sha256:55b12e47dcee83673a40d07686cfb6f9d6dfc285976ade9463f61d2bef3fad22 \
-    --hash=sha256:5661f868ef191d33dfc6a0cc7c5b3d495f0cc8bb3f8b30d87bda8755c61c95f5 \
-    --hash=sha256:57d2ee9b24b404ce75f3814f92073c0ed88106c950148d2427fe8d25ca254d1f \
-    --hash=sha256:599b03beb77633bffc095334338fad79cafc2b01fbd58953838130a9ae967d7b \
-    --hash=sha256:5baa9bc0dfbae8c507e51c8cf1b6d4628086f7a87bbd3a9952bd5f035451f1cc \
-    --hash=sha256:5f33a649bfcb8312524173cc4bbafa7dbb236e18eee9aa31a1d324ca0ddda28c \
-    --hash=sha256:6740be6d1bab69f08ab52cb15b08f76c143b6fe61c580ba62bd929f3ab7a1d42 \
-    --hash=sha256:6a434e59962a4fb9016bea327e1d14d6cd67670ecfb8942b4f4a0c24036634ce \
-    --hash=sha256:6db9e5bf3649ab506c6ae8a3ac85a00fb1ae3816d75962771b2df8adbc5d40d2 \
-    --hash=sha256:6fd88365da261c53d3e943fb37e0d0721b9cde119f6b2e3fc84369b6ab234d63 \
-    --hash=sha256:7002d5f9e1c3ddd991cdfdbfee18cc8c8b196b2445022892badacd6cb338bbbc \
-    --hash=sha256:70d438268e49f1a4bd096b6b6f7010f3ab48b5db2574dbf7d8c864c46ce7a06a \
-    --hash=sha256:72d7f2a5aeb4e82daa4ee18f125b4277f427033359be5c745ad709608446cc2c \
-    --hash=sha256:733e3cc7171fa1b899edbe725ef9382d0e960657dc1fd933f3281ae910c01dab \
-    --hash=sha256:734d20364dc2cfe03674883cafcf580b6e431c5ce42b476312b9285310230cf9 \
-    --hash=sha256:759adeb5b0c5775b563254ec63b5b79089fc0045b479143a0b1b8c0ebaae1253 \
-    --hash=sha256:769ce7e2acfd9a89f2bed3a9c0da229459516bbc00bd4c9e2ca492c613ae4861 \
-    --hash=sha256:810cb3fb5fa6e447216f4e82d9a85cb8aed0929ae3538153ddfe8a6e3121a58d \
-    --hash=sha256:81ce19c6046ace11da7001781be7317bb1dc389f399af4b2ed962190f76f9add \
-    --hash=sha256:846a4a8b0833e3cad57523d9e3bd50ec8ea05ab9d06c582f82a1340ba096af5f \
-    --hash=sha256:872273e629ca7e3d35f16a2dc6ede84e1d5c831e616b8277de6e4f83114e7c58 \
-    --hash=sha256:8848c207049ad49d318e5f64a3d4d7bb189f8328d0d98e65647788f2a085785c \
-    --hash=sha256:884af729b8ab50486acd94d9768c2b60914bf39b579ebba0a5cb73bfdfd61fd2 \
-    --hash=sha256:8c07f145d0b9e90cbd96035f31fb79199aef4da1872854e36ebeb258e3d57594 \
-    --hash=sha256:8cd3369e42c0246afaf9d669cfc19797e3a49e8c0a639544459c57597108b966 \
-    --hash=sha256:8e387adb0c692c6b5571bdeafc8ac9d1901ea30f10309134780b16ecd35e6605 \
-    --hash=sha256:90246fa9e6cb192a778ce6ce024057ec54317a894db7899c922dcdc1f4cbf6a5 \
-    --hash=sha256:90973a3a00f23afdfd1c9b06fb84289bf0220f247ef8a62501a1967c7af54f7b \
-    --hash=sha256:9493314a99e599163c854fb5900ad7f7ea38c5cb9d9103aa30b3c6b8181c01fa \
-    --hash=sha256:9f7747d3daa41a11f25f7cca5dc988fc51da97b311bed4c9d843860f79779283 \
-    --hash=sha256:a39ce3a7b0e6059be093213d637963101380157bcbad355916738fafb490698d \
-    --hash=sha256:a60fa1a25cca1bcc2bf87b8d6be37a741f0a3239fb5e9cfb7a37173b68ffcf87 \
-    --hash=sha256:a68e604c6d1b0338e46652e2688cbce8096ad9c03548b075fda9e2ea19a9b7dd \
-    --hash=sha256:a8af570fc29cd998a921c7131c8ac81d9434466d6d25300cb12a690fb56a8a08 \
-    --hash=sha256:aadc298969ad229d8e3029fc5cc751fdad286696230f9cf014e90ff9cd8e6ea0 \
-    --hash=sha256:ab56439c9f74c52770690c7b2f616b3bf775cb3920453ee355ac765c032d8bbf \
-    --hash=sha256:ab9f962a5b64a5c3c845d556b7dc4e6fb683f7b67179f8205e814bb2e0213ffe \
-    --hash=sha256:afbce6e3f0fac32dc87c2a0d84869d1a706460d64f39f3889386413e6e4d3d26 \
-    --hash=sha256:b3ff0ad440ad52dda64138f16895f66403f40192365e39b1010e889f289746b0 \
-    --hash=sha256:b580794e926cab7ff42ee4371ef14e0b22cb2bb722a607f77769136468f49a3f \
-    --hash=sha256:b85b960a4507b0714c0a1246d031be9118d908ee974dc085257297a955205f1d \
-    --hash=sha256:b98860aefbd3d9bc8e3c7f0eefb83b11142b16110739c68cd33d3b4d6e84e536 \
-    --hash=sha256:bb31f42ea095ea826463c770829aa188a86c9a5c976b1467cbbf583c811de833 \
-    --hash=sha256:bc0bca48ba24c6c866847fd20478a51dd547fa0ad258dab9615c414ec534bbc0 \
-    --hash=sha256:bd1470d2c53fe53269bf5619da7725d30dd9b9693f1689f7a85eab8dea734442 \
-    --hash=sha256:c09e097d0e46e3c289bedab9a475ae344b70c30ff5646e46af22b4e6fdc97b21 \
-    --hash=sha256:c1bec5d6a19f5fbe87e4940739cfc65e7bb53d8b353e1029b8037a1653b321bc \
-    --hash=sha256:c1c118a6b0e25bfc9a6802075d748fa6321714ffbdf3c88d29d9a0e3c7386c75 \
-    --hash=sha256:c23e532c8a2325a1e7486de8763a60dc43e83f01bcaeca07e3ba79652c156db1 \
-    --hash=sha256:c356dbddab0a529ed7574f78f559d75a223735c321c28f6f587fbf02b11ed301 \
-    --hash=sha256:c38515cb54902f7e97d0239e81ef46c4444f9475f4807fb9bbdb789b4089abcf \
-    --hash=sha256:c395bda8e7d8f51a02e80261fb57127979e5c472675d9a96b2860619ad47da48 \
-    --hash=sha256:c6be9cba65c65cc76dfa3d4619e359ff02a4476c74e179b215236c11a0b32345 \
-    --hash=sha256:cd526c8228e759c1006c4b7c9ac71dc4e925ced1a6a6a5a8e94643709738f63e \
-    --hash=sha256:cddc675ec31bca65473321f9a9794e488b43b3b8de5d02c8ef4810c5d5792163 \
-    --hash=sha256:cffc84ddec6da7f447677266fee2a3c40ecc78172f00752aa1150b8a8d65df1d \
-    --hash=sha256:d41e9845514754a42d1d83b2fca9d27fee2ca7b3b0bee6843ba5a9bb2b6e25ac \
-    --hash=sha256:d69fd559f9f0e8a52d2fce6f04ee143f86e70df0a189cd95164eddac599e810f \
-    --hash=sha256:d7d72843691f50b91127c50688df10cb72ec6f4c4b1d7e2c11ab33b16acf8e51 \
-    --hash=sha256:d9aac6081513f02eac3f8caace800dbfc5c608b69e4a7bef69e414eabfc95aa1 \
-    --hash=sha256:dbfae8e75b342e31fc6fd1a8bbb393b7cbb91d6cfd581650300a94381e7b7e2b \
-    --hash=sha256:e8208f2729cba030ff872a92064c97584eeb9502f53d32a05a0f05d5a17ca6c6 \
-    --hash=sha256:e95e321d0d763f2b6633512605f6112ebd70d5746f3ce05c941909d4a25233f2 \
-    --hash=sha256:e98ec9ec61cce5bc4b8b218322ad090b0994eb060bb04da704c62ef0a3d864e6 \
-    --hash=sha256:eab6de8a98b9a7772cf686d00b4de439fc7efb8ab05ae106ef227291d06f87c5 \
-    --hash=sha256:efe0ae052a8d023b87198921e8a7ce1dc7768816bcd2fbc20df171ac73a04891 \
-    --hash=sha256:f11a398d8170b7ac5000baf7f258dcda579ef3ea744e0cc6a165e0dfbc0d3198 \
-    --hash=sha256:f3fd9a1f87f8f0f3f8e9f9bd0195f7516562d13f5b178db8c5784d1f60b60bed \
-    --hash=sha256:f47b0815af3948ec6a440b3afa02f05b18cc0939549e91b5c677b5d9c2c8472a \
-    --hash=sha256:f991247276797d0c61ab7770bc9791eadc16f683b4d83517f624932adc1a8bab \
-    --hash=sha256:ffad64ce7ad3703d652a3fd9af26238377d24ce52c6ad8ff35d26d82f61f493f
-    # via -r requirements.in
-yarl==1.24.5 \
-    --hash=sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36 \
-    --hash=sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331 \
-    --hash=sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871 \
-    --hash=sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498 \
-    --hash=sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780 \
-    --hash=sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027 \
-    --hash=sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224 \
-    --hash=sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144 \
-    --hash=sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76 \
-    --hash=sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3 \
-    --hash=sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb \
-    --hash=sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740 \
-    --hash=sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec \
-    --hash=sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e \
-    --hash=sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a \
-    --hash=sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24 \
-    --hash=sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950 \
-    --hash=sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95 \
-    --hash=sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb \
-    --hash=sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928 \
-    --hash=sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c \
-    --hash=sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41 \
-    --hash=sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e \
-    --hash=sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550 \
-    --hash=sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f \
-    --hash=sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b \
-    --hash=sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e \
-    --hash=sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede \
-    --hash=sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad \
-    --hash=sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6 \
-    --hash=sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104 \
-    --hash=sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2 \
-    --hash=sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba \
-    --hash=sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9 \
-    --hash=sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12 \
-    --hash=sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1 \
-    --hash=sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027 \
-    --hash=sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385 \
-    --hash=sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2 \
-    --hash=sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840 \
-    --hash=sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2 \
-    --hash=sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4 \
-    --hash=sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc \
-    --hash=sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c \
-    --hash=sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213 \
-    --hash=sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba \
-    --hash=sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448 \
-    --hash=sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0 \
-    --hash=sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6 \
-    --hash=sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966 \
-    --hash=sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750 \
-    --hash=sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621 \
-    --hash=sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13 \
-    --hash=sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0 \
-    --hash=sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58 \
-    --hash=sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54 \
-    --hash=sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075 \
-    --hash=sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4 \
-    --hash=sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440 \
-    --hash=sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f \
-    --hash=sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4 \
-    --hash=sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c \
-    --hash=sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f \
-    --hash=sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d \
-    --hash=sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7 \
-    --hash=sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9 \
-    --hash=sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723 \
-    --hash=sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047 \
-    --hash=sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da \
-    --hash=sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293 \
-    --hash=sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b \
-    --hash=sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61 \
-    --hash=sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca \
-    --hash=sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed \
-    --hash=sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a \
-    --hash=sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a \
-    --hash=sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688 \
-    --hash=sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16 \
-    --hash=sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d \
-    --hash=sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077 \
-    --hash=sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce \
-    --hash=sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd \
-    --hash=sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d \
-    --hash=sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88 \
-    --hash=sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5 \
-    --hash=sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75 \
-    --hash=sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1 \
-    --hash=sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f \
-    --hash=sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b \
-    --hash=sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff \
-    --hash=sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9 \
-    --hash=sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f \
-    --hash=sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0 \
-    --hash=sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5 \
-    --hash=sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25 \
-    --hash=sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6 \
-    --hash=sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00 \
-    --hash=sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373 \
-    --hash=sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd \
-    --hash=sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1 \
-    --hash=sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2 \
-    --hash=sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d \
-    --hash=sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba \
-    --hash=sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104
-    # via aiohttp

docker/telemetry/workload/rpc_load_generator.py

@@ -1,860 +0,0 @@
-#!/usr/bin/env python3
-"""RPC Load Generator for rippled telemetry validation.
-
-Connects to one or more rippled WebSocket endpoints and fires all traced
-RPC commands at configurable rates with realistic production-like
-distribution.
-
-Command distribution (default weights, summing to 100):
-  40%  Health checks:   server_info, fee
-  30%  Wallet queries:  account_info, account_lines, account_objects
-  15%  Explorer:        ledger, ledger_data
-  10%  TX lookups:      tx, account_tx
-   5%  DEX queries:     book_offers, amm_info
-
-Path-finding RPC is deliberately absent — see "Pathfinding is not exercised"
-in README.md for why, and for how to put it back.
-
-Usage:
-    python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120
-
-    # Multiple endpoints (round-robin):
-    python3 rpc_load_generator.py \\
-        --endpoints ws://localhost:6006 ws://localhost:6007 \\
-        --rate 100 --duration 300
-
-    # Custom weights:
-    python3 rpc_load_generator.py --endpoints ws://localhost:6006 \\
-        --weights '{"server_info":60,"account_info":30,"ledger":10}'
-"""
-
-import argparse
-import asyncio
-import itertools
-import json
-import logging
-import math
-import random
-import sys
-import time
-import uuid
-from dataclasses import dataclass, field
-from typing import Any
-
-import websockets
-
-# websockets loads its submodules lazily, so websockets.exceptions is not
-# reachable through the package until something imports it. REQUEST_FAILURES
-# is built at import time and needs it now.
-import websockets.exceptions
-
-# ---------------------------------------------------------------------------
-# Configuration
-# ---------------------------------------------------------------------------
-
-# Default command distribution matching realistic production ratios.
-# Keys are RPC command names; values are relative weights.
-DEFAULT_WEIGHTS: dict[str, int] = {
-    # 40% health checks
-    "server_info": 25,
-    "fee": 15,
-    # 30% wallet queries
-    "account_info": 15,
-    "account_lines": 8,
-    "account_objects": 7,
-    # 15% explorer
-    "ledger": 10,
-    "ledger_data": 5,
-    # 10% tx lookups
-    "tx": 5,
-    "account_tx": 5,
-    # 5% DEX queries
-    "book_offers": 3,
-    "amm_info": 2,
-    # No path-finding command on purpose: every harness node disables
-    # pathfinding, so those calls only ever produced errors. README.md,
-    # "Pathfinding is not exercised", has the reason and the way back.
-}
-
-# Well-known genesis account for queries that require an account parameter.
-GENESIS_ACCOUNT = "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"
-
-# How long a single request waits for its reply.
-RECV_TIMEOUT_S = 10.0
-
-# Teardown budget for requests still in flight. Only the at-most-one request
-# per connection that already holds the gate can still finish, and its worst
-# case is the full receive timeout, so that plus a small grace is the whole
-# useful wait. Requests still queued behind the gate would need
-# queue_depth x round-trip, which is unbounded; they are cancelled instead.
-DRAIN_TIMEOUT_S = RECV_TIMEOUT_S + 2.0
-
-# Requests allowed to own one connection's recv() at a time. websockets
-# rejects a second concurrent recv() on the same socket, so this must stay 1
-# until a single reader task per connection demultiplexes replies to their
-# waiters; correlating by id, which send_rpc now does, is necessary for that
-# but not sufficient on its own. Concurrency comes from spreading requests
-# round-robin over the endpoints instead.
-MAX_INFLIGHT_PER_CONNECTION = 1
-
-# Source of the ``id`` sent with every request. xrpld echoes a request's id at
-# the top level of the reply, which is what lets send_rpc tell its own reply
-# from one that an earlier, timed-out request left in the receive buffer.
-_request_ids = itertools.count(1)
-
-# Fraction of dispatched requests that must reach the server for a run to
-# count as a measurement. The gate is one connection deep, so the ceiling is
-# len(connections) / round-trip requests per second; asking for more than
-# that silently drops the excess instead of erroring, which would report a
-# perfect score on a run that generated a fraction of its intended load.
-# Fifty percent mirrors the error-rate limit below, on the same reasoning: a
-# run in which most requests did not happen is not a valid baseline.
-MIN_DELIVERY_PCT = 50.0
-
-# Error rate above which the run is treated as a failure.
-MAX_ERROR_RATE_PCT = 50.0
-
-# Failures that mean "this request failed", not "the generator is broken":
-#   asyncio.TimeoutError - no reply within RECV_TIMEOUT_S. Same class as the
-#                          builtin TimeoutError on Python 3.11+.
-#   WebSocketException   - transport failure, including the connection being
-#                          closed while a receive was outstanding, and the
-#                          ConcurrencyError raised for a rejected concurrent
-#                          recv() (it subclasses WebSocketException).
-#   json.JSONDecodeError - reply body was not valid JSON.
-#   AttributeError       - reply parsed to something with no .get(), e.g. a
-#                          JSON array.
-REQUEST_FAILURES: tuple[type[BaseException], ...] = (
-    asyncio.TimeoutError,
-    websockets.exceptions.WebSocketException,
-    json.JSONDecodeError,
-    AttributeError,
-)
-
-logger = logging.getLogger("rpc_load_generator")
-
-
-# ---------------------------------------------------------------------------
-# Latency helpers
-# ---------------------------------------------------------------------------
-
-
-def _percentile(sorted_values: list[float], quantile: float) -> float:
-    """Return the nearest-rank percentile of an ascending list of values.
-
-    The nearest-rank index is ``ceil(n * q) - 1``, clamped to the last
-    element. Plain ``int(n * q)`` truncation selects one rank too high
-    whenever ``n * q`` is a whole number — at n=100 it picks index 99, the
-    maximum, so the reported p99 was really p100 (n=20 for p95).
-
-    Args:
-        sorted_values: Values sorted ascending.
-        quantile:      Quantile to select, in (0, 1] — e.g. 0.99.
-
-    Returns:
-        The selected value, or 0.0 when the list is empty.
-    """
-    n = len(sorted_values)
-    if n == 0:
-        return 0.0
-    idx = min(math.ceil(n * quantile) - 1, n - 1)
-    return sorted_values[max(idx, 0)]
-
-
-# ---------------------------------------------------------------------------
-# Data classes
-# ---------------------------------------------------------------------------
-
-
-@dataclass
-class LoadStats:
-    """Tracks request counts and latencies during a load run.
-
-    ``total_dispatched`` counts intent and ``total_sent`` counts outcome, so
-    the difference is the load that never happened. They diverge whenever the
-    requested rate exceeds what the connections can carry: the dispatch loop
-    keeps pace, the requests queue behind the per-connection gate, and
-    teardown cancels whatever never got its turn. Without the two extra
-    counters that shortfall shows up as nothing at all -- no error, no
-    warning, and an error_rate_pct of 0 over a fraction of the traffic.
-
-    Attributes:
-        total_dispatched: Requests the dispatch loop created a task for.
-        total_sent:       Requests that completed and were recorded.
-        total_success:    Requests that returned a valid result.
-        total_errors:     Requests that returned an error or timed out.
-        total_cancelled:  Requests cancelled at teardown, never recorded.
-        latencies:        Per-command round-trip times in seconds, for the
-                          requests that got a reply. Requests that never got
-                          one contribute no sample -- see record().
-        command_counts:   Per-command request count, replied or not.
-    """
-
-    total_dispatched: int = 0
-    total_sent: int = 0
-    total_success: int = 0
-    total_errors: int = 0
-    total_cancelled: int = 0
-    latencies: dict[str, list[float]] = field(default_factory=dict)
-    command_counts: dict[str, int] = field(default_factory=dict)
-
-    def record(self, command: str, latency: float | None, success: bool) -> None:
-        """Record the outcome of a single RPC call.
-
-        Pass ``latency=None`` when no reply arrived, i.e. a timeout or a
-        transport failure. Such a request still counts as an error, but it
-        contributes no latency sample: time-to-failure is not a round-trip
-        time, and a timeout would inject RECV_TIMEOUT_S into the distribution
-        and dominate the percentiles.
-
-        A reply carrying ``status: error`` is the opposite case. The round
-        trip completed and was timely, so its latency is a real measurement
-        and is kept even though the request is counted as an error.
-        """
-        self.total_sent += 1
-        if success:
-            self.total_success += 1
-        else:
-            self.total_errors += 1
-        self.command_counts[command] = self.command_counts.get(command, 0) + 1
-        if latency is not None:
-            self.latencies.setdefault(command, []).append(latency)
-
-    def summary(self) -> dict[str, Any]:
-        """Return a summary dict suitable for JSON serialization.
-
-        ``total_sent``, ``total_success``, ``total_errors``,
-        ``error_rate_pct`` and ``per_command`` keep their names and meanings;
-        workload_orchestrator.py reads the first and third of those. The three
-        delivery keys are additions. ``delivery_pct`` is 0.0 when nothing was
-        dispatched at all -- a run that opened no connection delivered none of
-        its load, and reporting 100% for it would be the same blind spot the
-        key exists to close.
-        """
-        # Keyed off command_counts, not latencies: a command whose every
-        # request timed out has a count but no samples, and dropping it from
-        # the report would hide the command that failed worst.
-        per_command: dict[str, Any] = {}
-        for cmd in sorted(self.command_counts):
-            sorted_lats = sorted(self.latencies.get(cmd, []))
-            per_command[cmd] = {
-                "count": self.command_counts[cmd],
-                "latency_samples": len(sorted_lats),
-                "p50_ms": round(_percentile(sorted_lats, 0.50) * 1000, 2),
-                "p95_ms": round(_percentile(sorted_lats, 0.95) * 1000, 2),
-                "p99_ms": round(_percentile(sorted_lats, 0.99) * 1000, 2),
-            }
-        return {
-            "total_dispatched": self.total_dispatched,
-            "total_sent": self.total_sent,
-            "total_success": self.total_success,
-            "total_errors": self.total_errors,
-            "total_cancelled": self.total_cancelled,
-            "error_rate_pct": (
-                round(self.total_errors / self.total_sent * 100, 2)
-                if self.total_sent
-                else 0
-            ),
-            "delivery_pct": (
-                round(self.total_sent / self.total_dispatched * 100, 2)
-                if self.total_dispatched
-                else 0.0
-            ),
-            "per_command": per_command,
-        }
-
-
-@dataclass
-class Connection:
-    """One open WebSocket endpoint together with its request gate.
-
-    websockets raises rather than mis-delivering when two coroutines call
-    ``recv()`` on the same socket, so every request holds ``gate`` across its
-    send and its matching receive. Parallelism comes from the round-robin
-    spread over endpoints: N endpoints allow N requests in flight.
-
-        run_load ──round-robin──> Connection[0] ─gate─> send_rpc (1 at a time)
-                              └─> Connection[1] ─gate─> send_rpc (1 at a time)
-
-    Attributes:
-        url:  WebSocket URL this connection was opened against, for logging.
-        ws:   The open connection.
-        gate: Limits concurrent send/receive pairs on ``ws`` to
-              MAX_INFLIGHT_PER_CONNECTION.
-    """
-
-    url: str
-    ws: websockets.ClientConnection
-    gate: asyncio.Semaphore = field(
-        default_factory=lambda: asyncio.Semaphore(MAX_INFLIGHT_PER_CONNECTION)
-    )
-
-
-# ---------------------------------------------------------------------------
-# RPC command builders
-# ---------------------------------------------------------------------------
-
-
-def build_rpc_request(command: str) -> dict[str, Any]:
-    """Build a native WebSocket command request for the given command.
-
-    Uses rippled's native WS format (``{"command": ...}``) with flat
-    parameters, NOT the JSON-RPC format (``{"method": ..., "params": [...]}``).
-
-    Args:
-        command: The rippled RPC command name.
-
-    Returns:
-        A dict representing the native WebSocket request body.
-    """
-    req: dict[str, Any] = {"command": command}
-
-    if command in ("server_info", "fee"):
-        pass  # No params needed.
-    elif command == "account_info":
-        req["account"] = GENESIS_ACCOUNT
-    elif command == "account_lines":
-        req["account"] = GENESIS_ACCOUNT
-    elif command == "account_objects":
-        req["account"] = GENESIS_ACCOUNT
-        req["limit"] = 10
-    elif command == "ledger":
-        req["ledger_index"] = "validated"
-    elif command == "ledger_data":
-        req["ledger_index"] = "validated"
-        req["limit"] = 5
-    elif command == "tx":
-        # Use a dummy hash — returns "txnNotFound" error but still exercises
-        # the full RPC span pipeline for this transport (rpc.ws_message ->
-        # rpc.command.tx). rpc.process is not in that chain: it is created
-        # only on the HTTP/JSON-RPC path, which this client never uses.
-        req["transaction"] = "0" * 64
-        req["binary"] = False
-    elif command == "account_tx":
-        req["account"] = GENESIS_ACCOUNT
-        req["ledger_index_min"] = -1
-        req["ledger_index_max"] = -1
-        req["limit"] = 5
-    elif command == "book_offers":
-        req["taker_pays"] = {"currency": "XRP"}
-        req["taker_gets"] = {
-            "currency": "USD",
-            "issuer": GENESIS_ACCOUNT,
-        }
-        req["limit"] = 5
-    elif command == "amm_info":
-        # AMM may not exist — the span is still created on the server side.
-        req["asset"] = {"currency": "XRP"}
-        req["asset2"] = {
-            "currency": "USD",
-            "issuer": GENESIS_ACCOUNT,
-        }
-
-    return req
-
-
-def choose_command(weights: dict[str, int]) -> str:
-    """Select a random RPC command based on configured weights.
-
-    Args:
-        weights: Mapping of command name to relative weight.
-
-    Returns:
-        A command name string.
-    """
-    commands = list(weights.keys())
-    w = [weights[c] for c in commands]
-    return random.choices(commands, weights=w, k=1)[0]
-
-
-# ---------------------------------------------------------------------------
-# WebSocket RPC client
-# ---------------------------------------------------------------------------
-
-
-async def _recv_matching_reply(
-    conn: Connection, request_id: int, command: str
-) -> dict[str, Any]:
-    """Read replies on ``conn`` until the one for ``request_id`` arrives.
-
-    Cancelling a ``recv()`` does not discard the reply it was waiting for: the
-    library queues incoming messages independently of any reader, so a request
-    that hit RECV_TIMEOUT_S leaves its reply in the buffer. The next request on
-    that connection used to read it and time its own round trip against
-    somebody else's reply -- a near-zero latency, and a ``status`` belonging to
-    a different command. The skew was permanent, because every later request on
-    the connection stayed one reply behind for the rest of the run, so a single
-    timeout quietly invalidated that connection's whole latency distribution.
-    Discarding replies by id puts the stream back in step.
-
-    The caller holds ``conn.gate``, so no other request can consume a reply
-    while this loop runs. The deadline covers the whole exchange rather than
-    each message, so a run of buffered replies cannot extend the wait without
-    bound.
-
-    Args:
-        conn:       Connection to read from, with its gate already held.
-        request_id: The ``id`` sent with this request.
-        command:    RPC command name, for logging.
-
-    Returns:
-        The parsed reply whose ``id`` matches, or that carries no ``id``.
-
-    Raises:
-        asyncio.TimeoutError: If no matching reply arrived within
-                              RECV_TIMEOUT_S.
-    """
-    deadline = time.monotonic() + RECV_TIMEOUT_S
-    while True:
-        remaining = deadline - time.monotonic()
-        if remaining <= 0:
-            raise asyncio.TimeoutError(f"{command} (id {request_id}) got no reply")
-        raw = await asyncio.wait_for(conn.ws.recv(), timeout=remaining)
-        reply = json.loads(raw)
-        # A reply that decodes to an array or a scalar has no .get(), and the
-        # AttributeError that follows names neither the command nor what
-        # arrived. Reject it here so the message carries both.
-        if not isinstance(reply, dict):
-            raise TypeError(
-                f"{command} (id {request_id}) got a JSON "
-                f"{type(reply).__name__} reply, expected an object"
-            )
-        # A reply with no id counts as this request's: a few xrpld error paths
-        # answer before the id is parsed, and treating those as stale would
-        # turn a reported error into a timeout.
-        reply_id = reply.get("id")
-        if reply_id is None or reply_id == request_id:
-            return reply
-        logger.debug(
-            "Discarded a reply for id %s while awaiting id %s (%s)",
-            reply_id,
-            request_id,
-            command,
-        )
-
-
-async def send_rpc(
-    conn: Connection,
-    command: str,
-    stats: LoadStats,
-    inject_traceparent: bool = True,
-) -> None:
-    """Send a single RPC request over WebSocket and record the result.
-
-    Holds ``conn.gate`` across the send and the matching receive so only one
-    request at a time owns the connection's ``recv()``. The latency clock
-    starts after the gate is acquired, so time spent queued behind a busy
-    connection is not charged to the server.
-
-    Every outcome is recorded, including failures, so error_rate_pct in the
-    summary reflects every request that was actually sent.
-
-    Args:
-        conn:                Target connection and its request gate.
-        command:             RPC command name.
-        stats:               LoadStats instance to record results.
-        inject_traceparent:  If True, add a W3C traceparent header field
-                             to the request for context propagation testing.
-    """
-    request = build_rpc_request(command)
-    request_id = next(_request_ids)
-    request["id"] = request_id
-
-    # Inject W3C traceparent for context propagation testing.
-    # The rippled WebSocket handler extracts this from the JSON body
-    # when present.
-    if inject_traceparent:
-        trace_id = uuid.uuid4().hex
-        span_id = uuid.uuid4().hex[:16]
-        request["traceparent"] = f"00-{trace_id}-{span_id}-01"
-
-    async with conn.gate:
-        t0 = time.monotonic()
-        # The try covers the I/O and the parse only. Recording sits outside it
-        # so a bug in record() surfaces as the task failure it is, instead of
-        # being counted as one more failed request.
-        try:
-            await conn.ws.send(json.dumps(request))
-            reply = await _recv_matching_reply(conn, request_id, command)
-            latency = time.monotonic() - t0
-            # Native WS responses have {"status": "success", "result": {...}}
-            # or {"status": "error", "error": "...", "error_message": "..."}.
-            success = reply.get("status") == "success"
-        except REQUEST_FAILURES as exc:
-            logger.debug("RPC %s failed: %s", command, exc)
-            # No reply, so no latency sample -- see LoadStats.record().
-            stats.record(command, None, False)
-            return
-        stats.record(command, latency, success)
-
-
-async def open_connections(endpoints: list[str]) -> list[Connection]:
-    """Open one persistent WebSocket connection per endpoint.
-
-    Endpoints that refuse the connection are logged and skipped, so a partly
-    reachable cluster still produces load.
-
-    Args:
-        endpoints: List of WebSocket URLs (ws://host:port).
-
-    Returns:
-        The connections that were established, possibly empty.
-    """
-    connections: list[Connection] = []
-    for ep in endpoints:
-        try:
-            ws = await websockets.connect(ep, ping_interval=20, ping_timeout=10)
-            connections.append(Connection(url=ep, ws=ws))
-            logger.info("Connected to %s", ep)
-        except Exception as exc:
-            logger.error("Failed to connect to %s: %s", ep, exc)
-    return connections
-
-
-async def drain_requests(inflight: set[asyncio.Task[None]]) -> int:
-    """Let in-flight requests finish, then cancel whatever is still stuck.
-
-    A request may wait up to RECV_TIMEOUT_S for its reply, so closing the
-    connections straight away would turn late replies into errors. Anything
-    still unfinished after DRAIN_TIMEOUT_S is cancelled, and the count is
-    returned so the caller can put the shortfall in the summary instead of
-    losing it to a log line.
-
-    Args:
-        inflight: Tasks still tracked as unfinished. Finished tasks remove
-                  themselves, so this is the outstanding set.
-
-    Returns:
-        Number of requests cancelled without being recorded.
-    """
-    pending = {task for task in inflight if not task.done()}
-    if not pending:
-        return 0
-
-    logger.info(
-        "Draining %d in-flight request(s), up to %.0fs...",
-        len(pending),
-        DRAIN_TIMEOUT_S,
-    )
-    _, stuck = await asyncio.wait(pending, timeout=DRAIN_TIMEOUT_S)
-    if not stuck:
-        logger.info("All in-flight requests completed.")
-        return 0
-
-    for task in stuck:
-        task.cancel()
-    await asyncio.gather(*stuck, return_exceptions=True)
-    # Most of these never left the client: they were still waiting for the
-    # per-connection gate. Up to one per connection had already been sent and
-    # was waiting on recv() when it was cancelled, so the server may have
-    # handled it. Either way CancelledError is not a REQUEST_FAILURES member,
-    # so none of them reached stats.record and none are in total_sent.
-    logger.warning(
-        "Cancelled %d request(s) unfinished after %.0fs — not counted in "
-        "total_sent; see total_cancelled and delivery_pct",
-        len(stuck),
-        DRAIN_TIMEOUT_S,
-    )
-    return len(stuck)
-
-
-def log_progress(stats: LoadStats, elapsed: float) -> None:
-    """Log throughput every 100 recorded requests.
-
-    Args:
-        stats:   Live counters.
-        elapsed: Seconds since the run started.
-    """
-    if stats.total_sent % 100 != 0 or stats.total_sent == 0:
-        return
-    logger.info(
-        "Progress: %d sent, %d errors, %.1f RPS (%.0fs elapsed)",
-        stats.total_sent,
-        stats.total_errors,
-        stats.total_sent / elapsed if elapsed > 0 else 0,
-        elapsed,
-    )
-
-
-async def dispatch_requests(
-    connections: list[Connection],
-    rate: float,
-    duration: float,
-    weights: dict[str, int],
-    stats: LoadStats,
-    inject_traceparent: bool,
-) -> None:
-    """Fire requests round-robin at the target rate, then drain them.
-
-    Each request runs as its own task so the dispatch loop keeps its pace
-    regardless of reply latency. Tasks are tracked, not forgotten, so
-    teardown can drain them and so an exception can never escape unseen.
-
-    Every task created counts towards ``stats.total_dispatched`` and every one
-    cancelled at teardown towards ``stats.total_cancelled``, which is what
-    makes an under-delivering run visible in the summary.
-
-    Args:
-        connections:        Open connections to spread requests over.
-        rate:               Target requests per second.
-        duration:           Total run time in seconds.
-        weights:            Command distribution weights.
-        stats:              LoadStats instance to record results in.
-        inject_traceparent: Whether to inject W3C traceparent headers.
-    """
-    interval = 1.0 / rate if rate > 0 else 0.1
-    start = time.monotonic()
-    conn_idx = 0
-    inflight: set[asyncio.Task[None]] = set()
-
-    def reap(task: asyncio.Task[None]) -> None:
-        """Untrack a finished request and report anything that escaped it."""
-        inflight.discard(task)
-        if not task.cancelled() and task.exception() is not None:
-            logger.error("RPC task failed unexpectedly: %s", task.exception())
-
-    try:
-        while (time.monotonic() - start) < duration:
-            conn = connections[conn_idx % len(connections)]
-            conn_idx += 1
-            task = asyncio.create_task(
-                send_rpc(conn, choose_command(weights), stats, inject_traceparent)
-            )
-            inflight.add(task)
-            task.add_done_callback(reap)
-            stats.total_dispatched += 1
-
-            await asyncio.sleep(interval)
-            log_progress(stats, time.monotonic() - start)
-    except asyncio.CancelledError:
-        logger.info("Load generation cancelled.")
-    finally:
-        stats.total_cancelled += await drain_requests(inflight)
-
-
-async def run_load(
-    endpoints: list[str],
-    rate: float,
-    duration: float,
-    weights: dict[str, int],
-    inject_traceparent: bool,
-) -> LoadStats:
-    """Run the RPC load generator against the given endpoints.
-
-    Distributes requests round-robin across endpoints at the specified
-    rate (requests per second) for the given duration. Each connection
-    serves one request at a time, so the ceiling per connection is
-    1 / round-trip-latency requests per second; add endpoints to raise it.
-
-    Args:
-        endpoints:          List of WebSocket URLs (ws://host:port).
-        rate:               Target requests per second.
-        duration:           Total run time in seconds.
-        weights:            Command distribution weights.
-        inject_traceparent: Whether to inject W3C traceparent headers.
-
-    Returns:
-        LoadStats with aggregated results.
-    """
-    stats = LoadStats()
-
-    connections = await open_connections(endpoints)
-    if not connections:
-        logger.error("No connections established. Aborting.")
-        return stats
-
-    logger.info(
-        "Starting load: rate=%s RPS, duration=%ss, endpoints=%d",
-        rate,
-        duration,
-        len(connections),
-    )
-
-    start = time.monotonic()
-    try:
-        await dispatch_requests(
-            connections, rate, duration, weights, stats, inject_traceparent
-        )
-    finally:
-        # Only reached once dispatch_requests has drained: closing a
-        # connection under an outstanding receive raises ConnectionClosed,
-        # which would be recorded as a failed request.
-        for conn in connections:
-            await conn.ws.close()
-
-    elapsed = time.monotonic() - start
-    logger.info(
-        "Load complete: %d of %d dispatched sent, %d success, %d errors, "
-        "%d cancelled in %.1fs (%.1f RPS)",
-        stats.total_sent,
-        stats.total_dispatched,
-        stats.total_success,
-        stats.total_errors,
-        stats.total_cancelled,
-        elapsed,
-        stats.total_sent / elapsed if elapsed > 0 else 0,
-    )
-
-    return stats
-
-
-# ---------------------------------------------------------------------------
-# CLI entry point
-# ---------------------------------------------------------------------------
-
-
-def parse_args() -> argparse.Namespace:
-    """Parse command-line arguments."""
-    parser = argparse.ArgumentParser(
-        description="RPC Load Generator for rippled telemetry validation",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Examples:
-  # Basic usage (50 RPS for 2 minutes):
-  python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120
-
-  # Multiple endpoints with custom weights:
-  python3 rpc_load_generator.py \\
-      --endpoints ws://localhost:6006 ws://localhost:6007 \\
-      --rate 100 --duration 300 \\
-      --weights '{"server_info": 80, "account_info": 20}'
-        """,
-    )
-    parser.add_argument(
-        "--endpoints",
-        nargs="+",
-        default=["ws://localhost:6006"],
-        help="WebSocket endpoints (default: ws://localhost:6006)",
-    )
-    parser.add_argument(
-        "--rate",
-        type=float,
-        default=50.0,
-        help="Target requests per second (default: 50)",
-    )
-    parser.add_argument(
-        "--duration",
-        type=float,
-        default=120.0,
-        help="Run duration in seconds (default: 120)",
-    )
-    parser.add_argument(
-        "--weights",
-        type=str,
-        default=None,
-        help="JSON string of command weights (overrides defaults)",
-    )
-    parser.add_argument(
-        "--no-traceparent",
-        action="store_true",
-        help="Disable W3C traceparent injection",
-    )
-    parser.add_argument(
-        "--output",
-        type=str,
-        default=None,
-        help="Write JSON summary to this file path",
-    )
-    parser.add_argument(
-        "--verbose",
-        action="store_true",
-        help="Enable debug logging",
-    )
-    return parser.parse_args()
-
-
-def main() -> None:
-    """Main entry point for the RPC load generator."""
-    args = parse_args()
-
-    logging.basicConfig(
-        level=logging.DEBUG if args.verbose else logging.INFO,
-        format="%(asctime)s [%(name)s] %(levelname)s %(message)s",
-    )
-
-    # Parse custom weights if provided.
-    weights = DEFAULT_WEIGHTS.copy()
-    if args.weights:
-        try:
-            custom = json.loads(args.weights)
-            # A JSON array or scalar decodes fine and then has no .items().
-            # AttributeError is not a ValueError, so the handler below would not
-            # catch it: report the type that arrived instead of a traceback.
-            if not isinstance(custom, dict):
-                logger.error(
-                    "Invalid --weights: expected a JSON object of command to "
-                    "weight, got a JSON %s: %s",
-                    type(custom).__name__,
-                    args.weights,
-                )
-                sys.exit(1)
-            # TypeError covers a non-numeric weight value (null, a list, an
-            # object), which int() rejects with TypeError rather than ValueError.
-            weights = {k: int(v) for k, v in custom.items()}
-            if not weights or sum(weights.values()) <= 0:
-                logger.error(
-                    "Invalid --weights: the values must sum to more than 0, got %s",
-                    weights,
-                )
-                sys.exit(1)
-            logger.info("Using custom weights: %s", weights)
-        except (json.JSONDecodeError, TypeError, ValueError) as exc:
-            logger.error("Invalid --weights JSON: %s", exc)
-            sys.exit(1)
-
-    # Run the load generator.
-    stats = asyncio.run(
-        run_load(
-            endpoints=args.endpoints,
-            rate=args.rate,
-            duration=args.duration,
-            weights=weights,
-            inject_traceparent=not args.no_traceparent,
-        )
-    )
-
-    summary = stats.summary()
-    print(json.dumps(summary, indent=2))
-
-    if args.output:
-        with open(args.output, "w") as f:
-            json.dump(summary, f, indent=2)
-        logger.info("Summary written to %s", args.output)
-
-    # Both gates are evaluated and reported before exiting, and the summary is
-    # already on disk, so the caller sees every reason plus the numbers behind
-    # it. A run that under-delivers has to fail as loudly as one that errors:
-    # every downstream span and metric assertion would otherwise be checked
-    # against a fraction of the intended traffic and still look healthy.
-    failures: list[str] = []
-    if summary["error_rate_pct"] > MAX_ERROR_RATE_PCT:
-        failures.append(
-            "error rate %.2f%% exceeds %.0f%% (%d of %d requests failed)"
-            % (
-                summary["error_rate_pct"],
-                MAX_ERROR_RATE_PCT,
-                summary["total_errors"],
-                summary["total_sent"],
-            )
-        )
-    if summary["delivery_pct"] < MIN_DELIVERY_PCT:
-        failures.append(
-            "delivered %.2f%% of dispatched requests, below %.0f%% "
-            "(%d sent, %d cancelled, of %d dispatched) — lower --rate or add "
-            "endpoints"
-            % (
-                summary["delivery_pct"],
-                MIN_DELIVERY_PCT,
-                summary["total_sent"],
-                summary["total_cancelled"],
-                summary["total_dispatched"],
-            )
-        )
-    for reason in failures:
-        logger.error("%s", reason)
-    if failures:
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()