docs(telemetry): add OTel-overhead benchmarking runbook

Documents how to measure rippled's OpenTelemetry overhead via the
ripple/perf-iac comparison pipeline (telemetry on vs off, identical load,
eBPF profile diff), the exact trigger command, the option to dispatch from
rippled CI (and its cross-org token blocker), and the lessons learned
(gitignored role tasks file, sequential matrix legs).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.codecov.yml

@@ -36,3 +36,8 @@ ignore:
   - "src/tests/"
   - "include/xrpl/beast/test/"
   - "include/xrpl/beast/unit_test/"
+  # Telemetry modules — conditionally compiled behind XRPL_ENABLE_TELEMETRY,
+  # which is not enabled in coverage builds.
+  - "src/xrpld/telemetry/"
+  - "src/libxrpl/beast/insight/OTelCollector.cpp"
+  - "include/xrpl/beast/insight/OTelCollector.h"

.github/scripts/levelization/results/loops.txt

@@ -25,3 +25,6 @@ Loop: xrpld.app xrpld.telemetry
 Loop: xrpld.overlay xrpld.rpc
   xrpld.rpc ~= xrpld.overlay
 
+Loop: xrpld.overlay xrpld.telemetry
+  xrpld.telemetry ~= xrpld.overlay
+

.github/scripts/levelization/results/ordering.txt

@@ -104,6 +104,7 @@ test.csf > xrpld.consensus
 test.csf > xrpl.json
 test.csf > xrpl.ledger
 test.csf > xrpl.protocol
+test.csf > xrpl.telemetry
 test.json > test.jtx
 test.json > xrpl.json
 test.jtx > xrpl.basics
@@ -192,6 +193,7 @@ test.unit_test > xrpl.basics
 test.unit_test > xrpl.protocol
 tests.libxrpl > xrpl.basics
 tests.libxrpl > xrpl.core
+tests.libxrpl > xrpld.telemetry
 tests.libxrpl > xrpl.json
 tests.libxrpl > xrpl.ledger
 tests.libxrpl > xrpl.net
@@ -272,7 +274,6 @@ xrpld.overlay > xrpl.core
 xrpld.overlay > xrpld.consensus
 xrpld.overlay > xrpld.core
 xrpld.overlay > xrpld.peerfinder
-xrpld.overlay > xrpld.telemetry
 xrpld.overlay > xrpl.json
 xrpld.overlay > xrpl.ledger
 xrpld.overlay > xrpl.protocol
@@ -288,6 +289,7 @@ xrpld.peerfinder > xrpl.rdb
 xrpld.perflog > xrpl.basics
 xrpld.perflog > xrpl.core
 xrpld.perflog > xrpld.rpc
+xrpld.perflog > xrpld.telemetry
 xrpld.perflog > xrpl.json
 xrpld.perflog > xrpl.protocol
 xrpld.rpc > xrpl.basics
@@ -308,4 +310,11 @@ xrpld.shamap > xrpld.core
 xrpld.shamap > xrpl.protocol
 xrpld.shamap > xrpl.shamap
 xrpld.telemetry > xrpl.basics
+xrpld.telemetry > xrpl.core
+xrpld.telemetry > xrpld.core
+xrpld.telemetry > xrpl.json
+xrpld.telemetry > xrpl.nodestore
+xrpld.telemetry > xrpl.protocol
+xrpld.telemetry > xrpl.rdb
+xrpld.telemetry > xrpl.server
 xrpld.telemetry > xrpl.telemetry

.github/scripts/rename/README.md

@@ -1,11 +1,11 @@
 ## Renaming ripple(d) to xrpl(d)
 
 In the initial phases of development of the XRPL, the open source codebase was
-called "rippled" and it remains with that name even today. Today, over 1000
+called "xrpld" and it remains with that name even today. Today, over 1000
 nodes run the application, and code contributions have been submitted by
 developers located around the world. The XRPL community is larger than ever.
 In light of the decentralized and diversified nature of XRPL, we will rename any
-references to `ripple` and `rippled` to `xrpl` and `xrpld`, when appropriate.
+references to `ripple` and `xrpld` to `xrpl` and `xrpld`, when appropriate.
 
 See [here](https://xls.xrpl.org/xls/XLS-0095-rename-rippled-to-xrpld.html) for
 more information.
@@ -22,17 +22,17 @@ run from the repository root.
 2. `.github/scripts/rename/copyright.sh`: This script will remove superfluous
    copyright notices.
 3. `.github/scripts/rename/cmake.sh`: This script will rename all CMake files
-   from `RippleXXX.cmake` or `RippledXXX.cmake` to `XrplXXX.cmake`, and any
-   references to `ripple` and `rippled` (with or without capital letters) to
+   from `RippleXXX.cmake` or `XrpldXXX.cmake` to `XrplXXX.cmake`, and any
+   references to `ripple` and `xrpld` (with or without capital letters) to
    `xrpl` and `xrpld`, respectively. The name of the binary will remain as-is,
    and will only be renamed to `xrpld` by a later script.
 4. `.github/scripts/rename/binary.sh`: This script will rename the binary from
-   `rippled` to `xrpld`, and reverses the symlink so that `rippled` points to
+   `xrpld` to `xrpld`, and reverses the symlink so that `xrpld` points to
    the `xrpld` binary.
 5. `.github/scripts/rename/namespace.sh`: This script will rename the C++
    namespaces from `ripple` to `xrpl`.
 6. `.github/scripts/rename/config.sh`: This script will rename the config from
-   `rippled.cfg` to `xrpld.cfg`, and updating the code accordingly. The old
+   `xrpld.cfg` to `xrpld.cfg`, and updating the code accordingly. The old
    filename will still be accepted.
 7. `.github/scripts/rename/docs.sh`: This script will rename any lingering
    references of `ripple(d)` to `xrpl(d)` in code, comments, and documentation.

.github/workflows/telemetry-validation.yml

@@ -0,0 +1,310 @@
+# Telemetry Validation CI Workflow
+#
+# Builds rippled with telemetry enabled, runs the multi-node workload
+# harness, validates all telemetry data, and runs performance benchmarks.
+#
+# This is a separate workflow from the main CI. It runs:
+#   - On manual dispatch (workflow_dispatch)
+#   - On pushes to telemetry-related branches
+#
+# The workflow is intentionally heavyweight (builds rippled, starts Docker
+# services, runs a multi-node cluster) — it validates the full telemetry
+# stack end-to-end rather than individual unit tests.
+#
+# Architecture: two jobs to leverage cached dependencies:
+#   1. build-xrpld — runs on a self-hosted runner inside the same container
+#      image the main CI uses (debian-bookworm-gcc-13). This ensures Conan
+#      packages are fetched from the XRPLF remote instead of built from
+#      source, and ccache hits the remote cache.
+#   2. validate-telemetry — runs on ubuntu-latest (which has Docker) to
+#      launch the telemetry stack (OTel collector, Prometheus, Tempo, etc.)
+#      and validate the full pipeline end-to-end.
+
+name: Telemetry Validation
+
+on:
+  workflow_dispatch:
+    inputs:
+      rpc_rate:
+        description: "RPC load rate (requests per second)"
+        required: false
+        default: "50"
+      rpc_duration:
+        description: "RPC load duration (seconds)"
+        required: false
+        default: "120"
+      tx_tps:
+        description: "Transaction submit rate (TPS)"
+        required: false
+        default: "5"
+      tx_duration:
+        description: "Transaction submit duration (seconds)"
+        required: false
+        default: "120"
+      run_benchmark:
+        description: "Run performance benchmarks"
+        required: false
+        type: boolean
+        default: false
+
+  push:
+    branches:
+      - "pratik/otel-phase*"
+      - "feature/otel-*"
+      - "feature/telemetry-*"
+    paths:
+      - ".github/workflows/telemetry-validation.yml"
+      - "docker/telemetry/**"
+      - "include/xrpl/basics/Telemetry*.h"
+      - "src/xrpld/app/misc/Telemetry*"
+
+concurrency:
+  group: telemetry-validation-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  BUILD_DIR: build
+
+jobs:
+  # ── Job 1: Build xrpld in the same container the main CI uses ──────
+  # This ensures Conan binary packages are fetched from the XRPLF remote
+  # (matching package IDs) and ccache hits the remote compilation cache.
+  build-xrpld:
+    name: Build xrpld
+    runs-on: [self-hosted, Linux, X64, heavy]
+    container: ghcr.io/xrplf/ci/debian-bookworm:gcc-13-sha-ab4d1f0
+    timeout-minutes: 60
+    env:
+      CCACHE_NAMESPACE: telemetry-validation
+      CCACHE_REMOTE_ONLY: true
+      CCACHE_REMOTE_STORAGE: http://cache.dev.ripplex.io:8080|layout=bazel
+      CCACHE_SLOPPINESS: include_file_ctime,include_file_mtime
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Prepare runner
+        uses: XRPLF/actions/prepare-runner@90f11ee655d1687824fb8793db770477d52afbab
+        with:
+          enable_ccache: ${{ github.repository_owner == 'XRPLF' }}
+
+      - name: Print build environment
+        uses: XRPLF/actions/print-build-env@59dec886e4afb05a1724443af08baccbc045b574
+
+      - name: Get number of processors
+        uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf
+        id: nproc
+        with:
+          subtract: 2
+
+      - name: Setup Conan
+        uses: ./.github/actions/setup-conan
+
+      - name: Build dependencies
+        uses: ./.github/actions/build-deps
+        with:
+          build_nproc: ${{ steps.nproc.outputs.nproc }}
+          build_type: Release
+          log_verbosity: verbose
+
+      - name: Configure CMake
+        working-directory: ${{ env.BUILD_DIR }}
+        run: |
+          cmake \
+              -G Ninja \
+              -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
+              -DCMAKE_BUILD_TYPE=Release \
+              ..
+
+      - name: Build xrpld
+        working-directory: ${{ env.BUILD_DIR }}
+        env:
+          BUILD_NPROC: ${{ steps.nproc.outputs.nproc }}
+        run: |
+          cmake \
+              --build . \
+              --config Release \
+              --parallel "${BUILD_NPROC}" \
+              --target xrpld
+
+      - name: Show ccache statistics
+        if: ${{ github.repository_owner == 'XRPLF' }}
+        run: ccache --show-stats -vv
+
+      - name: Upload xrpld binary
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: xrpld-telemetry
+          path: ${{ env.BUILD_DIR }}/xrpld
+          retention-days: 1
+          if-no-files-found: error
+
+  # ── Job 2: Run telemetry validation on ubuntu-latest (has Docker) ──
+  validate-telemetry:
+    name: Telemetry Stack Validation
+    needs: build-xrpld
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Install Python dependencies
+        run: pip3 install -r docker/telemetry/workload/requirements.txt
+
+      - name: Download xrpld binary
+        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
+        with:
+          name: xrpld-telemetry
+          path: ${{ env.BUILD_DIR }}
+
+      - name: Make binaries and scripts executable
+        run: |
+          chmod +x ${{ env.BUILD_DIR }}/xrpld
+          chmod +x docker/telemetry/workload/*.sh
+
+      - name: Run full telemetry validation
+        id: validation
+        env:
+          RPC_RATE: ${{ github.event.inputs.rpc_rate || '50' }}
+          RPC_DURATION: ${{ github.event.inputs.rpc_duration || '120' }}
+          TX_TPS: ${{ github.event.inputs.tx_tps || '5' }}
+          TX_DURATION: ${{ github.event.inputs.tx_duration || '120' }}
+          RUN_BENCHMARK: ${{ github.event.inputs.run_benchmark }}
+        run: |
+          ARGS="--xrpld ${{ env.BUILD_DIR }}/xrpld --skip-loki"
+          ARGS="$ARGS --rpc-rate $RPC_RATE"
+          ARGS="$ARGS --rpc-duration $RPC_DURATION"
+          ARGS="$ARGS --tx-tps $TX_TPS"
+          ARGS="$ARGS --tx-duration $TX_DURATION"
+          if [ "$RUN_BENCHMARK" = "true" ]; then
+              ARGS="$ARGS --with-benchmark"
+          fi
+          docker/telemetry/workload/run-full-validation.sh $ARGS
+        # continue-on-error allows subsequent steps (artifact upload,
+        # summary printing) to run even if validation fails. The final
+        # "Check validation result" step re-checks steps.validation.outcome
+        # (the pre-continue-on-error result) and fails the job properly.
+        continue-on-error: true
+
+      - name: Upload validation reports
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: telemetry-validation-reports
+          path: /tmp/xrpld-validation/reports/
+          retention-days: 30
+
+      - name: Upload node logs
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: xrpld-node-logs
+          path: /tmp/xrpld-validation/node*/debug.log
+          retention-days: 7
+
+      - name: Print validation summary
+        if: always()
+        run: |
+          REPORT="/tmp/xrpld-validation/reports/validation-report.json"
+          if [ -f "$REPORT" ]; then
+              echo "## Telemetry Validation Results" >>"$GITHUB_STEP_SUMMARY"
+              echo "" >>"$GITHUB_STEP_SUMMARY"
+              TOTAL=$(jq '.summary.total' "$REPORT")
+              PASSED=$(jq '.summary.passed' "$REPORT")
+              FAILED=$(jq '.summary.failed' "$REPORT")
+              echo "| Metric | Value |" >>"$GITHUB_STEP_SUMMARY"
+              echo "|--------|-------|" >>"$GITHUB_STEP_SUMMARY"
+              echo "| Total Checks | $TOTAL |" >>"$GITHUB_STEP_SUMMARY"
+              echo "| Passed | $PASSED |" >>"$GITHUB_STEP_SUMMARY"
+              echo "| Failed | $FAILED |" >>"$GITHUB_STEP_SUMMARY"
+              echo "" >>"$GITHUB_STEP_SUMMARY"
+              if [ "$FAILED" -gt 0 ]; then
+                  echo "### Failed Checks" >>"$GITHUB_STEP_SUMMARY"
+                  echo "" >>"$GITHUB_STEP_SUMMARY"
+                  jq -r '.checks[] | select(.passed == false) | "- **\(.name)**: \(.message)"' "$REPORT" >>"$GITHUB_STEP_SUMMARY"
+              fi
+          fi
+
+      # Publishes captured OTel timings + regression report to the Step Summary.
+      # When the committed baseline is a placeholder, emits a fenced JSON block
+      # that can be copy-pasted directly into baselines/baseline-timings.json.
+      # When the baseline is populated, summarises the top regressions so the
+      # PR author sees the failure reason without downloading artifacts.
+      - name: Print regression summary
+        if: always()
+        run: |
+          set -euo pipefail
+          TIMINGS="/tmp/xrpld-validation/reports/timings.json"
+          REGRESSION="/tmp/xrpld-validation/reports/regression-report.json"
+          BASELINE="docker/telemetry/workload/baselines/baseline-timings.json"
+
+          if [ ! -f "$TIMINGS" ]; then
+              echo "## Regression Gate: no timings captured" >>"$GITHUB_STEP_SUMMARY"
+              echo "::warning::capture_timings.py did not produce timings.json — regression gate was not evaluated."
+              exit 0
+          fi
+
+          if [ ! -f "$BASELINE" ]; then
+              echo "## Regression Gate: baseline file missing" >>"$GITHUB_STEP_SUMMARY"
+              echo "::error::baselines/baseline-timings.json not found in checkout"
+              exit 1
+          fi
+
+          # NOTE: do NOT use `jq -e` here. With -e, jq exits non-zero when the
+          # filter's result is boolean false — which is the normal case for a
+          # populated (non-placeholder) baseline — and that would be
+          # misreported as a parse failure. Plain `jq -r` exits 0 on any valid
+          # JSON, so a real non-zero exit genuinely means malformed JSON.
+          IS_PLACEHOLDER=$(jq -r '.placeholder == true or (.metrics | length == 0)' "$BASELINE") || {
+              echo "::error::Failed to parse baseline JSON"
+              exit 1
+          }
+
+          echo "## OTel Timings Regression Gate" >>"$GITHUB_STEP_SUMMARY"
+          echo "" >>"$GITHUB_STEP_SUMMARY"
+
+          if [ "$IS_PLACEHOLDER" = "true" ]; then
+              echo "### Paste into \`baselines/baseline-timings.json\`" >>"$GITHUB_STEP_SUMMARY"
+              echo "" >>"$GITHUB_STEP_SUMMARY"
+              echo "The committed baseline is a placeholder. Open a PR replacing" \
+                  "its contents with the JSON block below to activate the" \
+                  "regression gate." >>"$GITHUB_STEP_SUMMARY"
+              echo "" >>"$GITHUB_STEP_SUMMARY"
+              echo '```json' >>"$GITHUB_STEP_SUMMARY"
+              cat "$TIMINGS" >>"$GITHUB_STEP_SUMMARY"
+              echo '```' >>"$GITHUB_STEP_SUMMARY"
+          elif [ -f "$REGRESSION" ]; then
+              REGR_COUNT=$(jq -e '.summary.regressions' "$REGRESSION") || REGR_COUNT=0
+              IMPR_COUNT=$(jq -e '.summary.improvements' "$REGRESSION") || IMPR_COUNT=0
+              TOTAL=$(jq -e '.summary.total' "$REGRESSION") || TOTAL=0
+              echo "| Stat | Count |" >>"$GITHUB_STEP_SUMMARY"
+              echo "|------|-------|" >>"$GITHUB_STEP_SUMMARY"
+              echo "| Metrics compared | $TOTAL |" >>"$GITHUB_STEP_SUMMARY"
+              echo "| Regressions | $REGR_COUNT |" >>"$GITHUB_STEP_SUMMARY"
+              echo "| Improvements | $IMPR_COUNT |" >>"$GITHUB_STEP_SUMMARY"
+              echo "" >>"$GITHUB_STEP_SUMMARY"
+              if [ "$REGR_COUNT" -gt 0 ]; then
+                  echo "### Regressions" >>"$GITHUB_STEP_SUMMARY"
+                  echo "" >>"$GITHUB_STEP_SUMMARY"
+                  echo "| Metric | Baseline | Current | Δ | % | Unit |" >>"$GITHUB_STEP_SUMMARY"
+                  echo "|--------|---------:|--------:|--:|--:|------|" >>"$GITHUB_STEP_SUMMARY"
+                  jq -r '.metrics[] | select(.regressed) | "| \(.key) | \(.baseline) | \(.current) | \(.delta) | \(.pct_change)% | \(.unit) |"' \
+                      "$REGRESSION" >>"$GITHUB_STEP_SUMMARY"
+              fi
+          fi
+
+      - name: Cleanup
+        if: always()
+        run: |
+          docker/telemetry/workload/run-full-validation.sh --cleanup 2>/dev/null || true
+
+      - name: Check validation result
+        if: steps.validation.outcome == 'failure'
+        run: |
+          echo "Telemetry validation failed. Check the uploaded reports for details."
+          exit 1

.gitignore

@@ -86,3 +86,5 @@ __pycache__
 
 # clangd cache
 /.cache
+docker/telemetry/workload/__pycache__/
+.claude/

OpenTelemetryPlan/02-design-decisions.md

@@ -324,9 +324,9 @@ resource::SemanticConventions::SERVICE_INSTANCE_ID = <node_public_key_base58>
 #### TxQ Attributes
 
 ```cpp
-"xrpl.txq.queue_depth"      = int64    // Current queue depth
-"xrpl.txq.fee_level"        = int64    // Fee level of transaction
-"xrpl.txq.eviction_reason"  = string   // Why transaction was evicted
+"queue_depth"      = int64    // Current queue depth (planned, not yet implemented)
+"fee_level"        = int64    // Fee level of transaction (planned, not yet implemented)
+"eviction_reason"  = string   // Why transaction was evicted (planned, not yet implemented)
 ```
 
 #### Fee Attributes
@@ -659,6 +659,8 @@ span->SetAttribute("peer.id", peerId);
 
 ### 2.6.4 Coexistence Strategy
 
+> **Note**: Phase 7 replaces the StatsD bridge with native OTel Metrics SDK export. The diagram below shows the Phase 6 intermediate state. See [Phase7_taskList.md](./Phase7_taskList.md) for the migration design where Beast Insight emits via OTLP instead of StatsD.
+
 ```mermaid
 flowchart TB
     subgraph xrpld["xrpld Process"]
@@ -687,6 +689,8 @@ flowchart TB
 - **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/gRPC to a Collector, which then forwards to a trace backend (Tempo).
 - **Grafana (red, unified UI)**: All three data streams converge in Grafana, enabling operators to correlate logs, metrics, and traces in a single dashboard.
 
+**Phase 7 target state**: Beast Insight routes to `OTelCollector` (new `Collector` implementation) which exports via OTLP/HTTP to the same collector endpoint as traces. StatsD UDP path becomes a deprecated fallback (`[insight] server=statsd`). See [06-implementation-phases.md §6.8](./06-implementation-phases.md) and [Phase7_taskList.md](./Phase7_taskList.md) for details.
+
 ### 2.6.5 Correlation with PerfLog
 
 Trace IDs can be correlated with existing PerfLog entries for comprehensive debugging:

OpenTelemetryPlan/05-configuration-reference.md

@@ -933,18 +933,22 @@ jsonData:
     filterBySpanID: false
 ```
 
-### 5.8.7 Correlation with Insight/StatsD Metrics
+### 5.8.7 Correlation with Insight/OTel System Metrics
 
-To correlate traces with existing Beast Insight metrics:
+To correlate traces with Beast Insight system metrics:
 
 **Step 1: Export Insight metrics to Prometheus**
 
-```yaml
-# prometheus.yaml
-scrape_configs:
-  - job_name: "xrpld-statsd"
-    static_configs:
-      - targets: ["statsd-exporter:9102"]
+Beast Insight metrics are exported natively via OTLP to the OTel Collector,
+which exposes them on the Prometheus endpoint alongside spanmetrics. No
+separate StatsD exporter is needed when using `server=otel`.
+
+```ini
+# xrpld.cfg — native OTel metrics (recommended)
+[insight]
+server=otel
+endpoint=http://localhost:4318/v1/metrics
+prefix=xrpld
 ```
 
 **Step 2: Add exemplars to metrics**

OpenTelemetryPlan/08-appendix.md

@@ -43,6 +43,18 @@
 | **LoadManager**   | Dynamic fee escalation based on network load                  |
 | **SHAMap**        | SHA-256 hash-based map (Merkle trie variant) for ledger state |
 
+### Phase 9–11 Terms
+
+| Term                        | Definition                                                                |
+| --------------------------- | ------------------------------------------------------------------------- |
+| **MetricsRegistry**         | Centralized class for OTel async gauge registrations (Phase 9)            |
+| **ObservableGauge**         | OTel Metrics SDK async instrument polled via callback at fixed intervals  |
+| **PeriodicMetricReader**    | OTel SDK component that invokes gauge callbacks at configurable intervals |
+| **CountedObject**           | xrpld template that tracks live instance counts via atomic counters       |
+| **TxQ**                     | Transaction queue managing fee escalation and ordering                    |
+| **Load Factor**             | Combined multiplier affecting transaction cost (local, cluster, network)  |
+| **OTel Collector Receiver** | Custom Go plugin that polls xrpld RPC and emits OTel metrics (Phase 11)   |
+
 ---
 
 ## 8.2 Span Hierarchy Visualization
@@ -162,7 +174,8 @@ flowchart TB
 | ------- | ---------- | ------ | -------------------------------------------------------------- |
 | 1.0     | 2026-02-12 | -      | Initial implementation plan                                    |
 | 1.1     | 2026-02-13 | -      | Refactored into modular documents                              |
-| 1.2     | 2026-03-24 | -      | Review fixes: accuracy corrections, cross-document consistency |
+| 1.2     | 2026-03-09 | -      | Added Phases 9–11 (future enhancement plans)                   |
+| 1.3     | 2026-03-24 | -      | Review fixes: accuracy corrections, cross-document consistency |
 
 ---
 
@@ -170,31 +183,83 @@ flowchart TB
 
 ### Plan Documents
 
-| Document                                                         | Description                                        |
-| ---------------------------------------------------------------- | -------------------------------------------------- |
-| [OpenTelemetryPlan.md](./OpenTelemetryPlan.md)                   | Master overview and executive summary              |
-| [00-tracing-fundamentals.md](./00-tracing-fundamentals.md)       | Distributed tracing concepts and OTel primer       |
-| [01-architecture-analysis.md](./01-architecture-analysis.md)     | xrpld architecture and trace points                |
-| [02-design-decisions.md](./02-design-decisions.md)               | SDK selection, exporters, span conventions         |
-| [03-implementation-strategy.md](./03-implementation-strategy.md) | Directory structure, performance analysis          |
-| [04-code-samples.md](./04-code-samples.md)                       | C++ code examples for all components               |
-| [05-configuration-reference.md](./05-configuration-reference.md) | xrpld config, CMake, Collector configs             |
-| [06-implementation-phases.md](./06-implementation-phases.md)     | Timeline, tasks, risks, success metrics            |
-| [07-observability-backends.md](./07-observability-backends.md)   | Backend selection and architecture                 |
-| [08-appendix.md](./08-appendix.md)                               | Glossary, references, version history              |
-| [secure-OTel.md](./secure-OTel.md)                               | Threat model and hardening (mTLS, peer validation) |
-| [presentation.md](./presentation.md)                             | Slide deck for OTel plan overview                  |
+| Document                                                             | Description                                        |
+| -------------------------------------------------------------------- | -------------------------------------------------- |
+| [OpenTelemetryPlan.md](./OpenTelemetryPlan.md)                       | Master overview and executive summary              |
+| [00-tracing-fundamentals.md](./00-tracing-fundamentals.md)           | Distributed tracing concepts and OTel primer       |
+| [01-architecture-analysis.md](./01-architecture-analysis.md)         | xrpld architecture and trace points                |
+| [02-design-decisions.md](./02-design-decisions.md)                   | SDK selection, exporters, span conventions         |
+| [03-implementation-strategy.md](./03-implementation-strategy.md)     | Directory structure, performance analysis          |
+| [04-code-samples.md](./04-code-samples.md)                           | C++ code examples for all components               |
+| [05-configuration-reference.md](./05-configuration-reference.md)     | xrpld config, CMake, Collector configs             |
+| [06-implementation-phases.md](./06-implementation-phases.md)         | Timeline, tasks, risks, success metrics            |
+| [07-observability-backends.md](./07-observability-backends.md)       | Backend selection and architecture                 |
+| [08-appendix.md](./08-appendix.md)                                   | Glossary, references, version history              |
+| [secure-OTel.md](./secure-OTel.md)                                   | Threat model and hardening (mTLS, peer validation) |
+| [09-data-collection-reference.md](./09-data-collection-reference.md) | Span/metric/dashboard inventory                    |
+| [presentation.md](./presentation.md)                                 | Slide deck for OTel plan overview                  |
 
 ### Task Lists
 
-| Document                                   | Description                                         |
-| ------------------------------------------ | --------------------------------------------------- |
-| [POC_taskList.md](./POC_taskList.md)       | Proof-of-concept telemetry integration              |
-| [Phase2_taskList.md](./Phase2_taskList.md) | RPC layer trace instrumentation                     |
-| [Phase3_taskList.md](./Phase3_taskList.md) | Peer overlay & consensus tracing                    |
-| [Phase4_taskList.md](./Phase4_taskList.md) | Transaction lifecycle tracing                       |
-| [Phase5_taskList.md](./Phase5_taskList.md) | Ledger processing & advanced tracing                |
-| [presentation.md](./presentation.md)       | Presentation slides for OpenTelemetry plan overview |
+| Document                                                                   | Description                                         |
+| -------------------------------------------------------------------------- | --------------------------------------------------- |
+| [POC_taskList.md](./POC_taskList.md)                                       | Proof-of-concept telemetry integration              |
+| [Phase2_taskList.md](./Phase2_taskList.md)                                 | RPC layer trace instrumentation                     |
+| [Phase3_taskList.md](./Phase3_taskList.md)                                 | Peer overlay & consensus tracing                    |
+| [Phase4_taskList.md](./Phase4_taskList.md)                                 | Transaction lifecycle tracing                       |
+| [Phase5_taskList.md](./Phase5_taskList.md)                                 | Ledger processing & advanced tracing                |
+| [Phase5_IntegrationTest_taskList.md](./Phase5_IntegrationTest_taskList.md) | Observability stack integration tests               |
+| [Phase7_taskList.md](./Phase7_taskList.md)                                 | Native OTel metrics migration                       |
+| [Phase8_taskList.md](./Phase8_taskList.md)                                 | Log-trace correlation                               |
+| [Phase9_taskList.md](./Phase9_taskList.md)                                 | Internal metric instrumentation gap fill (future)   |
+| [Phase10_taskList.md](./Phase10_taskList.md)                               | Synthetic workload generation & validation (future) |
+| [Phase11_taskList.md](./Phase11_taskList.md)                               | Third-party data collection pipelines (future)      |
+| [presentation.md](./presentation.md)                                       | Presentation slides for OpenTelemetry plan overview |
+
+> **Note**: Phases 1 and 6 do not have separate task list files. Phase 1 tasks are documented in [06-implementation-phases.md §6.2](./06-implementation-phases.md). Phase 6 tasks are documented in [06-implementation-phases.md §6.7](./06-implementation-phases.md).
+
+---
+
+## 8.6 Phase 9–11 Cross-Reference Guide
+
+This guide maps Phase 9–11 content to its location across the documentation.
+
+### Phase 9: Internal Metric Instrumentation Gap Fill
+
+| Content                         | Location                                                                 |
+| ------------------------------- | ------------------------------------------------------------------------ |
+| Plan & architecture             | [06-implementation-phases.md §6.8.2](./06-implementation-phases.md)      |
+| Task list (10 tasks)            | [Phase9_taskList.md](./Phase9_taskList.md)                               |
+| Future metric definitions (~50) | [09-data-collection-reference.md §5b](./09-data-collection-reference.md) |
+| New class: `MetricsRegistry`    | `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (planned)                   |
+| New dashboards                  | `xrpld-fee-market`, `xrpld-job-queue` (planned)                          |
+
+**Metric categories**: NodeStore I/O, Cache Hit Rates, TxQ, PerfLog Per-RPC, PerfLog Per-Job, Counted Objects, Fee Escalation & Load Factors.
+
+### Phase 10: Synthetic Workload Generation & Telemetry Validation
+
+| Content              | Location                                                                 |
+| -------------------- | ------------------------------------------------------------------------ |
+| Plan & architecture  | [06-implementation-phases.md §6.8.3](./06-implementation-phases.md)      |
+| Task list (7 tasks)  | [Phase10_taskList.md](./Phase10_taskList.md)                             |
+| Validation inventory | [09-data-collection-reference.md §5c](./09-data-collection-reference.md) |
+| Test harness         | `docker/telemetry/docker-compose.workload.yaml` (planned)                |
+| CI workflow          | `.github/workflows/telemetry-validation.yml` (planned)                   |
+
+**Validates**: 16 spans, 22 attributes, 300+ metrics, 10 dashboards, log-trace correlation.
+
+### Phase 11: Third-Party Data Collection Pipelines
+
+| Content                           | Location                                                                 |
+| --------------------------------- | ------------------------------------------------------------------------ |
+| Plan & architecture               | [06-implementation-phases.md §6.8.4](./06-implementation-phases.md)      |
+| Task list (11 tasks)              | [Phase11_taskList.md](./Phase11_taskList.md)                             |
+| External metric definitions (~30) | [09-data-collection-reference.md §5d](./09-data-collection-reference.md) |
+| Custom OTel Collector receiver    | `docker/telemetry/otel-rippled-receiver/` (planned)                      |
+| Prometheus alerting rules (11)    | [09-data-collection-reference.md §5d](./09-data-collection-reference.md) |
+| New dashboards (4)                | Validator Health, Network Topology, Fee Market (External), DEX & AMM     |
+
+**Consumer categories**: Exchanges, Payment Processors, DeFi/AMM, NFT Marketplaces, Analytics Providers, Wallets, Compliance, Academic Researchers, Institutional Custody, CBDC Bridge Operators.
 
 ---
 

OpenTelemetryPlan/Benchmarking_OTel_Overhead.md

@@ -0,0 +1,144 @@
+<!-- cspell:ignore ondemand otelcol -->
+
+# Benchmarking OpenTelemetry Overhead
+
+How to empirically measure the runtime cost of rippled's OpenTelemetry
+instrumentation, using the `ripple/perf-iac` performance pipeline.
+
+> **Tracking:** [RIPD-7155](https://ripplelabs.atlassian.net/browse/RIPD-7155)
+> (under epic RIPD-5060).
+
+---
+
+## What is measured
+
+A perf-iac **Performance Comparison** run builds and deploys two rippled
+clusters on dedicated EC2, drives identical JMeter payment load at both, and
+profiles both:
+
+| Side          | rippled build                                             | runtime cfg                                      | collector                              | profiling |
+| ------------- | --------------------------------------------------------- | ------------------------------------------------ | -------------------------------------- | --------- |
+| **on-demand** | telemetry compiled in (phase-10 default `telemetry=True`) | `[telemetry] enabled=1`, OTLP → `127.0.0.1:4318` | node-local sidecar (receive + discard) | on        |
+| **baseline**  | telemetry compiled out (`telemetry=False`)                | none                                             | none                                   | on        |
+
+**Overhead = the delta between the two sides** — the rippled-process eBPF
+profile difference (CPU spent in span creation / attribute extraction on the
+hot path) plus the JMeter TPS / latency delta. The OTel trace data itself is
+discarded; only the _cost_ of producing it is measured.
+
+### Why a local discard-collector
+
+rippled's OTLP exporter runs on a background thread. If the endpoint is dead,
+that thread burns CPU on failed-export retries — and because the exporter is
+_inside_ the rippled process, that retry CPU lands in the rippled profile and
+inflates the apparent overhead. A node-local collector that accepts and
+discards (nop exporter) lets the export succeed instantly, keeping the profile
+clean. It is CPU-capped (50%) and, being a separate process, is excluded from
+the rippled-process profile regardless.
+
+---
+
+## Prerequisites (one-time)
+
+Two branches carry the benchmark setup:
+
+| Branch                              | Repo              | Purpose                                                                  |
+| ----------------------------------- | ----------------- | ------------------------------------------------------------------------ |
+| `pratik/otel-phase11-telemetry-off` | `XRPLF/rippled`   | baseline binary — `conanfile.py` `default_options.telemetry = False`     |
+| `pratik/otel-benchmarking-test`     | `ripple/perf-iac` | adds the per-side `telemetry` config key + `otel_collector` Ansible role |
+
+Both must be pushed. The perf-iac branch is the one to **run the workflow
+from** (see below) so the telemetry plumbing is present.
+
+---
+
+## Triggering a run (manual — recommended)
+
+Run the comparison from the perf-iac branch via `gh`, or via the Actions UI
+with **Use workflow from = `pratik/otel-benchmarking-test`**.
+
+```bash
+gh workflow run perf-internal.yml -R ripple/perf-iac \
+    --ref pratik/otel-benchmarking-test \
+    -f work-item=RIPD-7155 \
+    -f testname_base=otel_overhead_phase10 \
+    -f ondemand_performance_config='{"repo":"xrplf/rippled","ref":"pratik/otel-phase10-workload-validation","telemetry":"on","test_tpm":"60000","test_duration":"600","profiling":"true"}' \
+    -f baseline_performance_config='{"repo":"xrplf/rippled","ref":"pratik/otel-phase11-telemetry-off","telemetry":"off","profiling":"true"}'
+```
+
+| Field           | Meaning                                                         |
+| --------------- | --------------------------------------------------------------- |
+| `--ref`         | **must** be the perf-iac branch with the telemetry changes      |
+| `work-item`     | real Jira key, ≤32 chars (names the dynamic env)                |
+| `telemetry`     | per-side: `on` for on-demand, `off` for baseline — never merges |
+| `test_tpm`      | aggregate throughput per **minute** (`60000` ≈ 1000 TPS)        |
+| `test_duration` | seconds (`600` = 10 min)                                        |
+| `profiling`     | `true` on **both** sides — this is the measurement              |
+
+Shared keys (`test_tpm`, `test_duration`) inherit baseline ← on-demand, so set
+them once. Omitting `ssh-public-key` auto-destroys the env after the run.
+
+### Reading results
+
+- Report URL appears in the **Performance Testing** job log.
+- Slack notice to `#ripplex-performance-rippled-ci`.
+- Compare the two sides' rippled-process profiles + the TPS/latency table.
+
+---
+
+## Triggering from rippled CI (optional — needs a cross-org token)
+
+It is possible to add a `workflow_dispatch` job in `XRPLF/rippled` that shells
+out to dispatch the perf-iac run, so the benchmark can be kicked off from the
+rippled repo. **This is not wired up yet** because of a cross-org auth
+requirement, documented here so DevOps can decide.
+
+### The blocker
+
+- rippled lives in **`XRPLF`**; perf-iac lives in **`ripple`** (different orgs).
+- A workflow's default `GITHUB_TOKEN` is scoped to its own repo and **cannot**
+  dispatch a workflow in another org.
+- A **PAT or GitHub App token** with `actions: write` on `ripple/perf-iac`
+  must be stored as a secret (e.g. `PERF_IAC_DISPATCH_TOKEN`) in
+  `XRPLF/rippled`. Provisioning that token is an org-admin decision.
+
+### Sketch (once the token exists)
+
+A `.github/workflows/otel-benchmark-trigger.yml` in rippled, `workflow_dispatch`
+with inputs for the two refs / TPM / duration, whose single step dispatches
+perf-iac:
+
+```text
+steps:
+  - dispatch perf-internal.yml on ripple/perf-iac
+    using gh CLI (or actions/github-script) authenticated with
+    secrets.PERF_IAC_DISPATCH_TOKEN, passing the same -f inputs as the
+    manual command above.
+```
+
+Notes / caveats:
+
+- This only _kicks off_ the perf-iac run; the actual provisioning, build,
+  load, and profiling still execute **in** perf-iac under its own OIDC role
+  and repo `vars` — so no rippled-side AWS access is needed, only the
+  dispatch token.
+- Results still surface in the perf-iac Actions run + Slack, not in rippled CI.
+  If rippled-side visibility is wanted, the trigger job can poll the dispatched
+  run and echo its conclusion/report URL into the rippled job summary.
+
+---
+
+## Lessons learned
+
+- **A parent-directory `.gitignore` (`tasks/`) silently excluded the Ansible
+  role's `tasks/main.yml`.** The role committed without its tasks file and ran
+  as a no-op — the collector never installed, leaving the OTLP endpoint dead.
+  Always verify what is _tracked_ (`git ls-files <role>/`) after committing a
+  new role, not just what exists locally; run `ansible-playbook --syntax-check`
+  on the pushed tree.
+- **Matrix legs run `max-parallel: 1`** — on-demand and baseline run
+  sequentially on one dynamic env (good for comparability; doubles wall-clock).
+- Validate the role mechanics locally (syntax-check, render templates); the
+  full integration (real AMI apt install, 5-node provisioning, load) only
+  exercises in the pipeline — so a short `test_duration` smoke run is the
+  cheapest way to shake out integration bugs before a long measurement run.

OpenTelemetryPlan/OpenTelemetryPlan.md

@@ -56,6 +56,7 @@ flowchart TB
         appendix["08-appendix.md"]
         secure["secure-OTel.md"]
         poc["POC_taskList.md"]
+        dataref["09-data-collection-reference.md"]
     end
 
     overview --> fundamentals
@@ -73,6 +74,7 @@ flowchart TB
     backends --> appendix
     backends --> secure
     phases --> poc
+    appendix --> dataref
 
     style overview fill:#1b5e20,stroke:#0d3d14,color:#fff,stroke-width:2px
     style fundamentals fill:#00695c,stroke:#004d40,color:#fff
@@ -90,6 +92,7 @@ flowchart TB
     style appendix fill:#4a148c,stroke:#2e0d57,color:#fff
     style secure fill:#4a148c,stroke:#2e0d57,color:#fff
     style poc fill:#4a148c,stroke:#2e0d57,color:#fff
+    style dataref fill:#4a148c,stroke:#2e0d57,color:#fff
 ```
 
 </div>
@@ -98,19 +101,20 @@ flowchart TB
 
 ## Table of Contents
 
-| Section | Document                                                   | Description                                                            |
-| ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------- |
-| **0**   | [Tracing Fundamentals](./00-tracing-fundamentals.md)       | Distributed tracing concepts, span relationships, context propagation  |
-| **1**   | [Architecture Analysis](./01-architecture-analysis.md)     | xrpld component analysis, trace points, instrumentation priorities     |
-| **2**   | [Design Decisions](./02-design-decisions.md)               | SDK selection, exporters, span naming, attributes, context propagation |
-| **3**   | [Implementation Strategy](./03-implementation-strategy.md) | Directory structure, key principles, performance optimization          |
-| **4**   | [Code Samples](./04-code-samples.md)                       | C++ implementation examples for core infrastructure and key modules    |
-| **5**   | [Configuration Reference](./05-configuration-reference.md) | xrpld config, CMake integration, Collector configurations              |
-| **6**   | [Implementation Phases](./06-implementation-phases.md)     | 5-phase timeline, tasks, risks, success metrics                        |
-| **7**   | [Observability Backends](./07-observability-backends.md)   | Backend selection guide and production architecture                    |
-| **8**   | [Appendix](./08-appendix.md)                               | Glossary, references, version history                                  |
-| **Sec** | [Securing the OTel Pipeline](./secure-OTel.md)             | Threat model and hardening (mTLS, peer trace-context validation)       |
-| **POC** | [POC Task List](./POC_taskList.md)                         | Proof of concept tasks for RPC tracing end-to-end demo                 |
+| Section | Document                                                       | Description                                                            |
+| ------- | -------------------------------------------------------------- | ---------------------------------------------------------------------- |
+| **0**   | [Tracing Fundamentals](./00-tracing-fundamentals.md)           | Distributed tracing concepts, span relationships, context propagation  |
+| **1**   | [Architecture Analysis](./01-architecture-analysis.md)         | xrpld component analysis, trace points, instrumentation priorities     |
+| **2**   | [Design Decisions](./02-design-decisions.md)                   | SDK selection, exporters, span naming, attributes, context propagation |
+| **3**   | [Implementation Strategy](./03-implementation-strategy.md)     | Directory structure, key principles, performance optimization          |
+| **4**   | [Code Samples](./04-code-samples.md)                           | C++ implementation examples for core infrastructure and key modules    |
+| **5**   | [Configuration Reference](./05-configuration-reference.md)     | xrpld config, CMake integration, Collector configurations              |
+| **6**   | [Implementation Phases](./06-implementation-phases.md)         | 5-phase timeline, tasks, risks, success metrics                        |
+| **7**   | [Observability Backends](./07-observability-backends.md)       | Backend selection guide and production architecture                    |
+| **8**   | [Appendix](./08-appendix.md)                                   | Glossary, references, version history                                  |
+| **9**   | [Data Collection Reference](./09-data-collection-reference.md) | Complete inventory of spans, attributes, metrics, and dashboards       |
+| **Sec** | [Securing the OTel Pipeline](./secure-OTel.md)                 | Threat model and hardening (mTLS, peer trace-context validation)       |
+| **POC** | [POC Task List](./POC_taskList.md)                             | Proof of concept tasks for RPC tracing end-to-end demo                 |
 
 ---
 
@@ -188,17 +192,20 @@ OpenTelemetry Collector configurations are provided for development and producti
 
 ## 6. Implementation Phases
 
-The implementation spans 9 weeks across 5 phases:
+The implementation spans 13 weeks across 8 phases:
 
-| Phase | Duration  | Focus               | Key Deliverables                                    |
-| ----- | --------- | ------------------- | --------------------------------------------------- |
-| 1     | Weeks 1-2 | Core Infrastructure | SDK integration, Telemetry interface, Configuration |
-| 2     | Weeks 3-4 | RPC Tracing         | HTTP context extraction, Handler instrumentation    |
-| 3     | Weeks 5-6 | Transaction Tracing | Protocol Buffer context, Relay propagation          |
-| 4     | Weeks 7-8 | Consensus Tracing   | Round spans, Proposal/validation tracing            |
-| 5     | Week 9    | Documentation       | Runbook, Dashboards, Training                       |
+| Phase | Duration    | Focus                 | Key Deliverables                                            |
+| ----- | ----------- | --------------------- | ----------------------------------------------------------- |
+| 1     | Weeks 1-2   | Core Infrastructure   | SDK integration, Telemetry interface, Configuration         |
+| 2     | Weeks 3-4   | RPC Tracing           | HTTP context extraction, Handler instrumentation            |
+| 3     | Weeks 5-6   | Transaction Tracing   | Protocol Buffer context, Relay propagation                  |
+| 4     | Weeks 7-8   | Consensus Tracing     | Round spans, Proposal/validation tracing                    |
+| 5     | Week 9      | Documentation         | Runbook, Dashboards, Training                               |
+| 6     | Week 10     | StatsD Metrics Bridge | OTel Collector StatsD receiver, 3 Grafana dashboards        |
+| 7     | Weeks 11-12 | Native OTel Metrics   | OTelCollector impl, OTLP metrics export, StatsD deprecation |
+| 8     | Week 13     | Log-Trace Correlation | trace_id in logs, Loki ingestion, Tempo↔Loki linking        |
 
-**Total Effort**: 47 person-days (2 developers working in parallel)
+**Total Effort**: 65.1 developer-days with 2 developers
 
 ➡️ **[View full Implementation Phases](./06-implementation-phases.md)**
 
@@ -224,6 +231,14 @@ The appendix contains a glossary of OpenTelemetry and xrpld-specific terms, refe
 
 ---
 
+## 9. Data Collection Reference
+
+A single-source-of-truth reference documenting every piece of telemetry data collected by xrpld. Covers all 16 OpenTelemetry spans with their 22 attributes, all StatsD metrics (gauges, counters, histograms, overlay traffic), SpanMetrics-derived Prometheus metrics, and all 10 Grafana dashboards. Includes Tempo search guides and Prometheus query examples.
+
+➡️ **[View Data Collection Reference](./09-data-collection-reference.md)**
+
+---
+
 ## Securing the OTel Pipeline
 
 Threat model and hardening guidance for production deployments where xrpld nodes ship telemetry to a centrally-hosted collector across an untrusted network. Covers the two attack surfaces (collector ingress and peer trace-context spoofing) and the chosen defenses: mTLS as primary collector auth, NetworkPolicy as defense-in-depth, and source-side validation plus per-peer rate limiting for the `protocol::TraceContext` field on peer messages.

OpenTelemetryPlan/Phase10_taskList.md

@@ -0,0 +1,258 @@
+# Phase 10: Synthetic Workload Generation & Telemetry Validation — Task List
+
+> **Status**: Future Enhancement
+>
+> **Goal**: Build tools that generate realistic XRPL traffic to validate the full Phases 1-9 telemetry stack end-to-end — all spans, attributes, metrics, dashboards, and log-trace correlation — under controlled load.
+>
+> **Scope**: Python/shell test harness + multi-node docker-compose environment + automated validation scripts + performance benchmarks.
+>
+> **Branch**: `pratik/otel-phase10-workload-validation` (from `pratik/otel-phase9-metric-gap-fill`)
+>
+> **Depends on**: Phase 9 (internal metric gap fill) — validates the full metric surface
+
+### Related Plan Documents
+
+| Document                                                             | Relevance                                                       |
+| -------------------------------------------------------------------- | --------------------------------------------------------------- |
+| [06-implementation-phases.md](./06-implementation-phases.md)         | Phase 10 plan: motivation, architecture, exit criteria (§6.8.3) |
+| [09-data-collection-reference.md](./09-data-collection-reference.md) | Defines the full inventory of spans/metrics to validate         |
+| [Phase9_taskList.md](./Phase9_taskList.md)                           | Prerequisite — all internal metrics must be emitting            |
+
+### Why This Phase Exists
+
+Before Phases 1-9 can be considered production-ready, we need proof that:
+
+1. All required spans fire with correct attributes under real transaction workloads
+2. All 255+ StatsD metrics + ~50 Phase 9 metrics appear in Prometheus with non-zero values
+3. Log-trace correlation (Phase 8) produces clickable trace_id links in Loki
+4. All 10 Grafana dashboards render meaningful data (no empty panels)
+5. Performance overhead stays within bounds (< 3% CPU, < 5MB memory)
+6. The telemetry stack survives sustained load without data loss or queue backpressure
+
+---
+
+## Task 10.1: Multi-Node Test Harness
+
+**Objective**: Create a docker-compose environment with 3-5 validator nodes that produces real consensus rounds.
+
+**What to do**:
+
+- Create `docker/telemetry/docker-compose.workload.yaml`:
+  - 5 xrpld validator nodes with UNL configured for each other
+  - All telemetry enabled: `[telemetry] enabled=1`, `[insight] server=otel`
+  - Full OTel stack: Collector, Tempo, Prometheus, Loki, Grafana
+  - Shared network with service discovery
+
+- Each node should:
+  - Generate validator keys at startup
+  - Configure all 5 nodes in its UNL
+  - Enable all trace categories including `trace_peer=1`
+  - Write logs to a file tailed by the OTel Collector filelog receiver
+
+- Include a `Makefile` target: `make telemetry-workload-up` / `make telemetry-workload-down`
+
+**Key files**:
+
+- New: `docker/telemetry/docker-compose.workload.yaml`
+- New: `docker/telemetry/workload/generate-validator-keys.sh`
+- New: `docker/telemetry/workload/xrpld-validator.cfg.template`
+
+---
+
+## Task 10.2: RPC Load Generator
+
+**Objective**: Configurable tool that fires all traced RPC commands at controlled rates.
+
+**What to do**:
+
+- Create `docker/telemetry/workload/rpc_load_generator.py`:
+  - Connects to one or more xrpld WebSocket endpoints
+  - Fires all RPC commands that have trace spans: `server_info`, `ledger`, `tx`, `account_info`, `account_lines`, `fee`, `submit`, etc.
+  - Configurable parameters: rate (RPS), duration, command distribution weights
+  - Injects `traceparent` HTTP headers to test W3C context propagation
+  - Logs progress and errors to stdout
+
+- Command distribution should match realistic production ratios:
+  - 40% `server_info` / `fee` (health checks)
+  - 30% `account_info` / `account_lines` / `account_objects` (wallet queries)
+  - 15% `ledger` / `ledger_data` (explorer queries)
+  - 10% `tx` / `account_tx` (transaction lookups)
+  - 5% `book_offers` / `amm_info` (DEX queries)
+
+**Key files**:
+
+- New: `docker/telemetry/workload/rpc_load_generator.py`
+- New: `docker/telemetry/workload/requirements.txt`
+
+---
+
+## Task 10.3: Transaction Submitter
+
+**Objective**: Generate diverse transaction types to exercise `tx.*` and `ledger.*` spans.
+
+**What to do**:
+
+- Create `docker/telemetry/workload/tx_submitter.py`:
+  - Pre-funds test accounts from genesis account
+  - Submits a mix of transaction types:
+    - `Payment` (XRP and issued currencies) — exercises `tx.process`, `tx.apply`
+    - `OfferCreate` / `OfferCancel` — DEX activity
+    - `TrustSet` — trust line creation for issued currencies
+    - `NFTokenMint` / `NFTokenCreateOffer` / `NFTokenAcceptOffer` — NFT activity
+    - `EscrowCreate` / `EscrowFinish` — escrow lifecycle
+    - `AMMCreate` / `AMMDeposit` / `AMMWithdraw` — AMM pool operations (if amendment enabled)
+  - Configurable: TPS target, transaction mix weights, duration
+  - Monitors submission results and tracks success/failure rates
+
+- The transaction mix ensures the telemetry captures the full range of ledger activity that third parties care about.
+
+**Key files**:
+
+- New: `docker/telemetry/workload/tx_submitter.py`
+- New: `docker/telemetry/workload/test_accounts.json` (pre-generated keypairs)
+
+---
+
+## Task 10.4: Telemetry Validation Suite
+
+**Objective**: Automated scripts that verify all expected telemetry data exists after a workload run.
+
+**What to do**:
+
+- Create `docker/telemetry/workload/validate_telemetry.py`:
+
+  **Span validation** (queries Tempo API):
+  - Assert all required span names appear in traces (conditional spans — `grpc.*`,
+    `ledger.acquire`, `txq.*`, `consensus.mode_change` — are marked `optional` and
+    skipped when not exercised by the workload)
+  - Assert each span has its required attributes (bare/underscore keys per the
+    2026-05-13 span-attr naming redesign; dotted `xrpl.*` reserved for resource attrs)
+  - Assert parent-child relationships are correct (`rpc.ws_message` → `rpc.process` → `rpc.command.*`)
+  - Assert span durations are reasonable (> 0, < 60s)
+
+  **Metric validation** (queries Prometheus API):
+  - Assert all SpanMetrics-derived metrics are non-zero: `traces_span_metrics_calls_total`, `traces_span_metrics_duration_milliseconds_bucket`
+  - Assert all StatsD metrics are non-zero: `xrpld_LedgerMaster_Validated_Ledger_Age`, `xrpld_Peer_Finder_Active_*`, etc.
+  - Assert all Phase 9 metrics are non-zero: `xrpld_nodestore_*`, `xrpld_cache_*`, `xrpld_txq_*`, `xrpld_rpc_method_*`, `xrpld_object_count`, `xrpld_load_factor*`
+  - Assert metric label cardinality is within bounds
+
+  **Log-trace correlation validation** (queries Loki API):
+  - Assert logs contain `trace_id=` and `span_id=` fields
+  - Pick a random trace_id from Tempo → query Loki for matching logs → assert results exist
+  - Assert Grafana derived field links are functional
+
+  **Dashboard validation**:
+  - For each of the 10 Grafana dashboards, query the dashboard API and assert no panels show "No data"
+
+- Output: JSON report with pass/fail per check, suitable for CI.
+
+**Key files**:
+
+- New: `docker/telemetry/workload/validate_telemetry.py`
+- New: `docker/telemetry/workload/expected_spans.json` (span inventory for validation)
+- New: `docker/telemetry/workload/expected_metrics.json` (metric inventory for validation)
+
+---
+
+## Task 10.5: Performance Benchmark Suite
+
+**Objective**: Measure CPU/memory/latency overhead of the telemetry stack.
+
+**What to do**:
+
+- Create `docker/telemetry/workload/benchmark.sh`:
+  - **Baseline run**: Start cluster with `[telemetry] enabled=0`, run transaction workload for 5 minutes, record metrics
+  - **Telemetry run**: Start cluster with full telemetry enabled, run identical workload, record metrics
+  - **Comparison**: Calculate deltas for:
+    - CPU usage (per-node average)
+    - Memory RSS (per-node peak)
+    - RPC p99 latency
+    - Transaction throughput (TPS)
+    - Consensus round time p95
+    - Ledger close time p95
+
+- Output: Markdown table comparing baseline vs. telemetry, with pass/fail against targets:
+  - CPU overhead < 3%
+  - Memory overhead < 5MB
+  - RPC latency impact < 2ms p99
+  - Throughput impact < 5%
+  - Consensus impact < 1%
+
+- Store results in `docker/telemetry/workload/benchmark-results/` for historical tracking.
+
+**Key files**:
+
+- New: `docker/telemetry/workload/benchmark.sh`
+- New: `docker/telemetry/workload/collect_system_metrics.sh`
+
+---
+
+## Task 10.6: CI Integration
+
+**Objective**: Wire the validation suite into CI for regression detection.
+
+**What to do**:
+
+- Create a CI workflow (GitHub Actions or equivalent) that:
+  1. Builds xrpld with `-DXRPL_ENABLE_TELEMETRY=ON`
+  2. Starts the multi-node workload harness
+  3. Runs the RPC load generator + transaction submitter for 2 minutes
+  4. Runs the validation suite
+  5. Runs the benchmark suite
+  6. Fails the build if any validation check fails or benchmark exceeds thresholds
+  7. Archives the validation report and benchmark results as artifacts
+
+- This should be a separate workflow (not part of the main CI), triggered manually or on telemetry-related branch changes.
+
+**Key files**:
+
+- New: `.github/workflows/telemetry-validation.yml`
+- New: `docker/telemetry/workload/run-full-validation.sh` (orchestrator script)
+
+---
+
+## Task 10.7: Documentation
+
+**Objective**: Document the workload tools and validation process.
+
+**What to do**:
+
+- Create `docker/telemetry/workload/README.md`:
+  - Quick start guide for running workload harness
+  - Configuration options for load generator and tx submitter
+  - How to read validation reports
+  - How to run benchmarks and interpret results
+
+- Update `docs/telemetry-runbook.md`:
+  - Add "Validating Telemetry Stack" section
+  - Add "Performance Benchmarking" section
+
+- Update `OpenTelemetryPlan/09-data-collection-reference.md`:
+  - Add "Validation" section with expected metric/span counts
+
+---
+
+## Exit Criteria — Delivered in PR #6519
+
+- [x] Multi-node validator cluster starts and reaches consensus
+- [x] RPC load generator fires all traced RPC commands at configurable rates
+- [x] Transaction submitter generates 6+ transaction types at configurable TPS
+- [x] Validation suite confirms all required spans, attributes, and metrics
+- [x] Log-trace correlation validated end-to-end (Loki ↔ Tempo)
+- [x] Grafana dashboards render data (no empty panels)
+- [x] Overhead benchmark (`benchmark.sh`) measures telemetry-off vs telemetry-on deltas
+- [x] CI workflow runs validation on telemetry branch changes
+- [x] Validation report output is CI-parseable (JSON with exit codes)
+- [x] OTel-driven regression gate captures per-span/per-RPC/per-job timings from
+      Prometheus and compares against a committed baseline
+
+## Follow-up Work (tracked in separate PRs)
+
+- [ ] FU-2: Automate baseline persistence across CI runs (artifact uploaded
+      on merge to `develop`, downloaded on PR runs). Current mechanism
+      requires a manual baseline-refresh PR.
+- [ ] FU-4: Replace the proxy measurements in `benchmark.sh` (wall-clock curl
+      p99, ledger-cadence-as-TPS, ledger-cadence-as-consensus-p95) with
+      PromQL quantile queries from the same pipeline the regression gate uses.
+- [ ] FU-6: Grafana dashboard plotting historical baseline values keyed by
+      commit SHA, for triaging noisy regressions.

OpenTelemetryPlan/Phase11_taskList.md

@@ -0,0 +1,544 @@
+# Phase 11: Third-Party Data Collection Pipelines — Task List
+
+> **Status**: Future Enhancement
+>
+> **Goal**: Build a custom OTel Collector receiver that periodically polls xrpld's admin RPCs and exports structured metrics for external consumers — making all XRPL health, validator, peer, fee, and DEX data available as Prometheus/OTLP metrics without xrpld code changes.
+>
+> **Scope**: Go-based OTel Collector receiver plugin + Grafana dashboards + Prometheus alerting rules.
+>
+> **Branch**: `pratik/otel-phase11-third-party-collection` (from `pratik/otel-phase10-workload-validation`)
+>
+> **Depends on**: Phase 10 (validation harness for testing the new receiver)
+
+### Related Plan Documents
+
+| Document                                                             | Relevance                                                       |
+| -------------------------------------------------------------------- | --------------------------------------------------------------- |
+| [06-implementation-phases.md](./06-implementation-phases.md)         | Phase 11 plan: motivation, architecture, exit criteria (§6.8.4) |
+| [09-data-collection-reference.md](./09-data-collection-reference.md) | Defines full metric inventory including third-party metrics     |
+| [Phase10_taskList.md](./Phase10_taskList.md)                         | Prerequisite — validation harness for testing                   |
+
+### Third-Party Consumer Gap Analysis
+
+This phase addresses the cross-cutting gap identified during research: **xrpld has no native Prometheus/OTLP metrics export for data accessible only via RPC**. Every consumer (exchanges, payment processors, analytics providers, validators, researchers, compliance firms, custodians) must build custom JSON-RPC polling and conversion. This receiver centralizes that work.
+
+| Consumer Category          | Data Unlocked by This Phase                                        |
+| -------------------------- | ------------------------------------------------------------------ |
+| **Exchanges**              | Real-time fee estimates, TxQ capacity, server health scores        |
+| **Payment Processors**     | Settlement latency percentiles, corridor health, path availability |
+| **Analytics Providers**    | Validator metrics, network topology, amendment voting status       |
+| **DeFi / AMM**             | AMM pool TVL, DEX order book depth, trade volumes                  |
+| **Validators / Operators** | Per-peer latency, version distribution, UNL health, alerting       |
+| **Compliance**             | Transaction volume trends, network growth metrics                  |
+| **Academic Researchers**   | Consensus performance time-series, decentralization metrics        |
+| **CBDC / Tokenization**    | Token supply tracking, trust line adoption, freeze status          |
+| **Institutional Custody**  | Multi-sig status, escrow tracking, reserve calculations            |
+| **Wallet Providers**       | Server health for node selection, fee prediction data              |
+
+---
+
+## Task 11.1: OTel Collector Receiver Scaffold
+
+**Objective**: Create the Go project structure for a custom OTel Collector receiver that polls xrpld JSON-RPC.
+
+**What to do**:
+
+- Create `docker/telemetry/otel-rippled-receiver/`:
+  - `receiver.go` — implements `receiver.Metrics` interface
+  - `config.go` — configuration struct (endpoint, poll interval, enabled RPCs)
+  - `factory.go` — receiver factory registration
+  - `go.mod` / `go.sum` — Go module with OTel Collector SDK dependency
+
+- Configuration model:
+
+  ```yaml
+  xrpld_receiver:
+    endpoint: "http://localhost:5005" # xrpld admin RPC
+    poll_interval: 30s # how often to poll
+    enabled_collectors:
+      - server_info
+      - get_counts
+      - fee
+      - peers
+      - validators
+      - feature
+      - server_state
+    amm_pools: [] # optional: AMM pool IDs to track
+    book_offers_pairs: [] # optional: currency pairs for DEX depth
+  ```
+
+- Build a custom OTel Collector binary that includes this receiver alongside the standard receivers.
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/receiver.go`
+- New: `docker/telemetry/otel-rippled-receiver/config.go`
+- New: `docker/telemetry/otel-rippled-receiver/factory.go`
+- New: `docker/telemetry/otel-rippled-receiver/go.mod`
+- New: `docker/telemetry/otel-rippled-receiver/Dockerfile`
+
+---
+
+## Task 11.2: server_info / server_state Collector
+
+**Objective**: Poll `server_info` and `server_state` and export all fields as OTel metrics.
+
+**What to do**:
+
+- Implement `serverInfoCollector` that calls `server_info` (admin) and extracts:
+
+  **Node Health Gauges:**
+  - `xrpl_server_state` (enum → int: disconnected=0, connected=1, syncing=2, tracking=3, full=4, proposing=5)
+  - `xrpl_server_state_duration_seconds`
+  - `xrpl_uptime_seconds`
+  - `xrpl_io_latency_ms`
+  - `xrpl_amendment_blocked` (0 or 1)
+  - `xrpl_peers_count`
+  - `xrpl_peer_disconnects_total`
+  - `xrpl_peer_disconnects_resources_total`
+  - `xrpl_jq_trans_overflow_total`
+
+  **Consensus Gauges:**
+  - `xrpl_last_close_proposers`
+  - `xrpl_last_close_converge_time_seconds`
+  - `xrpl_validation_quorum`
+
+  **Ledger Gauges:**
+  - `xrpl_validated_ledger_seq`
+  - `xrpl_validated_ledger_age_seconds`
+  - `xrpl_validated_ledger_base_fee_drops`
+  - `xrpl_validated_ledger_reserve_base_drops`
+  - `xrpl_validated_ledger_reserve_inc_drops`
+  - `xrpl_close_time_offset_seconds` (0 when absent)
+
+  **Load Factor Gauges:**
+  - `xrpl_load_factor`
+  - `xrpl_load_factor_server`
+  - `xrpl_load_factor_fee_escalation`
+  - `xrpl_load_factor_fee_queue`
+  - `xrpl_load_factor_local`
+  - `xrpl_load_factor_net`
+  - `xrpl_load_factor_cluster`
+
+  **State Accounting Gauges** (per state: disconnected, connected, syncing, tracking, full):
+  - `xrpl_state_duration_seconds{state="<name>"}`
+  - `xrpl_state_transitions_total{state="<name>"}`
+
+  **Validator Info** (when node is a validator):
+  - `xrpl_validator_list_count`
+  - `xrpl_validator_list_expiration_seconds` (epoch)
+  - `xrpl_validator_list_active` (0 or 1)
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/collectors/server_info.go`
+
+---
+
+## Task 11.3: get_counts Collector
+
+**Objective**: Poll `get_counts` and export internal object counts and NodeStore stats.
+
+**What to do**:
+
+- Implement `getCountsCollector`:
+
+  **Database Gauges:**
+  - `xrpl_db_size_kb{db="total"}`, `xrpl_db_size_kb{db="ledger"}`, `xrpl_db_size_kb{db="transaction"}`
+
+  **NodeStore Gauges:**
+  - `xrpl_nodestore_reads_total`, `xrpl_nodestore_reads_hit`, `xrpl_nodestore_writes_total`
+  - `xrpl_nodestore_read_bytes`, `xrpl_nodestore_written_bytes`
+  - `xrpl_nodestore_read_duration_us`, `xrpl_nodestore_write_load`
+  - `xrpl_nodestore_read_queue`, `xrpl_nodestore_read_threads_running`
+
+  **Cache Gauges:**
+  - `xrpl_cache_hit_rate{cache="SLE"}`, `xrpl_cache_hit_rate{cache="ledger"}`, `xrpl_cache_hit_rate{cache="accepted_ledger"}`
+  - `xrpl_cache_size{cache="treenode"}`, `xrpl_cache_size{cache="fullbelow"}`, `xrpl_cache_size{cache="accepted_ledger"}`
+
+  **Object Count Gauges:**
+  - `xrpl_object_count{type="<name>"}` for each counted object type (Transaction, Ledger, NodeObject, STTx, STLedgerEntry, InboundLedger, Pathfinder, etc.)
+
+  **Rates:**
+  - `xrpl_historical_fetch_per_minute`
+  - `xrpl_local_txs`
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/collectors/get_counts.go`
+
+---
+
+## Task 11.4: Peer Topology Collector
+
+**Objective**: Poll `peers` and export per-peer and aggregate network metrics.
+
+**What to do**:
+
+- Implement `peersCollector`:
+
+  **Aggregate Gauges:**
+  - `xrpl_peers_inbound_count`
+  - `xrpl_peers_outbound_count`
+  - `xrpl_peers_cluster_count`
+
+  **Per-Peer Gauges** (with labels `peer_key` truncated to 8 chars for cardinality control):
+  - `xrpl_peer_latency_ms{peer="<key>", version="<ver>", inbound="<bool>"}`
+  - `xrpl_peer_uptime_seconds{peer="<key>"}`
+  - `xrpl_peer_load{peer="<key>"}`
+
+  **Distribution Gauges** (aggregated across all peers):
+  - `xrpl_peer_latency_p50_ms`, `xrpl_peer_latency_p95_ms`, `xrpl_peer_latency_p99_ms`
+  - `xrpl_peer_version_count{version="<semver>"}` — count of peers per software version
+
+  **Tracking Status:**
+  - `xrpl_peer_diverged_count` — peers with `track=diverged`
+  - `xrpl_peer_unknown_count` — peers with `track=unknown`
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/collectors/peers.go`
+
+**Cardinality note**: Per-peer metrics use truncated keys. For large peer sets (50+), the aggregate distribution gauges are preferred over per-peer labels.
+
+---
+
+## Task 11.5: Validator & Amendment Collector
+
+**Objective**: Poll `validators` and `feature` to export validator health and amendment voting status.
+
+**What to do**:
+
+- Implement `validatorCollector`:
+
+  **From `validators` RPC:**
+  - `xrpl_trusted_validators_count`
+  - `xrpl_validator_signing` (0 or 1 — whether local validator is signing)
+
+  **From `feature` RPC:**
+  - `xrpl_amendment_enabled_count` — total enabled amendments
+  - `xrpl_amendment_majority_count` — amendments with majority but not yet enabled
+  - `xrpl_amendment_vetoed_count` — locally vetoed amendments
+  - `xrpl_amendment_unsupported_majority` (0 or 1) — any unsupported amendment has majority (critical alert)
+
+  **Per-amendment with majority** (limited cardinality — only amendments with `majority` set):
+  - `xrpl_amendment_majority_time{name="<amendment>"}` — epoch time when majority was gained
+  - `xrpl_amendment_votes{name="<amendment>"}` — current vote count
+  - `xrpl_amendment_threshold{name="<amendment>"}` — votes needed
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/collectors/validators.go`
+
+---
+
+## Task 11.6: Fee & TxQ Collector
+
+**Objective**: Poll `fee` RPC and export real-time fee market data.
+
+**What to do**:
+
+- Implement `feeCollector` that calls the public `fee` RPC:
+
+  **Fee Level Gauges:**
+  - `xrpl_fee_current_ledger_size` — transactions in current open ledger
+  - `xrpl_fee_expected_ledger_size` — expected transactions at close
+  - `xrpl_fee_max_queue_size` — maximum transaction queue size
+  - `xrpl_fee_open_ledger_fee_drops` — minimum fee for open ledger inclusion
+  - `xrpl_fee_median_fee_drops` — median fee level
+  - `xrpl_fee_minimum_fee_drops` — base reference fee
+  - `xrpl_fee_queue_size` — current queue depth
+
+- This overlaps with Phase 9's internal TxQ metrics but provides an external-only collection path that doesn't require xrpld code changes.
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/collectors/fee.go`
+
+---
+
+## Task 11.7: DEX & AMM Collector (Optional)
+
+**Objective**: Periodically poll configured AMM pools and order book pairs for DeFi metrics.
+
+**What to do**:
+
+- Implement `dexCollector` (enabled only when `amm_pools` or `book_offers_pairs` are configured):
+
+  **AMM Pool Gauges** (per configured pool):
+  - `xrpl_amm_reserve{pool="<id>", asset="<currency>"}` — pool reserve amount
+  - `xrpl_amm_lp_token_supply{pool="<id>"}` — outstanding LP tokens
+  - `xrpl_amm_trading_fee{pool="<id>"}` — pool trading fee (basis points)
+  - `xrpl_amm_tvl_drops{pool="<id>"}` — total value locked (XRP-denominated)
+
+  **Order Book Gauges** (per configured pair):
+  - `xrpl_orderbook_bid_depth{pair="<base>/<quote>"}` — total bid volume
+  - `xrpl_orderbook_ask_depth{pair="<base>/<quote>"}` — total ask volume
+  - `xrpl_orderbook_spread{pair="<base>/<quote>"}` — best bid-ask spread
+  - `xrpl_orderbook_offer_count{pair="<base>/<quote>", side="bid|ask"}` — number of offers
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/collectors/dex.go`
+
+**Note**: This is optional because it requires explicit configuration of which pools/pairs to track. Default configuration tracks no DEX data.
+
+---
+
+## Task 11.8: Prometheus Alerting Rules
+
+**Objective**: Create production-ready alerting rules for the metrics exported by this receiver.
+
+**What to do**:
+
+- Create `docker/telemetry/prometheus/rippled-alerts.yml`:
+
+  **Tier 1 — Critical (page immediately):**
+
+  ```yaml
+  - alert: XRPLServerNotFull
+    expr: xrpl_server_state < 4
+    for: 15m
+
+  - alert: XRPLAmendmentBlocked
+    expr: xrpl_amendment_blocked == 1
+    for: 1m
+
+  - alert: XRPLNoPeers
+    expr: xrpl_peers_count == 0
+    for: 5m
+
+  - alert: XRPLLedgerStale
+    expr: xrpl_validated_ledger_age_seconds > 120
+    for: 2m
+
+  - alert: XRPLHighIOLatency
+    expr: xrpl_io_latency_ms > 100
+    for: 5m
+
+  - alert: XRPLUnsupportedAmendmentMajority
+    expr: xrpl_amendment_unsupported_majority == 1
+    for: 1m
+  ```
+
+  **Tier 2 — Warning (investigate within hours):**
+
+  ```yaml
+  - alert: XRPLLowPeerCount
+    expr: xrpl_peers_count < 10
+    for: 15m
+
+  - alert: XRPLHighLoadFactor
+    expr: xrpl_load_factor > 10
+    for: 10m
+
+  - alert: XRPLSlowConsensus
+    expr: xrpl_last_close_converge_time_seconds > 6
+    for: 5m
+
+  - alert: XRPLValidatorListExpiring
+    expr: (xrpl_validator_list_expiration_seconds - time()) < 86400
+    for: 1h
+
+  - alert: XRPLClockDrift
+    expr: xrpl_close_time_offset_seconds > 0
+    for: 5m
+
+  - alert: XRPLStateFlapping
+    expr: rate(xrpl_state_transitions_total{state="full"}[1h]) > 2
+    for: 30m
+  ```
+
+**Key files**:
+
+- New: `docker/telemetry/prometheus/rippled-alerts.yml`
+- Update: `docker/telemetry/prometheus/prometheus.yml` (add rule_files reference)
+
+---
+
+## Task 11.9: New Grafana Dashboards
+
+**Objective**: Create 4 new dashboards for the data exported by the receiver.
+
+**What to do**:
+
+- **Validator Health** (`xrpld-validator-health`):
+  - Server state timeline, state duration breakdown
+  - Proposer count trend, converge time trend, validation quorum
+  - Validator list expiration countdown
+  - Amendment voting status (majority/enabled/vetoed)
+
+- **Network Topology** (`xrpld-network-topology`):
+  - Peer count (inbound/outbound/cluster), peer version distribution
+  - Peer latency distribution (p50/p95/p99), diverged peer count
+  - Geographic distribution (if enriched with GeoIP)
+  - Peer uptime distribution
+
+- **Fee Market** (`xrpld-fee-market-external`):
+  - Current fee levels (open ledger, median, minimum), fee escalation timeline
+  - Queue depth vs. capacity, transactions per ledger
+  - Load factor breakdown (server/network/cluster/escalation)
+
+- **DEX & AMM Overview** (`xrpld-dex-amm`) (only populated when DEX collectors are configured):
+  - AMM pool TVL, reserve ratios, LP token supply
+  - Order book depth per pair, spread trends
+  - Trading fee revenue estimates
+
+**Key files**:
+
+- New: `docker/telemetry/grafana/dashboards/rippled-validator-health.json`
+- New: `docker/telemetry/grafana/dashboards/rippled-network-topology.json`
+- New: `docker/telemetry/grafana/dashboards/rippled-fee-market-external.json`
+- New: `docker/telemetry/grafana/dashboards/rippled-dex-amm.json`
+
+---
+
+## Task 11.10: Integration with Phase 10 Validation
+
+**Objective**: Extend the Phase 10 validation suite to verify this receiver's metrics.
+
+**What to do**:
+
+- Update `docker/telemetry/workload/validate_telemetry.py`:
+  - Add assertions for all `xrpl_*` metrics produced by the receiver
+  - Verify metric labels have expected values
+  - Verify alerting rules fire correctly (inject a "bad" state and check alert)
+
+- Update `docker/telemetry/docker-compose.workload.yaml`:
+  - Add the custom OTel Collector build with the xrpld receiver
+  - Configure the receiver to poll one of the test nodes
+
+**Key files**:
+
+- Update: `docker/telemetry/workload/validate_telemetry.py`
+- Update: `docker/telemetry/docker-compose.workload.yaml`
+- Update: `docker/telemetry/workload/expected_metrics.json`
+
+---
+
+## Task 11.11: Documentation
+
+**Objective**: Document the receiver, its metrics, deployment, and alerting.
+
+**What to do**:
+
+- Create `docker/telemetry/otel-rippled-receiver/README.md`:
+  - Architecture overview (how the receiver fits into the OTel Collector)
+  - Configuration reference (all config options with defaults)
+  - Metric reference table (all exported metrics with types and labels)
+  - Deployment guide (building custom collector binary, docker-compose integration)
+
+- Update `OpenTelemetryPlan/09-data-collection-reference.md`:
+  - Add "Third-Party Metrics (OTel Collector Receiver)" section
+  - Add new Grafana dashboard reference (4 dashboards)
+  - Add alerting rules reference
+
+- Update `docs/telemetry-runbook.md`:
+  - Add "Third-Party Metrics Receiver" troubleshooting section
+  - Add alerting playbook (what to do for each Tier 1/Tier 2 alert)
+
+---
+
+## Task 11.12: Alert Rules for External Dashboard Parity Metrics
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md) — 18 alert rules ported from the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
+>
+> **Upstream**: Phase 7 Tasks 7.9-7.16 (metrics), Phase 9 Tasks 9.11-9.13 (dashboards).
+> **Downstream**: None — terminal task in the parity chain.
+
+**Objective**: Add Grafana alerting rules for the Phase 7+ parity metrics (validation agreement, validator health, peer quality, state tracking, ledger economy). These complement Task 11.8's `xrpl_*` alerts by covering the `xrpld_*` internal metrics.
+
+**Critical Group** (8 rules, eval interval 10s):
+
+| Rule                | Condition                                                     | For |
+| ------------------- | ------------------------------------------------------------- | --- |
+| Agreement Below 90% | `xrpld_validation_agreement{metric="agreement_pct_24h"} < 90` | 30s |
+| Not Proposing       | `xrpld_state_tracking{metric="state_value"} < 6`              | 10s |
+| Unhealthy State     | `xrpld_state_tracking{metric="state_value"} < 4`              | 10s |
+| Amendment Blocked   | `xrpld_validator_health{metric="amendment_blocked"} == 1`     | 1m  |
+| UNL Expiring        | `xrpld_validator_health{metric="unl_expiry_days"} < 14`       | 1h  |
+| High IO Latency     | `histogram_quantile(0.95, xrpld_ios_latency_bucket) > 50`     | 1m  |
+| High Load Factor    | `xrpld_load_factor_metrics{metric="load_factor"} > 1000`      | 1m  |
+| Peer Count Critical | `xrpld_server_info{metric="peers"} < 5`                       | 1m  |
+
+**Network Group** (3 rules, eval interval 10s):
+
+| Rule                      | Condition                                                         | For |
+| ------------------------- | ----------------------------------------------------------------- | --- |
+| Peer Drop >10%            | `delta(xrpld_server_info{metric="peers"}[30s]) / ... * 100 < -10` | 30s |
+| Peer Drop >30%            | Same formula, threshold -30                                       | 30s |
+| P90 Latency + Disconnects | `peer_latency_p90_ms > 500 AND rate(disconnects) > 0`             | 2m  |
+
+**Performance Group** (7 rules, eval interval 10s):
+
+| Rule                | Condition                                                    | For |
+| ------------------- | ------------------------------------------------------------ | --- |
+| CPU High            | Per-core CPU > 80% (requires node_exporter)                  | 2m  |
+| Memory Critical     | Memory usage > 90% (requires node_exporter)                  | 1m  |
+| Disk Warning        | Disk usage > 85% (requires node_exporter)                    | 2m  |
+| Job Queue Overflow  | `rate(xrpld_jq_trans_overflow_total[5m]) > 0`                | 1m  |
+| Upgrade Recommended | `xrpld_peer_quality{metric="peers_higher_version_pct"} > 60` | 1m  |
+| TX Rate Drop        | Transaction rate dropped > 50% in 5m window                  | 5m  |
+| Stale Ledger        | `xrpld_ledger_economy{metric="ledger_age_seconds"} > 30`     | 1m  |
+
+**Notification channel templates**: Email/SMTP, Discord, Slack, PagerDuty.
+
+**Key files**:
+
+- New/extend: `docker/telemetry/grafana/alerting/alert-rules-parity.yaml`
+- New: `docker/telemetry/grafana/alerting/contact-points.yaml` (template configs)
+- New: `docker/telemetry/grafana/alerting/notification-policies.yaml`
+
+**Exit Criteria**:
+
+- [ ] All 18 rules evaluate without errors in Grafana alerting UI
+- [ ] Critical rules fire within expected timeframe when conditions are met
+- [ ] Notification channel templates are documented (not hard-coded to any service)
+
+---
+
+## Task 11.13: Dual-Datasource Architecture Documentation
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md)
+
+**Objective**: Document the external dashboard's "fast path" pattern as a future optimization for real-time panels.
+
+**Pattern**: A lightweight Prometheus scrape endpoint (separate from OTLP pipeline) that polls critical metrics every 2-5s, bypassing the 10s OTLP metric reader interval and Prometheus scrape interval.
+
+**Use case**: Real-time state panels (server state, ledger age, peer count) where 10-15s latency is too slow for operational dashboards.
+
+**Decision**: Document as a future option, not implement now. The current 10s interval is acceptable for v1. The external dashboard achieves 2-5s freshness by polling RPC directly, which is what the Phase 11 receiver already does. Adding a separate scrape endpoint to xrpld would only be needed if sub-second metric freshness is required from the internal metrics pipeline.
+
+**What to document**:
+
+- Architecture comparison: OTLP pipeline (10-15s) vs. direct scrape (2-5s) vs. push gateway
+- When to consider: operator feedback indicating 10s is insufficient for alerting SLOs
+- How to implement if needed: add `/metrics` HTTP endpoint to xrpld with Prometheus client library
+- Trade-offs: additional port, additional dependency, duplication with OTLP metrics
+
+**Key files**:
+
+- Update: `OpenTelemetryPlan/09-data-collection-reference.md` (add "Future: Dual-Datasource Architecture" section)
+- Update: `docs/telemetry-runbook.md` (add brief note in performance tuning section)
+
+**Exit Criteria**:
+
+- [ ] Architecture comparison documented with clear trade-offs
+- [ ] Decision rationale recorded (why deferred, when to revisit)
+
+---
+
+## Exit Criteria
+
+- [ ] Custom OTel Collector receiver builds and starts without errors
+- [ ] All `xrpl_*` metrics from server_info, get_counts, peers, validators, fee appear in Prometheus
+- [ ] Metrics update at configured poll interval (default 30s)
+- [ ] 4 new Grafana dashboards operational with data
+- [ ] Prometheus alerting rules fire correctly for simulated failure conditions
+- [ ] DEX/AMM collector works when configured (optional — not required for base exit criteria)
+- [ ] Phase 10 validation suite passes with receiver metrics included
+- [ ] Receiver handles xrpld restart/unavailability gracefully (no crash, logs warning, retries)
+- [ ] Documentation complete: receiver README, metric reference, alerting playbook
+- [ ] Go receiver has unit tests with >80% coverage
+- [ ] 18 Grafana alert rules for Phase 7+ parity metrics evaluate correctly (Task 11.12)
+- [ ] Dual-datasource architecture documented with trade-offs (Task 11.13)

OpenTelemetryPlan/Phase2_taskList.md

@@ -204,3 +204,36 @@ Node health (`amendment_blocked`, `server_state`) is not part of the telemetry s
 **Deferred with rationale**: Tasks 2.1 (→Phase 3), 2.5 (low priority).
 **Dropped**: Task 2.8 (node health not duplicated on traces).
 **Superseded**: Task 2.2 (Phase 1c SpanGuard factory covers this).
+
+---
+
+## Known Issues / Future Work
+
+### Thread safety of TelemetryImpl::stop() vs startSpan()
+
+`TelemetryImpl::stop()` resets `sdkProvider_` (a `std::shared_ptr`) without
+synchronization. `getTracer()` reads the same member from RPC handler threads.
+This is a data race if any thread calls `startSpan()` concurrently with `stop()`.
+
+**Current mitigation**: `Application::stop()` shuts down `serverHandler_`,
+`overlay_`, and `jobQueue_` before calling `telemetry_->stop()`, so no callers
+remain. See comments in `Telemetry.cpp:stop()` and `Application.cpp`.
+
+**TODO**: Add an `std::atomic<bool> stopped_` flag checked in `getTracer()` to
+make this robust against future shutdown order changes.
+
+### Macro incompatibility: XRPL_TRACE_SPAN vs XRPL_TRACE_SET_ATTR
+
+`XRPL_TRACE_SPAN` and `XRPL_TRACE_SPAN_KIND` declare `_xrpl_guard_` as a bare
+`SpanGuard`, but `XRPL_TRACE_SET_ATTR` and `XRPL_TRACE_EXCEPTION` call
+`_xrpl_guard_.has_value()` which requires `std::optional<SpanGuard>`. Using
+`XRPL_TRACE_SPAN` followed by `XRPL_TRACE_SET_ATTR` in the same scope would
+fail to compile.
+
+**Current mitigation**: No call site currently uses `XRPL_TRACE_SPAN` — all
+production code uses the conditional macros (`XRPL_TRACE_RPC`, `XRPL_TRACE_TX`,
+etc.) which correctly wrap the guard in `std::optional`.
+
+**TODO**: Either make `XRPL_TRACE_SPAN`/`XRPL_TRACE_SPAN_KIND` also wrap in
+`std::optional`, or document that `XRPL_TRACE_SET_ATTR` is only compatible with
+the conditional macros.

OpenTelemetryPlan/Phase3_taskList.md

@@ -276,7 +276,7 @@
 
 - [ ] `tx.receive` spans carry `peer_version` attribute with a non-empty version string
 - [ ] Attribute is omitted (not set to empty string) when `getVersion()` returns empty
-- [ ] Attribute visible in Jaeger span detail view
+- [ ] Attribute visible in Tempo trace detail view
 
 ---
 
@@ -529,3 +529,14 @@ This gives the best of both worlds: guaranteed cross-node correlation via determ
 - [ ] <5% overhead on transaction throughput
 - [x] Deterministic trace_id: same trace_id for same tx across all nodes
 - [x] Protobuf span_id propagation preserves parent-child ordering when available
+
+---
+
+## Known Issues / Future Work
+
+### Unused trace_state proto field
+
+The `TraceContext.trace_state` field (field 4) in `xrpl.proto` is reserved for
+W3C `tracestate` vendor-specific key-value pairs but is not read or written by
+`TraceContextPropagator`. Wire it when cross-vendor trace propagation is needed.
+No wire cost since proto `optional` fields are zero-cost when absent.

OpenTelemetryPlan/Phase4_taskList.md

@@ -366,7 +366,7 @@ Two strategies for cross-node trace correlation, switchable via config:
 Derive `trace_id = SHA256(previousLedger.id())[0:16]` so all nodes in the same
 consensus round share the same trace_id without P2P context propagation.
 
-- **Pros**: All nodes appear in the same trace in Tempo/Jaeger automatically.
+- **Pros**: All nodes appear in the same trace in Tempo automatically.
   No collector-side post-processing needed.
 - **Cons**: Overrides OTel's random trace_id generation; requires custom
   `IdGenerator` or manual span context construction.
@@ -913,7 +913,7 @@ Received messages use **span links** (follows-from), NOT parent-child:
 
 - The receiver's processing span links to the sender's context
 - This preserves each node's independent trace tree
-- Cross-node correlation visible via linked traces in Tempo/Jaeger
+- Cross-node correlation visible via linked traces in Tempo
 
 ## Interaction with Deterministic Trace ID (Strategy A)
 

OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md

@@ -0,0 +1,221 @@
+# Phase 5: Integration Test Task List
+
+> **Goal**: End-to-end verification of the complete telemetry pipeline using a
+> 6-node consensus network. Proves that RPC, transaction, and consensus spans
+> flow through the observability stack (otel-collector, Tempo, Prometheus,
+> Grafana) under realistic conditions.
+>
+> **Scope**: Integration test script, manual testing plan, 6-node local network
+> setup, Tempo/Prometheus/Grafana verification.
+>
+> **Branch**: `pratik/otel-phase5-docs-deployment`
+
+### Related Plan Documents
+
+| Document                                                         | Relevance                                  |
+| ---------------------------------------------------------------- | ------------------------------------------ |
+| [07-observability-backends.md](./07-observability-backends.md)   | Tempo, Grafana, Prometheus setup           |
+| [05-configuration-reference.md](./05-configuration-reference.md) | Collector config, Docker Compose           |
+| [06-implementation-phases.md](./06-implementation-phases.md)     | Phase 5 tasks, definition of done          |
+| [Phase5_taskList.md](./Phase5_taskList.md)                       | Phase 5 main task list (5.6 = integration) |
+
+---
+
+## Task IT.1: Create Integration Test Script
+
+**Objective**: Automated bash script that stands up a 6-node xrpld network
+with telemetry, exercises all span categories, and verifies data in
+Tempo/Prometheus.
+
+**What to do**:
+
+- Create `docker/telemetry/integration-test.sh`:
+  - Prerequisites check (docker, xrpld binary, curl, jq)
+  - Start observability stack via `docker compose`
+  - Generate 6 validator key pairs via temp standalone xrpld
+  - Generate 6 node configs + shared `validators.txt`
+  - Start 6 xrpld nodes in consensus mode (`--start`, no `-a`)
+  - Wait for all nodes to reach `"proposing"` state (120s timeout)
+
+**Key new file**: `docker/telemetry/integration-test.sh`
+
+**Verification**:
+
+- [ ] Script starts without errors
+- [ ] All 6 nodes reach "proposing" state
+- [ ] Observability stack is healthy (otel-collector, Tempo, Prometheus, Grafana)
+
+---
+
+## Task IT.2: RPC Span Verification (Phase 2)
+
+**Objective**: Verify RPC spans flow through the telemetry pipeline.
+
+**What to do**:
+
+- Send `server_info`, `server_state`, `ledger` RPCs to node1 (port 5005)
+- Wait for batch export (5s)
+- Query Tempo API for:
+  - `rpc.request` spans (ServerHandler::onRequest)
+  - `rpc.process` spans (ServerHandler::processRequest)
+  - `rpc.command.server_info` spans (callMethod)
+  - `rpc.command.server_state` spans (callMethod)
+  - `rpc.command.ledger` spans (callMethod)
+- Verify `command` attribute present on `rpc.command.*` spans
+
+**Verification**:
+
+- [ ] Tempo shows `rpc.request` traces
+- [ ] Tempo shows `rpc.process` traces
+- [ ] Tempo shows `rpc.command.*` traces with correct attributes
+
+---
+
+## Task IT.3: Transaction Span Verification (Phase 3)
+
+**Objective**: Verify transaction spans flow through the telemetry pipeline.
+
+**What to do**:
+
+- Get genesis account sequence via `account_info` RPC
+- Submit Payment transaction using genesis seed (`snoPBrXtMeMyMHUVTgbuqAfg1SUTb`)
+- Wait for consensus inclusion (10s)
+- Query Tempo API for:
+  - `tx.process` spans (NetworkOPsImp::processTransaction) on submitting node
+  - `tx.receive` spans (PeerImp::handleTransaction) on peer nodes
+- Verify `xrpl.tx.hash` attribute on `tx.process` spans
+- Verify `xrpl.peer.id` attribute on `tx.receive` spans
+
+**Verification**:
+
+- [ ] Tempo shows `tx.process` traces with `xrpl.tx.hash`
+- [ ] Tempo shows `tx.receive` traces with `xrpl.peer.id`
+
+---
+
+## Task IT.4: Consensus Span Verification (Phase 4)
+
+**Objective**: Verify consensus spans flow through the telemetry pipeline.
+
+**What to do**:
+
+- Consensus runs automatically in 6-node network
+- Query Tempo API for:
+  - `consensus.proposal.send` (Adaptor::propose)
+  - `consensus.ledger_close` (Adaptor::onClose)
+  - `consensus.accept` (Adaptor::onAccept)
+  - `consensus.validation.send` (Adaptor::validate)
+- Verify attributes:
+  - `xrpl.consensus.mode` on `consensus.ledger_close`
+  - `xrpl.consensus.proposers` on `consensus.accept`
+  - `xrpl.consensus.ledger.seq` on `consensus.validation.send`
+
+**Verification**:
+
+- [ ] Tempo shows `consensus.ledger_close` traces with `xrpl.consensus.mode`
+- [ ] Tempo shows `consensus.accept` traces with `xrpl.consensus.proposers`
+- [ ] Tempo shows `consensus.proposal.send` traces
+- [ ] Tempo shows `consensus.validation.send` traces
+
+---
+
+## Task IT.5: Spanmetrics Verification (Phase 5)
+
+**Objective**: Verify spanmetrics connector derives RED metrics from spans.
+
+**What to do**:
+
+- Query Prometheus for `traces_span_metrics_calls_total`
+- Query Prometheus for `traces_span_metrics_duration_milliseconds_count`
+- Verify Grafana loads at `http://localhost:3000`
+
+**Verification**:
+
+- [ ] Prometheus returns non-empty results for `traces_span_metrics_calls_total`
+- [ ] Prometheus returns non-empty results for duration histogram
+- [ ] Grafana UI accessible with dashboards visible
+
+---
+
+## Task IT.6: Manual Testing Plan
+
+**Objective**: Document how to run tests manually for future reference.
+
+**What to do**:
+
+- Create `docker/telemetry/TESTING.md` with:
+  - Prerequisites section
+  - Single-node standalone test (quick verification)
+  - 6-node consensus test (full verification)
+  - Expected span catalog (all 12 span names with attributes)
+  - Verification queries (Tempo API, Prometheus API)
+  - Troubleshooting guide
+
+**Key new file**: `docker/telemetry/TESTING.md`
+
+**Verification**:
+
+- [ ] Document covers both single-node and multi-node testing
+- [ ] All 12 span names documented with source file and attributes
+- [ ] Troubleshooting section covers common failure modes
+
+---
+
+## Task IT.7: Run and Verify
+
+**Objective**: Execute the integration test and validate results.
+
+**What to do**:
+
+- Run `docker/telemetry/integration-test.sh` locally
+- Debug any failures
+- Leave stack running for manual verification
+- Share URLs:
+  - Tempo: `http://localhost:3200`
+  - Grafana: `http://localhost:3000`
+  - Prometheus: `http://localhost:9090`
+
+**Verification**:
+
+- [ ] Script completes with all checks passing
+- [ ] Tempo UI shows xrpld service with all expected span names
+- [ ] Grafana dashboards load and show data
+
+---
+
+## Task IT.8: Commit
+
+**Objective**: Commit all new files to Phase 5 branch.
+
+**What to do**:
+
+- Run `pcc` (pre-commit checks)
+- Commit 3 new files to `pratik/otel-phase5-docs-deployment`
+
+**Verification**:
+
+- [ ] `pcc` passes
+- [ ] Commit created on Phase 5 branch
+
+---
+
+## Summary
+
+| Task | Description                   | New Files | Depends On |
+| ---- | ----------------------------- | --------- | ---------- |
+| IT.1 | Integration test script       | 1         | Phase 5    |
+| IT.2 | RPC span verification         | 0         | IT.1       |
+| IT.3 | Transaction span verification | 0         | IT.1       |
+| IT.4 | Consensus span verification   | 0         | IT.1       |
+| IT.5 | Spanmetrics verification      | 0         | IT.1       |
+| IT.6 | Manual testing plan           | 1         | --         |
+| IT.7 | Run and verify                | 0         | IT.1-IT.6  |
+| IT.8 | Commit                        | 0         | IT.7       |
+
+**Exit Criteria**:
+
+- [ ] All 6 xrpld nodes reach "proposing" state
+- [ ] All 11 expected span names visible in Tempo
+- [ ] Spanmetrics available in Prometheus
+- [ ] Grafana dashboards show data
+- [ ] Manual testing plan document complete

OpenTelemetryPlan/Phase7_taskList.md

@@ -0,0 +1,587 @@
+# Phase 7: Native OTel Metrics Migration — Task List
+
+> **Goal**: Replace `StatsDCollector` with a native OpenTelemetry Metrics SDK implementation behind the existing `beast::insight::Collector` interface, eliminating the StatsD UDP dependency.
+>
+> **Scope**: New `OTelCollectorImpl` class, `CollectorManager` config change, OTel Collector pipeline update, Grafana dashboard metric name migration, integration tests.
+>
+> **Branch**: `pratik/otel-phase7-native-metrics` (from `pratik/otel-phase6-statsd`)
+
+### Related Plan Documents
+
+| Document                                                             | Relevance                                                       |
+| -------------------------------------------------------------------- | --------------------------------------------------------------- |
+| [06-implementation-phases.md](./06-implementation-phases.md)         | Phase 7 plan: motivation, architecture, exit criteria (§6.8)    |
+| [02-design-decisions.md](./02-design-decisions.md)                   | Collector interface design, beast::insight coexistence strategy |
+| [05-configuration-reference.md](./05-configuration-reference.md)     | `[insight]` and `[telemetry]` config sections                   |
+| [09-data-collection-reference.md](./09-data-collection-reference.md) | Complete metric inventory that must be preserved                |
+
+---
+
+## Task 7.1: Add OTel Metrics SDK to Build Dependencies
+
+**Objective**: Enable the OTel C++ Metrics SDK components in the build system.
+
+**What to do**:
+
+- Edit `conanfile.py`:
+  - Add OTel metrics SDK components to the dependency list when `telemetry=True`
+  - Components needed: `opentelemetry-cpp::metrics`, `opentelemetry-cpp::otlp_http_metric_exporter`
+
+- Edit `CMakeLists.txt` (telemetry section):
+  - Link `opentelemetry::metrics` and `opentelemetry::otlp_http_metric_exporter` targets
+
+**Key modified files**:
+
+- `conanfile.py`
+- `CMakeLists.txt` (or the relevant telemetry cmake target)
+
+**Reference**: [05-configuration-reference.md §5.3](./05-configuration-reference.md) — CMake integration
+
+---
+
+## Task 7.2: Implement OTelCollector Class
+
+**Objective**: Create the core `OTelCollector` implementation that maps beast::insight instruments to OTel Metrics SDK instruments.
+
+**What to do**:
+
+- Create `include/xrpl/beast/insight/OTelCollector.h`:
+  - Public factory: `static std::shared_ptr<OTelCollector> New(std::string const& endpoint, std::string const& prefix, beast::Journal journal)`
+  - Derives from `StatsDCollector` (or directly from `Collector` — TBD based on shared code)
+
+- Create `src/libxrpl/beast/insight/OTelCollector.cpp` (~400-500 lines):
+  - **OTelCounterImpl**: Wraps `opentelemetry::metrics::Counter<int64_t>`. `increment(amount)` calls `counter->Add(amount)`.
+  - **OTelGaugeImpl**: Uses `opentelemetry::metrics::ObservableGauge<uint64_t>` with an async callback. `set(value)` stores value atomically; callback reads it during collection.
+  - **OTelMeterImpl**: Wraps `opentelemetry::metrics::Counter<uint64_t>`. `increment(amount)` calls `counter->Add(amount)`. Semantically identical to Counter but unsigned.
+  - **OTelEventImpl**: Wraps `opentelemetry::metrics::Histogram<double>`. `notify(duration)` calls `histogram->Record(duration.count())`. Uses explicit bucket boundaries matching SpanMetrics: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms.
+  - **OTelHookImpl**: Stores handler function. Called during periodic metric collection (same 1s pattern via PeriodicMetricReader).
+  - **OTelCollectorImp**: Main class.
+    - Creates `MeterProvider` with `PeriodicMetricReader` (1s export interval)
+    - Creates `OtlpHttpMetricExporter` pointing to `[telemetry]` endpoint
+    - Sets resource attributes (service.name, service.instance.id) matching trace exporter
+    - Implements all `make_*()` factory methods
+    - Prefixes metric names with `[insight] prefix=` value
+
+- Guard all OTel SDK includes with `#ifdef XRPL_ENABLE_TELEMETRY` to compile to `NullCollector` equivalents when telemetry disabled.
+
+**Key new files**:
+
+- `include/xrpl/beast/insight/OTelCollector.h`
+- `src/libxrpl/beast/insight/OTelCollector.cpp`
+
+**Key patterns to follow**:
+
+- Match `StatsDCollector.cpp` structure: private impl classes, intrusive list for metrics, strand-based thread safety
+- Match existing telemetry code style from `src/libxrpl/telemetry/Telemetry.cpp`
+- Use RAII for MeterProvider lifecycle (shutdown on destructor)
+
+**Reference**: [04-code-samples.md](./04-code-samples.md) — code style and patterns
+
+---
+
+## Task 7.3: Update CollectorManager
+
+**Objective**: Add `server=otel` config option to route metric creation to the new OTel backend.
+
+**What to do**:
+
+- Edit `src/xrpld/app/main/CollectorManager.cpp`:
+  - In the constructor, add a third branch after `server == "statsd"`:
+    ```cpp
+    else if (server == "otel")
+    {
+        // Read endpoint from [telemetry] section
+        auto const endpoint = get(telemetryParams, "endpoint",
+            "http://localhost:4318/v1/metrics");
+        std::string const& prefix(get(params, "prefix"));
+        collector_ = beast::insight::OTelCollector::New(
+            endpoint, prefix, journal);
+    }
+    ```
+  - This requires access to the `[telemetry]` config section — may need to pass it as a parameter or read from Application config.
+
+- Edit `src/xrpld/app/main/CollectorManager.h`:
+  - Add `#include <xrpl/beast/insight/OTelCollector.h>`
+
+**Key modified files**:
+
+- `src/xrpld/app/main/CollectorManager.cpp`
+- `src/xrpld/app/main/CollectorManager.h`
+
+---
+
+## Task 7.4: Update OTel Collector Configuration
+
+**Objective**: Add a metrics pipeline to the OTLP receiver and remove the StatsD receiver dependency.
+
+**What to do**:
+
+- Edit `docker/telemetry/otel-collector-config.yaml`:
+  - Remove `statsd` receiver (no longer needed when `server=otel`)
+  - Add metrics pipeline under `service.pipelines`:
+    ```yaml
+    metrics:
+      receivers: [otlp, spanmetrics]
+      processors: [batch]
+      exporters: [prometheus]
+    ```
+  - The OTLP receiver already listens on :4318 — it just needs to be added to the metrics pipeline receivers.
+  - Keep `spanmetrics` connector in the metrics pipeline so span-derived RED metrics continue working.
+
+- Edit `docker/telemetry/docker-compose.yml`:
+  - Remove UDP :8125 port mapping from otel-collector service
+  - Update xrpld service config: change `[insight] server=statsd` to `server=otel`
+
+**Key modified files**:
+
+- `docker/telemetry/otel-collector-config.yaml`
+- `docker/telemetry/docker-compose.yml`
+
+**Note**: Keep a commented-out `statsd` receiver block for operators who need backward compatibility.
+
+---
+
+## Task 7.5: Preserve Metric Names in Prometheus
+
+**Objective**: Ensure existing Grafana dashboards continue working with identical metric names.
+
+**What to do**:
+
+- In `OTelCollector.cpp`, construct OTel instrument names to match existing Prometheus metric names:
+  - beast::insight `make_gauge("LedgerMaster", "Validated_Ledger_Age")` → OTel instrument name: `xrpld_LedgerMaster_Validated_Ledger_Age`
+  - The prefix + group + name concatenation must produce the same string as `StatsDCollector`'s format
+  - Use underscores as separators (matching StatsD convention)
+
+- Verify in integration test that key Prometheus queries still return data:
+  - `xrpld_LedgerMaster_Validated_Ledger_Age`
+  - `xrpld_Peer_Finder_Active_Inbound_Peers`
+  - `xrpld_rpc_requests`
+
+**Key consideration**: OTel Prometheus exporter may normalize metric names differently than StatsD receiver. Test this early (Task 7.2) and adjust naming strategy if needed. The OTel SDK's Prometheus exporter adds `_total` suffix to counters and converts dots to underscores — match existing conventions.
+
+---
+
+## Task 7.6: Update Grafana Dashboards
+
+**Objective**: Update the 3 StatsD dashboards if any metric names change due to OTLP export format differences.
+
+**What to do**:
+
+- If Task 7.5 confirms metric names are preserved exactly, no dashboard changes needed.
+- If OTLP export produces different names (e.g., `_total` suffix on counters), update:
+  - `docker/telemetry/grafana/dashboards/statsd-node-health.json`
+  - `docker/telemetry/grafana/dashboards/statsd-network-traffic.json`
+  - `docker/telemetry/grafana/dashboards/statsd-rpc-pathfinding.json`
+- Rename dashboard titles from "StatsD" to "System Metrics" or similar (since they're no longer StatsD-sourced).
+
+**Key modified files**:
+
+- `docker/telemetry/grafana/dashboards/statsd-*.json` (3 files, conditionally)
+
+---
+
+## Task 7.7: Update Integration Tests
+
+**Objective**: Verify the full OTLP metrics pipeline end-to-end.
+
+**What to do**:
+
+- Edit `docker/telemetry/integration-test.sh`:
+  - Update test config to use `[insight] server=otel`
+  - Verify metrics arrive in Prometheus via OTLP (not StatsD)
+  - Add check that StatsD receiver is no longer required
+  - Preserve all existing metric presence checks
+
+**Key modified files**:
+
+- `docker/telemetry/integration-test.sh`
+
+---
+
+## Task 7.8: Update Documentation
+
+**Objective**: Update all plan docs, runbook, and reference docs to reflect the migration.
+
+**What to do**:
+
+- Edit `docs/telemetry-runbook.md`:
+  - Update `[insight]` config examples to show `server=otel`
+  - Update troubleshooting section (no more StatsD UDP debugging)
+
+- Edit `OpenTelemetryPlan/09-data-collection-reference.md`:
+  - Update Data Flow Overview diagram (remove StatsD receiver)
+  - Update Section 2 header from "StatsD Metrics" to "System Metrics (OTel native)"
+  - Update config examples
+
+- Edit `OpenTelemetryPlan/05-configuration-reference.md`:
+  - Add `server=otel` option to `[insight]` section docs
+
+- Edit `docker/telemetry/TESTING.md`:
+  - Update setup instructions to use `server=otel`
+
+**Key modified files**:
+
+- `docs/telemetry-runbook.md`
+- `OpenTelemetryPlan/09-data-collection-reference.md`
+- `OpenTelemetryPlan/05-configuration-reference.md`
+- `docker/telemetry/TESTING.md`
+
+---
+
+## Task 7.9: ValidationTracker — Validation Agreement Computation
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md) — the most valuable metric from the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
+>
+> **Upstream**: Phase 4 Task 4.8 (validation span attributes provide ledger hash context).
+> **Downstream**: Phase 9 (Validator Health dashboard), Phase 10 (validation checks), Phase 11 (agreement alert rules).
+
+**Objective**: Implement a stateful class that tracks whether our validator's validations agree with network consensus, maintaining rolling 1h and 24h windows with an 8-second grace period and 5-minute late repair window.
+
+**Architecture**:
+
+```
+consensus.validation.send ────> ValidationTracker ────> MetricsRegistry
+(records our validation         (reconciles after       (exports agreement
+ for ledger X)                   8s grace period)        gauges every 10s)
+
+ledger.validate ──────────────> ValidationTracker
+(records which ledger           (marks ledger X as
+ network validated)              agreed or missed)
+```
+
+**What to do**:
+
+- Create `src/xrpld/telemetry/ValidationTracker.h`:
+  - `recordOurValidation(ledgerHash, ledgerSeq)` — called when we send a validation
+  - `recordNetworkValidation(ledgerHash, seq)` — called when a ledger is fully validated
+  - `reconcile()` — called periodically; reconciles pending ledger events after 8s grace period
+  - Getters: `agreementPct1h()`, `agreementPct24h()`, `agreements1h()`, `missed1h()`, `agreements24h()`, `missed24h()`, `totalAgreements()`, `totalMissed()`, `totalValidationsSent()`, `totalValidationsChecked()`
+  - Thread-safety: atomics for counters, mutex for window deques
+
+- Create `src/xrpld/telemetry/detail/ValidationTracker.cpp`:
+  - Reconciliation logic: after 8s grace period, check if `weValidated && networkValidated && sameHash` → agreement; else missed
+  - Late repair: if a late validation arrives within 5 minutes, correct a false-positive miss
+  - Sliding window: `std::deque<WindowEvent>` evicts entries older than 1h/24h on each reconciliation pass
+  - Ring buffer of 1000 `LedgerEvent` structs for pending reconciliation
+
+- Add recording hooks (modifying Phase 4 code from Phase 7 branch):
+  - `RCLConsensus.cpp` `validate()`: call `tracker.recordOurValidation()`
+  - `LedgerMaster.cpp` fully-validated path: call `tracker.recordNetworkValidation()`
+
+**Key data structures**:
+
+```cpp
+struct LedgerEvent {
+    uint256 ledgerHash;
+    LedgerIndex seq;
+    TimePoint closeTime;
+    bool weValidated = false;
+    bool networkValidated = false;
+    bool reconciled = false;
+    bool agreed = false;
+};
+
+struct WindowEvent {
+    TimePoint time;
+    bool agreed;
+};
+```
+
+**Key new files**:
+
+- `src/xrpld/telemetry/ValidationTracker.h`
+- `src/xrpld/telemetry/detail/ValidationTracker.cpp`
+
+**Key modified files**:
+
+- `src/xrpld/telemetry/MetricsRegistry.h` (add ValidationTracker member)
+- `src/xrpld/telemetry/MetricsRegistry.cpp` (add gauge callback reading from tracker)
+- `src/xrpld/app/consensus/RCLConsensus.cpp` (add recording hooks)
+- `src/xrpld/app/ledger/detail/LedgerMaster.cpp` (add recording hook)
+
+**Exit Criteria**:
+
+- [ ] ValidationTracker correctly tracks agreement with 8s grace period
+- [ ] 5-minute late repair corrects false-positive misses
+- [ ] Thread-safe (atomics + mutex for window deques)
+- [ ] Rolling windows correctly evict stale entries
+- [ ] Unit tests: normal agreement, missed validation, late repair, window eviction
+
+---
+
+## Task 7.10: Validator Health Observable Gauges
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md)
+
+**Objective**: Export amendment blocked, UNL health, and quorum data as a native OTel observable gauge.
+
+**What to do**:
+
+- In `MetricsRegistry.cpp` `registerAsyncGauges()`, add:
+
+```cpp
+validatorHealthGauge_ = meter_->CreateDoubleObservableGauge(
+    "xrpld_validator_health", "Validator health indicators");
+```
+
+**Gauge label values**:
+
+| Label `metric=`     | Type   | Source                                            |
+| ------------------- | ------ | ------------------------------------------------- |
+| `amendment_blocked` | int64  | `app_.getOPs().isAmendmentBlocked()` → 0/1        |
+| `unl_blocked`       | int64  | `app_.getOPs().isUNLBlocked()` → 0/1              |
+| `unl_expiry_days`   | double | `app_.validators().expires()` → days until expiry |
+| `validation_quorum` | int64  | `app_.validators().quorum()`                      |
+
+### Sub-task 7.10a: Per-Validator Validation Count (Flag Ledger Window)
+
+**Objective**: Track how many ledgers each UNL validator has validated over
+the last 256 consecutive ledgers (one flag ledger window). This is the key
+UNL participation metric — validators consistently below threshold may be
+candidates for removal from the UNL.
+
+**What to do**:
+
+- Add a new observable gauge:
+
+```cpp
+validatorParticipationGauge_ = meter_->CreateInt64ObservableGauge(
+    "xrpld_validator_participation",
+    "Per-validator validation count over the last 256 ledgers");
+```
+
+- The callback queries `app_.getValidations()` to get the trusted
+  validation set for each of the last 256 ledger hashes (from
+  `LedgerMaster::getValidatedLedger()` walking backwards). For each
+  validator public key in the UNL, count how many of those 256 ledgers
+  have a matching validation.
+
+- **Label dimensions**:
+  - `validator` — base58-encoded validator master public key
+  - `exported_instance` — this node's identity (standard)
+
+- **Emission**: every flag ledger (256 ledgers, ~15 minutes) or on a
+  10-second async gauge callback with cached results (recompute only
+  at flag ledger boundaries).
+
+- **Data source**: `RCLValidations::getTrustedForLedger(hash, seq)` returns
+  `std::vector<std::shared_ptr<STValidation>>` with `getSignerPublic()`
+  for each. The UNL list is from `app_.getValidators().getTrustedMasterKeys()`.
+
+- **Dashboard panel**: Add a table panel to the Validator Health dashboard
+  showing `xrpld_validator_participation` grouped by `validator` label,
+  with a threshold color (green >= 240, yellow >= 200, red < 200).
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
+
+**Exit Criteria**:
+
+- [ ] Gauge emits one time series per UNL validator
+- [ ] Values range 0-256 and update at flag ledger boundaries
+- [ ] Grafana table panel shows per-validator participation
+- [ ] Validators below 75% participation are highlighted in red
+
+---
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
+
+**Exit Criteria**:
+
+- [ ] All 4 base label values emitted every 10s
+- [ ] `unl_expiry_days` is negative when expired, positive when active
+- [ ] Per-validator participation gauge emits at flag ledger boundaries
+- [ ] Values visible in Prometheus
+
+---
+
+## Task 7.11: Peer Quality Observable Gauges
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md)
+
+**Objective**: Export peer health aggregates (latency P90, insane peers, version awareness) as a native OTel observable gauge.
+
+**What to do**:
+
+- In `MetricsRegistry.cpp` `registerAsyncGauges()`, add a callback that iterates `app_.overlay().foreach(...)` to:
+  - Collect per-peer latency values, sort, compute P90
+  - Count peers with `tracking_ == diverged` (insane)
+  - Compare peer `getVersion()` to own version for upgrade awareness
+
+**Gauge label values**:
+
+| Label `metric=`            | Type   | Source                                |
+| -------------------------- | ------ | ------------------------------------- |
+| `peer_latency_p90_ms`      | double | P90 from sorted peer latencies        |
+| `peers_insane_count`       | int64  | Peers with diverged tracking status   |
+| `peers_higher_version_pct` | double | % of peers on newer xrpld version     |
+| `upgrade_recommended`      | int64  | 1 if `peers_higher_version_pct > 60%` |
+
+**Implementation note**: The callback runs every 10s on the metrics reader thread. Iterating ~50-200 peers is acceptable overhead.
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
+
+**Exit Criteria**:
+
+- [ ] P90 latency computed correctly
+- [ ] Insane count matches `peers` RPC output
+- [ ] Version comparison handles format variations (e.g., "xrpld-2.4.0-rc1")
+
+---
+
+## Task 7.12: Ledger Economy Observable Gauges
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md)
+
+**Objective**: Export fee, reserve, ledger age, and transaction rate as a native OTel observable gauge.
+
+**Gauge label values**:
+
+| Label `metric=`      | Type   | Source                                              |
+| -------------------- | ------ | --------------------------------------------------- |
+| `base_fee_xrp`       | double | Base fee from validated ledger fee settings (drops) |
+| `reserve_base_xrp`   | double | Account reserve from validated ledger (drops)       |
+| `reserve_inc_xrp`    | double | Owner reserve increment (drops)                     |
+| `ledger_age_seconds` | double | `now - lastValidatedCloseTime`                      |
+| `transaction_rate`   | double | Derived: tx count delta / time delta (smoothed)     |
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
+
+**Exit Criteria**:
+
+- [ ] Fee values match `server_info` RPC output
+- [ ] `ledger_age_seconds` increases monotonically between ledger closes
+- [ ] `transaction_rate` is smoothed (rolling average)
+
+---
+
+## Task 7.13: State Tracking Observable Gauges
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md)
+
+**Objective**: Export extended state value (0-6 encoding combining OperatingMode + ConsensusMode) and time-in-current-state.
+
+**Gauge label values**:
+
+| Label `metric=`                 | Type   | Source                                          |
+| ------------------------------- | ------ | ----------------------------------------------- |
+| `state_value`                   | int64  | 0-6 encoding (see spec for mapping)             |
+| `time_in_current_state_seconds` | double | `now - lastModeChangeTime` from StateAccounting |
+
+**State value encoding**: 0=disconnected, 1=connected, 2=syncing, 3=tracking, 4=full, 5=validating (full + validating), 6=proposing (full + proposing).
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
+
+**Exit Criteria**:
+
+- [ ] `state_value` correctly combines OperatingMode and ConsensusMode
+- [ ] `time_in_current_state_seconds` resets on mode change
+
+---
+
+## Task 7.14: Storage Detail and Sync Info Gauges
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md)
+
+**Objective**: Export NuDB-specific storage size and initial sync duration.
+
+**Gauge label values**:
+
+| Gauge Name             | Label `metric=`                 | Type   | Source                        |
+| ---------------------- | ------------------------------- | ------ | ----------------------------- |
+| `xrpld_storage_detail` | `nudb_bytes`                    | int64  | NuDB backend file size        |
+| `xrpld_sync_info`      | `initial_sync_duration_seconds` | double | Time from start to first FULL |
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
+
+**Exit Criteria**:
+
+- [ ] NuDB file size reported in bytes (0 if NuDB not configured)
+- [ ] Sync duration captured once and remains stable after reaching FULL
+
+---
+
+## Task 7.15: New Synchronous Counters
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md)
+
+**Objective**: Add 7 new event counters incremented at their respective instrumentation sites.
+
+| Counter Name                        | Increment Site                   | Source File           |
+| ----------------------------------- | -------------------------------- | --------------------- |
+| `xrpld_ledgers_closed_total`        | `onAccept()` in consensus        | RCLConsensus.cpp      |
+| `xrpld_validations_sent_total`      | `validate()` in consensus        | RCLConsensus.cpp      |
+| `xrpld_validations_checked_total`   | Network validation received      | LedgerMaster.cpp      |
+| `xrpld_validation_agreements_total` | ValidationTracker reconciliation | ValidationTracker.cpp |
+| `xrpld_validation_missed_total`     | ValidationTracker reconciliation | ValidationTracker.cpp |
+| `xrpld_state_changes_total`         | `setMode()` in NetworkOPs        | NetworkOPs.cpp        |
+| `xrpld_jq_trans_overflow_total`     | Job queue overflow path          | JobQueue.cpp          |
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (declarations), plus recording sites in RCLConsensus.cpp, LedgerMaster.cpp, NetworkOPs.cpp, JobQueue.cpp
+
+**Exit Criteria**:
+
+- [ ] All 7 counters monotonically increase during normal operation
+- [ ] Counter values match expected rates (e.g., ledgers_closed ≈ 1 per 3-5s)
+
+---
+
+## Task 7.16: Validation Agreement Observable Gauge
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md)
+
+**Objective**: Export rolling window agreement stats from `ValidationTracker` (Task 7.9).
+
+**Gauge label values**:
+
+| Gauge Name                   | Label `metric=`     | Type   | Source                      |
+| ---------------------------- | ------------------- | ------ | --------------------------- |
+| `xrpld_validation_agreement` | `agreement_pct_1h`  | double | `tracker.agreementPct1h()`  |
+|                              | `agreements_1h`     | int64  | `tracker.agreements1h()`    |
+|                              | `missed_1h`         | int64  | `tracker.missed1h()`        |
+|                              | `agreement_pct_24h` | double | `tracker.agreementPct24h()` |
+|                              | `agreements_24h`    | int64  | `tracker.agreements24h()`   |
+|                              | `missed_24h`        | int64  | `tracker.missed24h()`       |
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.cpp`
+
+**Exit Criteria**:
+
+- [ ] Agreement percentages in range [0.0, 100.0]
+- [ ] Window stats stabilize after 1h/24h of operation
+
+---
+
+## Summary Table
+
+| Task | Description                            | New Files | Modified Files | Depends On |
+| ---- | -------------------------------------- | --------- | -------------- | ---------- |
+| 7.1  | Add OTel Metrics SDK to build deps     | 0         | 2              | —          |
+| 7.2  | Implement OTelCollector class          | 2         | 0              | 7.1        |
+| 7.3  | Update CollectorManager config routing | 0         | 2              | 7.2        |
+| 7.4  | Update OTel Collector YAML and Docker  | 0         | 2              | 7.3        |
+| 7.5  | Preserve metric names in Prometheus    | 0         | 1              | 7.2        |
+| 7.6  | Update Grafana dashboards (if needed)  | 0         | 3              | 7.5        |
+| 7.7  | Update integration tests               | 0         | 1              | 7.4        |
+| 7.8  | Update documentation                   | 0         | 4              | 7.6        |
+| 7.9  | ValidationTracker (agreement tracking) | 2         | 4              | 7.2, P4.8  |
+| 7.10 | Validator health observable gauges     | 0         | 2              | 7.2        |
+| 7.11 | Peer quality observable gauges         | 0         | 2              | 7.2        |
+| 7.12 | Ledger economy observable gauges       | 0         | 2              | 7.2        |
+| 7.13 | State tracking observable gauges       | 0         | 2              | 7.2        |
+| 7.14 | Storage detail and sync info gauges    | 0         | 2              | 7.2        |
+| 7.15 | New synchronous counters               | 0         | 6              | 7.2        |
+| 7.16 | Validation agreement observable gauge  | 0         | 1              | 7.9        |
+
+**Parallel work**: Tasks 7.4 and 7.5 can run in parallel after 7.2/7.3 complete. Task 7.6 depends on 7.5's findings. Tasks 7.7 and 7.8 can run in parallel after 7.6. Tasks 7.10-7.14 can all run in parallel after 7.2. Task 7.15 depends on 7.2. Task 7.16 depends on 7.9. Task 7.9 depends on 7.2 and Phase 4 Task 4.8.
+
+**Exit Criteria** (from [06-implementation-phases.md §6.8](./06-implementation-phases.md)):
+
+- [ ] All 255+ metrics visible in Prometheus via OTLP pipeline (no StatsD receiver)
+- [ ] `server=otel` is the default in development docker-compose
+- [ ] `server=statsd` still works as a fallback
+- [ ] Existing Grafana dashboards display data correctly
+- [ ] Integration test passes with OTLP-only metrics pipeline
+- [ ] No performance regression vs StatsD baseline (< 1% CPU overhead)
+- [ ] Deferred Task 6.1 (`|m` wire format) no longer relevant — Meter mapped to OTel Counter
+- [ ] ValidationTracker agreement % stabilizes after 1h under normal consensus
+- [ ] All new gauges and counters visible in Prometheus with non-zero values

OpenTelemetryPlan/Phase8_taskList.md

@@ -0,0 +1,241 @@
+# Phase 8: Log-Trace Correlation and Centralized Log Ingestion — Task List
+
+> **Goal**: Inject trace context (trace_id, span_id) into xrpld's Journal log output for log-trace correlation, and add OTel Collector filelog receiver to ingest logs into Grafana Loki for unified observability.
+>
+> **Scope**: Two independent sub-phases — 8a (code change: trace_id in logs) and 8b (infra only: filelog receiver to Loki). No changes to the `beast::Journal` public API.
+>
+> **Branch**: `pratik/otel-phase8-log-correlation` (from `pratik/otel-phase7-native-metrics`)
+
+### Related Plan Documents
+
+| Document                                                         | Relevance                                                      |
+| ---------------------------------------------------------------- | -------------------------------------------------------------- |
+| [06-implementation-phases.md](./06-implementation-phases.md)     | Phase 8 plan: motivation, architecture, exit criteria (§6.8.1) |
+| [07-observability-backends.md](./07-observability-backends.md)   | Loki backend recommendation, Grafana data source provisioning  |
+| [Phase7_taskList.md](./Phase7_taskList.md)                       | Prerequisite — native OTel metrics pipeline must be working    |
+| [05-configuration-reference.md](./05-configuration-reference.md) | `[telemetry]` config (trace_id injection toggle)               |
+
+---
+
+## Task 8.1: Inject trace_id into Logs::format()
+
+**Objective**: Add OTel trace context to every log line that is emitted within an active span.
+
+**What to do**:
+
+- Edit `src/libxrpl/basics/Log.cpp`:
+  - In `Logs::format()` (around line 346), after severity is appended, check for active OTel span. The implementation checks the context value directly to avoid the heap allocation that `GetSpan()` performs on the no-span path:
+    ```cpp
+    #ifdef XRPL_ENABLE_TELEMETRY
+    {
+        auto context = opentelemetry::context::RuntimeContext::GetCurrent();
+        auto spanValue = context.GetValue(opentelemetry::trace::kSpanKey);
+        if (opentelemetry::nostd::holds_alternative<
+                opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>>(spanValue))
+        {
+            auto span = opentelemetry::nostd::get<
+                opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>>(spanValue);
+            auto spanCtx = span->GetContext();
+            if (spanCtx.IsValid())
+            {
+                char traceId[32], spanId[16];
+                spanCtx.trace_id().ToLowerBase16(
+                    opentelemetry::nostd::span<char, 32>{traceId});
+                spanCtx.span_id().ToLowerBase16(
+                    opentelemetry::nostd::span<char, 16>{spanId});
+                output += "trace_id=";
+                output.append(traceId, 32);
+                output += " span_id=";
+                output.append(spanId, 16);
+                output += ' ';
+            }
+        }
+    }
+    #endif
+    ```
+  - Add `#include` for OTel context headers, guarded by `#ifdef XRPL_ENABLE_TELEMETRY`
+
+- Edit `include/xrpl/basics/Log.h`:
+  - No changes needed — format() signature unchanged
+
+**Key modified files**:
+
+- `src/libxrpl/basics/Log.cpp`
+
+**Performance note**: The implementation checks the thread-local context value directly (avoiding the heap allocation that `GetSpan()` performs on the no-span path). On threads without an active span (~99% of log lines), the cost is a thread-local read + variant type check (~15-20ns). On the active-span path, an additional shared_ptr copy + `GetContext()` + `IsValid()` adds ~50ns total. Overhead is negligible at typical logging rates.
+
+---
+
+## Task 8.2: Add Loki to Docker Compose Stack
+
+**Objective**: Add Grafana Loki as a log storage backend in the development observability stack.
+
+**What to do**:
+
+- Edit `docker/telemetry/docker-compose.yml`:
+  - Add Loki service:
+    ```yaml
+    loki:
+      image: grafana/loki:2.9.0
+      ports:
+        - "3100:3100"
+      command: -config.file=/etc/loki/local-config.yaml
+    ```
+  - Add Loki as a Grafana data source in provisioning
+
+- Create `docker/telemetry/grafana/provisioning/datasources/loki.yaml`:
+  - Configure Loki data source with derived fields linking `trace_id` to Tempo
+
+**Key new files**:
+
+- `docker/telemetry/grafana/provisioning/datasources/loki.yaml`
+
+**Key modified files**:
+
+- `docker/telemetry/docker-compose.yml`
+
+---
+
+## Task 8.3: Add Filelog Receiver to OTel Collector
+
+**Objective**: Configure the OTel Collector to tail xrpld's log file and export to Loki.
+
+**What to do**:
+
+- Edit `docker/telemetry/otel-collector-config.yaml`:
+  - Add `filelog` receiver:
+    ```yaml
+    receivers:
+      filelog:
+        include: [/var/log/rippled/debug.log]
+        operators:
+          - type: regex_parser
+            regex: '^(?P<timestamp>\S+)\s+(?P<partition>\S+):(?P<severity>\S+)\s+(?:trace_id=(?P<trace_id>[a-f0-9]+)\s+span_id=(?P<span_id>[a-f0-9]+)\s+)?(?P<message>.*)$'
+            timestamp:
+              parse_from: attributes.timestamp
+              layout: "%Y-%m-%dT%H:%M:%S.%fZ"
+    ```
+  - Add logs pipeline:
+    ```yaml
+    service:
+      pipelines:
+        logs:
+          receivers: [filelog]
+          processors: [batch]
+          exporters: [otlp/loki]
+    ```
+  - Add Loki exporter:
+    ```yaml
+    exporters:
+      otlp/loki:
+        endpoint: loki:3100
+        tls:
+          insecure: true
+    ```
+
+- Mount xrpld's log directory into the collector container via docker-compose volume
+
+**Key modified files**:
+
+- `docker/telemetry/otel-collector-config.yaml`
+- `docker/telemetry/docker-compose.yml`
+
+---
+
+## Task 8.4: Configure Grafana Trace-to-Log Correlation
+
+**Objective**: Enable one-click navigation from Tempo traces to Loki logs in Grafana.
+
+**What to do**:
+
+- Edit Grafana Tempo data source provisioning to add `tracesToLogs` configuration:
+
+  ```yaml
+  tracesToLogs:
+    datasourceUid: loki
+    filterByTraceID: true
+    filterBySpanID: false
+    tags: ["partition", "severity"]
+  ```
+
+- Edit Grafana Loki data source provisioning to add `derivedFields` linking trace_id back to Tempo:
+  ```yaml
+  derivedFields:
+    - datasourceUid: tempo
+      matcherRegex: "trace_id=(\\w+)"
+      name: TraceID
+      url: "$${__value.raw}"
+  ```
+
+**Key modified files**:
+
+- `docker/telemetry/grafana/provisioning/datasources/loki.yaml`
+- `docker/telemetry/grafana/provisioning/datasources/` (Tempo data source file)
+
+---
+
+## Task 8.5: Update Integration Tests
+
+**Objective**: Verify trace_id appears in logs and Loki correlation works.
+
+**What to do**:
+
+- Edit `docker/telemetry/integration-test.sh`:
+  - After sending RPC requests (which create spans), grep xrpld's log output for `trace_id=`
+  - Verify trace_id matches a trace visible in Tempo
+  - Optionally: query Loki via API to confirm log ingestion
+
+**Key modified files**:
+
+- `docker/telemetry/integration-test.sh`
+
+---
+
+## Task 8.6: Update Documentation
+
+**Objective**: Document the log correlation feature in runbook and reference docs.
+
+**What to do**:
+
+- Edit `docs/telemetry-runbook.md`:
+  - Add "Log-Trace Correlation" section explaining how to use Grafana Tempo -> Loki linking
+  - Add LogQL query examples for filtering by trace_id
+
+- Edit `OpenTelemetryPlan/09-data-collection-reference.md`:
+  - Add new section "3. Log Correlation" between SpanMetrics and StatsD sections
+  - Document the log format with trace_id injection
+  - Document Loki as a new backend
+
+- Edit `docker/telemetry/TESTING.md`:
+  - Add log correlation verification steps
+
+**Key modified files**:
+
+- `docs/telemetry-runbook.md`
+- `OpenTelemetryPlan/09-data-collection-reference.md`
+- `docker/telemetry/TESTING.md`
+
+---
+
+## Summary Table
+
+| Task | Description                                | Sub-Phase | New Files | Modified Files | Depends On |
+| ---- | ------------------------------------------ | --------- | --------- | -------------- | ---------- |
+| 8.1  | Inject trace_id into Logs::format()        | 8a        | 0         | 1              | Phase 7    |
+| 8.2  | Add Loki to Docker Compose stack           | 8b        | 1         | 1              | --         |
+| 8.3  | Add filelog receiver to OTel Collector     | 8b        | 0         | 2              | 8.1, 8.2   |
+| 8.4  | Configure Grafana trace-to-log correlation | 8b        | 0         | 2              | 8.3        |
+| 8.5  | Update integration tests                   | 8a + 8b   | 0         | 1              | 8.4        |
+| 8.6  | Update documentation                       | 8a + 8b   | 0         | 3              | 8.5        |
+
+**Parallel work**: Task 8.2 (Loki infra) can run in parallel with Task 8.1 (code change). Tasks 8.3-8.6 are sequential.
+
+**Exit Criteria** (from [06-implementation-phases.md §6.8.1](./06-implementation-phases.md)):
+
+- [ ] Log lines within active spans contain `trace_id=<hex> span_id=<hex>`
+- [ ] Log lines outside spans have no trace context (no empty fields)
+- [ ] Loki ingests xrpld logs via OTel Collector filelog receiver
+- [ ] Grafana Tempo -> Loki one-click correlation works
+- [ ] Grafana Loki -> Tempo reverse lookup works via derived field
+- [ ] Integration test verifies trace_id presence in logs
+- [ ] No performance regression from trace_id injection (< 0.1% overhead)

OpenTelemetryPlan/Phase9_taskList.md

@@ -0,0 +1,447 @@
+# Phase 9: Internal Metric Instrumentation Gap Fill — Task List
+
+> **Status**: Future Enhancement
+>
+> **Goal**: Instrument xrpld to emit ~50+ metrics that exist in `get_counts`/`server_info`/TxQ/PerfLog but currently lack time-series export via the OTel or beast::insight pipelines.
+>
+> **Scope**: Hybrid approach — extend `beast::insight` for metrics near existing registrations, use OTel Metrics SDK `ObservableGauge` callbacks for new categories (TxQ, PerfLog, CountedObjects).
+>
+> **Branch**: `pratik/otel-phase9-metric-gap-fill` (from `pratik/otel-phase8-log-correlation`)
+>
+> **Depends on**: Phase 7 (native OTel metrics pipeline) and Phase 8 (log-trace correlation)
+
+### Related Plan Documents
+
+| Document                                                             | Relevance                                                      |
+| -------------------------------------------------------------------- | -------------------------------------------------------------- |
+| [06-implementation-phases.md](./06-implementation-phases.md)         | Phase 9 plan: motivation, architecture, exit criteria (§6.8.2) |
+| [09-data-collection-reference.md](./09-data-collection-reference.md) | Current metric inventory + future metrics section              |
+| [Phase7_taskList.md](./Phase7_taskList.md)                           | Prerequisite — OTel Metrics SDK and `OTelCollector` class      |
+| [Phase8_taskList.md](./Phase8_taskList.md)                           | Prerequisite — log-trace correlation                           |
+
+### Third-Party Consumer Context
+
+These metrics serve multiple external consumer categories identified during research:
+
+| Consumer Category         | Key Metrics They Need                                           |
+| ------------------------- | --------------------------------------------------------------- |
+| **Exchanges**             | Fee escalation levels, TxQ depth, settlement latency            |
+| **Payment Processors**    | Load factors, io_latency, transaction throughput                |
+| **Analytics Providers**   | NodeStore I/O, cache hit rates, counted objects                 |
+| **Validators/Operators**  | Per-job execution times, PerfLog RPC counters, consensus timing |
+| **Academic Researchers**  | Consensus performance time-series, fee market dynamics          |
+| **Institutional Custody** | Server health scores, reserve calculations, node availability   |
+
+---
+
+## Task 9.1: NodeStore I/O Metrics
+
+**Objective**: Export node store read/write performance as time-series metrics.
+
+**What to do**:
+
+- In `src/libxrpl/nodestore/Database.cpp`, extend existing `beast::insight` registrations to add:
+  - Gauge: `node_reads_total` (cumulative read operations)
+  - Gauge: `node_reads_hit` (cache-served reads)
+  - Gauge: `node_writes` (cumulative write operations)
+  - Gauge: `node_written_bytes` (cumulative bytes written)
+  - Gauge: `node_read_bytes` (cumulative bytes read)
+  - Gauge: `node_reads_duration_us` (cumulative read time in microseconds)
+  - Gauge: `write_load` (current write load score)
+  - Gauge: `read_queue` (items in read queue)
+
+- These values are already computed in `Database::getCountsJson()` (line ~236). Wire the same counters to `beast::insight` hooks.
+
+**Key modified files**:
+
+- `src/libxrpl/nodestore/Database.cpp`
+- `src/libxrpl/nodestore/Database.h` (add insight members)
+
+**Derived Prometheus metrics**: `xrpld_nodestore_reads_total`, `xrpld_nodestore_reads_hit`, `xrpld_nodestore_write_load`, etc.
+
+**Grafana dashboard**: Add "NodeStore I/O" panel group to _Node Health_ dashboard.
+
+---
+
+## Task 9.2: Cache Hit Rate Metrics
+
+**Objective**: Export SHAMap and ledger cache performance as time-series gauges.
+
+**What to do**:
+
+- Register OTel `ObservableGauge` callbacks (via Phase 7's `OTelCollector`) for:
+  - `SLE_hit_rate` — SLE cache hit rate (0.0–1.0)
+  - `ledger_hit_rate` — Ledger object cache hit rate
+  - `AL_hit_rate` — AcceptedLedger cache hit rate
+  - `treenode_cache_size` — SHAMap TreeNode cache size (entries)
+  - `treenode_track_size` — Tracked tree nodes
+  - `fullbelow_size` — FullBelow cache size
+
+- The callback should read from the same sources as `GetCounts.cpp` handler (line ~43).
+
+- Create a centralized `MetricsRegistry` class that holds all OTel async gauge registrations, polled at 10-second intervals by the `PeriodicMetricReader`.
+
+**Key modified files**:
+
+- New: `src/xrpld/telemetry/MetricsRegistry.h` / `.cpp`
+- `src/xrpld/rpc/handlers/GetCounts.cpp` (extract shared access methods)
+- `src/xrpld/app/main/Application.cpp` (register MetricsRegistry at startup)
+
+**Derived Prometheus metrics**: `xrpld_cache_SLE_hit_rate`, `xrpld_cache_ledger_hit_rate`, `xrpld_cache_treenode_size`, etc.
+
+---
+
+## Task 9.3: Transaction Queue (TxQ) Metrics
+
+**Objective**: Export TxQ depth, capacity, and fee escalation levels as time-series.
+
+**What to do**:
+
+- Register OTel `ObservableGauge` callbacks for TxQ state (from `TxQ.h` line ~143):
+  - `txq_count` — Current transactions in queue
+  - `txq_max_size` — Maximum queue capacity
+  - `txq_in_ledger` — Transactions in current open ledger
+  - `txq_per_ledger` — Expected transactions per ledger
+  - `txq_reference_fee_level` — Reference fee level
+  - `txq_min_processing_fee_level` — Minimum fee to get processed
+  - `txq_med_fee_level` — Median fee level in queue
+  - `txq_open_ledger_fee_level` — Open ledger fee escalation level
+
+- Add to the `MetricsRegistry` (Task 9.2).
+
+**Key modified files**:
+
+- `src/xrpld/telemetry/MetricsRegistry.cpp` (add TxQ callbacks)
+- `src/xrpld/app/tx/detail/TxQ.h` (expose metrics accessor if needed)
+
+**Derived Prometheus metrics**: `xrpld_txq_count`, `xrpld_txq_max_size`, `xrpld_txq_open_ledger_fee_level`, etc.
+
+**Grafana dashboard**: New _Fee Market & TxQ_ dashboard (`xrpld-fee-market`).
+
+---
+
+## Task 9.4: PerfLog Per-RPC Method Metrics
+
+**Objective**: Export per-RPC-method call counts and latency as OTel metrics.
+
+**What to do**:
+
+- Register OTel instruments for PerfLog RPC counters (from `PerfLogImp.cpp` line ~63):
+  - Counter: `xrpld_rpc_method_started_total{method="<name>"}` — calls started
+  - Counter: `xrpld_rpc_method_finished_total{method="<name>"}` — calls completed
+  - Counter: `xrpld_rpc_method_errored_total{method="<name>"}` — calls errored
+  - Histogram: `xrpld_rpc_method_duration_us{method="<name>"}` — execution time distribution
+
+- Use OTel `Counter<int64_t>` and `Histogram<double>` instruments with `method` attribute label.
+
+- Hook into the existing PerfLog callback mechanism rather than adding new instrumentation points.
+
+**Key modified files**:
+
+- `src/xrpld/perflog/detail/PerfLogImp.cpp` (add OTel instrument updates alongside existing JSON counters)
+- `src/xrpld/telemetry/MetricsRegistry.cpp` (register instruments)
+
+**Derived Prometheus metrics**: `xrpld_rpc_method_started_total{method="server_info"}`, `xrpld_rpc_method_duration_us_bucket{method="ledger"}`, etc.
+
+**Grafana dashboard**: Add "Per-Method RPC Breakdown" panel group to _RPC Performance_ dashboard.
+
+---
+
+## Task 9.5: PerfLog Per-Job-Type Metrics
+
+**Objective**: Export per-job-type queue and execution metrics.
+
+**What to do**:
+
+- Register OTel instruments for PerfLog job counters:
+  - Counter: `xrpld_job_queued_total{job_type="<name>"}` — jobs queued
+  - Counter: `xrpld_job_started_total{job_type="<name>"}` — jobs started
+  - Counter: `xrpld_job_finished_total{job_type="<name>"}` — jobs completed
+  - Histogram: `xrpld_job_queued_duration_us{job_type="<name>"}` — time spent waiting in queue
+  - Histogram: `xrpld_job_running_duration_us{job_type="<name>"}` — execution time distribution
+
+- Hook into PerfLog's existing job tracking alongside Task 9.4.
+
+**Key modified files**:
+
+- `src/xrpld/perflog/detail/PerfLogImp.cpp`
+- `src/xrpld/telemetry/MetricsRegistry.cpp`
+
+**Derived Prometheus metrics**: `xrpld_job_queued_total{job_type="ledgerData"}`, `xrpld_job_running_duration_us_bucket{job_type="transaction"}`, etc.
+
+**Grafana dashboard**: New _Job Queue Analysis_ dashboard (`xrpld-job-queue`).
+
+---
+
+## Task 9.6: Counted Object Instance Metrics
+
+**Objective**: Export live instance counts for key internal object types.
+
+**What to do**:
+
+- Register OTel `ObservableGauge` callbacks for `CountedObject<T>` instance counts:
+  - `xrpld_object_count{type="Transaction"}` — live Transaction objects
+  - `xrpld_object_count{type="Ledger"}` — live Ledger objects
+  - `xrpld_object_count{type="NodeObject"}` — live NodeObject instances
+  - `xrpld_object_count{type="STTx"}` — serialized transaction objects
+  - `xrpld_object_count{type="STLedgerEntry"}` — serialized ledger entries
+  - `xrpld_object_count{type="InboundLedger"}` — ledgers being fetched
+  - `xrpld_object_count{type="Pathfinder"}` — active pathfinding computations
+  - `xrpld_object_count{type="PathRequest"}` — active path requests
+  - `xrpld_object_count{type="HashRouterEntry"}` — hash router entries
+
+- The `CountedObject` template already tracks these via atomic counters. The callback just reads the current counts.
+
+**Key modified files**:
+
+- `src/xrpld/telemetry/MetricsRegistry.cpp` (add counted object callbacks)
+- `include/xrpl/basics/CountedObject.h` (may need static accessor for iteration)
+
+**Derived Prometheus metrics**: `xrpld_object_count{type="Transaction"}`, `xrpld_object_count{type="NodeObject"}`, etc.
+
+**Grafana dashboard**: Add "Object Instance Counts" panel to _Node Health_ dashboard.
+
+---
+
+## Task 9.7: Fee Escalation & Load Factor Metrics
+
+**Objective**: Export the full load factor breakdown as time-series.
+
+**What to do**:
+
+- Register OTel `ObservableGauge` callbacks for load factors (from `NetworkOPs.cpp` line ~2694):
+  - `load_factor` — combined transaction cost multiplier
+  - `load_factor_server` — server + cluster + network contribution
+  - `load_factor_local` — local server load only
+  - `load_factor_net` — network-wide load estimate
+  - `load_factor_cluster` — cluster peer load
+  - `load_factor_fee_escalation` — open ledger fee escalation
+  - `load_factor_fee_queue` — queue entry fee level
+
+- These overlap with some existing StatsD metrics but provide finer granularity (individual factor breakdown vs. combined value).
+
+**Key modified files**:
+
+- `src/xrpld/telemetry/MetricsRegistry.cpp`
+- `src/xrpld/app/misc/NetworkOPs.cpp` (expose load factor accessors if needed)
+
+**Derived Prometheus metrics**: `xrpld_load_factor`, `xrpld_load_factor_fee_escalation`, etc.
+
+**Grafana dashboard**: Add "Load Factor Breakdown" panel to _Fee Market & TxQ_ dashboard.
+
+---
+
+## Task 9.7a: push_metrics.py Parity — Missing Observable Gauges
+
+**Objective**: Fill the remaining metric gaps between the external `push_metrics.py` script (in `ripplex-ansible`) and the internal OTel `MetricsRegistry` observable gauges. After this task, all metrics collected by `push_metrics.py` that CAN be collected internally are covered.
+
+**What was done**:
+
+- Extended existing `cacheHitRateGauge_` callback with `AL_size` (AcceptedLedger cache size)
+- Extended existing `nodeStoreGauge_` callback with 4 new metrics from `getCountsJson()`:
+  - `node_reads_duration_us` (JSON string — uses `std::stoll(asString())`)
+  - `read_request_bundle` (native JSON int)
+  - `read_threads_running` (native JSON int)
+  - `read_threads_total` (native JSON int)
+- Added new `xrpld_server_info` Int64ObservableGauge with 8 metrics:
+  - `server_state` — operating mode as int (0=DISCONNECTED .. 4=FULL)
+  - `uptime` — seconds since server start
+  - `peers` — total peer count
+  - `validated_ledger_seq` — validated ledger sequence (atomic read)
+  - `ledger_current_index` — current open ledger sequence
+  - `peer_disconnects_resources` — cumulative resource-related disconnects
+  - `last_close_proposers` — from `getConsensusInfo()["previous_proposers"]`
+  - `last_close_converge_time_ms` — from `getConsensusInfo()["previous_mseconds"]`
+- Added new `xrpld_build_info` Int64ObservableGauge (info-style, value=1 with `version` label)
+- Added new `xrpld_complete_ledgers` Int64ObservableGauge parsing comma-separated ranges into `{bound, index}` pairs
+- Added new `xrpld_db_metrics` Int64ObservableGauge with 4 metrics:
+  - `db_kb_total`, `db_kb_ledger`, `db_kb_transaction` (SQLite stat queries)
+  - `historical_perminute` (historical ledger fetch rate)
+
+**Key modified files**:
+
+- `src/xrpld/telemetry/MetricsRegistry.h` (4 new gauge members, updated ASCII diagram)
+- `src/xrpld/telemetry/MetricsRegistry.cpp` (4 new callback registrations, 2 callback extensions)
+
+**Not implementable inside xrpld**:
+
+- `connection_count_51233/51234` — OS-level port connection counts from external shell script (`get_connection.sh`)
+
+**Derived Prometheus metrics**: `xrpld_server_info{metric="server_state"}`, `xrpld_build_info{version="2.4.0"}`, `xrpld_complete_ledgers{bound="start",index="0"}`, `xrpld_db_metrics{metric="db_kb_total"}`, etc.
+
+**Grafana dashboard**: New panels added to _Node Health_ dashboard (`system-node-health.json`).
+
+---
+
+## Task 9.8: New Grafana Dashboards
+
+**Objective**: Create Grafana dashboards for the new metric categories.
+
+**What to do**:
+
+- Create 2 new dashboards:
+  1. **Fee Market & TxQ** (`xrpld-fee-market`) — TxQ depth/capacity, fee levels, load factor breakdown, fee escalation timeline
+  2. **Job Queue Analysis** (`xrpld-job-queue`) — Per-job-type rates, queue wait times, execution times, job queue depth
+
+- Update 2 existing dashboards:
+  1. **Node Health** (`xrpld-statsd-node-health`) — Add NodeStore I/O panels, cache hit rate panels, object instance counts
+  2. **RPC Performance** (`xrpld-rpc-perf`) — Add per-method RPC breakdown panels
+
+**Key modified files**:
+
+- New: `docker/telemetry/grafana/dashboards/rippled-fee-market.json`
+- New: `docker/telemetry/grafana/dashboards/rippled-job-queue.json`
+- `docker/telemetry/grafana/dashboards/rippled-statsd-node-health.json`
+- `docker/telemetry/grafana/dashboards/rippled-rpc-perf.json`
+
+---
+
+## Task 9.9: Update Documentation
+
+**Objective**: Update telemetry reference docs with all new metrics.
+
+**What to do**:
+
+- Update `OpenTelemetryPlan/09-data-collection-reference.md`:
+  - Add new section for OTel SDK-exported metrics (NodeStore, cache, TxQ, PerfLog, CountedObjects, load factors)
+  - Update Grafana dashboard reference table (add 2 new dashboards)
+  - Add Prometheus query examples for new metrics
+
+- Update `docs/telemetry-runbook.md`:
+  - Add alerting rules for new metrics (NodeStore write_load, TxQ capacity, cache hit rate degradation)
+  - Add troubleshooting entries for new metric categories
+
+**Key modified files**:
+
+- `OpenTelemetryPlan/09-data-collection-reference.md`
+- `docs/telemetry-runbook.md`
+
+---
+
+## Task 9.10: Integration Tests
+
+**Objective**: Verify all new metrics appear in Prometheus after a test workload.
+
+**What to do**:
+
+- Extend the existing telemetry integration test:
+  - Start xrpld with `[telemetry] enabled=1` and `[insight] server=otel`
+  - Submit a batch of RPC calls and transactions
+  - Query Prometheus for each new metric family
+  - Assert non-zero values for: NodeStore reads, cache hit rates, TxQ count, PerfLog RPC counters, object counts, load factors
+
+- Add unit tests for the `MetricsRegistry` class:
+  - Verify callback registration and deregistration
+  - Verify metric values match `get_counts` JSON output
+  - Verify graceful behavior when telemetry is disabled
+
+**Key modified files**:
+
+- `src/test/telemetry/MetricsRegistry_test.cpp` (new)
+- Existing integration test script (extend assertions)
+
+---
+
+## Task 9.11: Validator Health Dashboard (External Dashboard Parity)
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md) — dashboards for Phase 7 metrics inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
+>
+> **Upstream**: Phase 7 Tasks 7.9-7.16 (metrics must be emitting).
+> **Downstream**: Phase 10 (dashboard load checks), Phase 11 (alert rules reference these panels).
+
+**Objective**: Create a Grafana dashboard for validation agreement, amendment/UNL health, and state tracking.
+
+**Dashboard**: `xrpld-validator-health.json`
+
+| Panel                      | Type       | PromQL                                                         |
+| -------------------------- | ---------- | -------------------------------------------------------------- |
+| Agreement % (1h)           | stat       | `xrpld_validation_agreement{metric="agreement_pct_1h"}`        |
+| Agreement % (24h)          | stat       | `xrpld_validation_agreement{metric="agreement_pct_24h"}`       |
+| Agreements vs Missed (1h)  | bargauge   | `agreements_1h` and `missed_1h` side by side                   |
+| Agreements vs Missed (24h) | bargauge   | `agreements_24h` and `missed_24h` side by side                 |
+| Validation Rate            | stat       | `rate(xrpld_validations_sent_total[5m]) * 60`                  |
+| Validations Checked Rate   | stat       | `rate(xrpld_validations_checked_total[5m]) * 60`               |
+| Amendment Blocked          | stat       | `xrpld_validator_health{metric="amendment_blocked"}`           |
+| UNL Expiry (days)          | stat       | `xrpld_validator_health{metric="unl_expiry_days"}`             |
+| Validation Quorum          | stat       | `xrpld_validator_health{metric="validation_quorum"}`           |
+| State Value Timeline       | timeseries | `xrpld_state_tracking{metric="state_value"}`                   |
+| Time in Current State      | stat       | `xrpld_state_tracking{metric="time_in_current_state_seconds"}` |
+| State Changes Rate         | stat       | `rate(xrpld_state_changes_total[1h])`                          |
+| Ledgers Closed Rate        | stat       | `rate(xrpld_ledgers_closed_total[5m]) * 60`                    |
+
+**Dashboard conventions**: `$node` template variable for `exported_instance` filtering, dark theme, matching existing panel sizes and color schemes.
+
+**Key new files**: `docker/telemetry/grafana/dashboards/rippled-validator-health.json`
+
+**Exit Criteria**:
+
+- [ ] All 13 panels render with non-zero data during normal operation
+- [ ] `$node` filter works correctly for multi-node deployments
+- [ ] Amendment blocked and UNL expiry panels use color thresholds (red=blocked/expiring)
+
+---
+
+## Task 9.12: Peer Quality Dashboard (External Dashboard Parity)
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md)
+
+**Objective**: Create a Grafana dashboard for peer health aggregates.
+
+**Dashboard**: `xrpld-peer-quality.json`
+
+| Panel                  | Type       | PromQL                                                         |
+| ---------------------- | ---------- | -------------------------------------------------------------- |
+| P90 Peer Latency       | timeseries | `xrpld_peer_quality{metric="peer_latency_p90_ms"}`             |
+| Insane/Diverged Peers  | stat       | `xrpld_peer_quality{metric="peers_insane_count"}`              |
+| Higher Version Peers % | stat       | `xrpld_peer_quality{metric="peers_higher_version_pct"}`        |
+| Upgrade Recommended    | stat       | `xrpld_peer_quality{metric="upgrade_recommended"}`             |
+| Resource Disconnects   | timeseries | `xrpld_Overlay_Peer_Disconnects_Charges`                       |
+| Inbound vs Outbound    | bargauge   | `xrpld_Peer_Finder_Active_Inbound_Peers`, `..._Outbound_Peers` |
+
+**Key new files**: `docker/telemetry/grafana/dashboards/rippled-peer-quality.json`
+
+**Exit Criteria**:
+
+- [ ] All 6 panels render correctly
+- [ ] P90 latency panel shows trend over time
+- [ ] Upgrade recommended panel uses color threshold (red=1, green=0)
+
+---
+
+## Task 9.13: Ledger Economy Dashboard Panels (External Dashboard Parity)
+
+> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md)
+
+**Objective**: Add "Ledger Economy" row to the existing `system-node-health.json` dashboard.
+
+| Panel                | Type       | PromQL                                              |
+| -------------------- | ---------- | --------------------------------------------------- |
+| Base Fee (drops)     | stat       | `xrpld_ledger_economy{metric="base_fee_xrp"}`       |
+| Reserve Base (drops) | stat       | `xrpld_ledger_economy{metric="reserve_base_xrp"}`   |
+| Reserve Inc (drops)  | stat       | `xrpld_ledger_economy{metric="reserve_inc_xrp"}`    |
+| Ledger Age           | stat       | `xrpld_ledger_economy{metric="ledger_age_seconds"}` |
+| Transaction Rate     | timeseries | `xrpld_ledger_economy{metric="transaction_rate"}`   |
+
+**Key modified files**: `docker/telemetry/grafana/dashboards/system-node-health.json`
+
+**Exit Criteria**:
+
+- [ ] 5 new panels render correctly in existing dashboard
+- [ ] Fee values match `server_info` RPC output
+- [ ] Transaction rate shows smooth trend (not spiky)
+
+---
+
+## Exit Criteria
+
+- [ ] All ~50 new metrics visible in Prometheus via OTLP pipeline
+- [ ] `MetricsRegistry` class registers/deregisters cleanly with OTel SDK
+- [ ] Async gauge callbacks execute at 10s intervals without performance impact
+- [ ] 2 new Grafana dashboards operational (Fee Market, Job Queue)
+- [ ] 2 existing dashboards updated with new panel groups
+- [ ] Integration test validates all new metric families are non-zero
+- [ ] No performance regression (< 0.5% CPU overhead from new callbacks)
+- [ ] Documentation updated with full new metric inventory
+- [ ] Validator Health dashboard renders all 13 panels
+- [ ] Peer Quality dashboard renders all 6 panels
+- [ ] Ledger Economy panels added to system-node-health dashboard

OpenTelemetryPlan/presentation.md

@@ -1,21 +1,23 @@
-# OpenTelemetry Distributed Tracing for xrpld
+# OpenTelemetry Observability for xrpld
+
+> Status: Phases 1-8 shipped. Traces, metrics, logs all live via OTel.
 
 ---
 
 ## Slide 1: Introduction
 
-> **CNCF** = Cloud Native Computing Foundation
+> **CNCF** = Cloud Native Computing Foundation | **OTel** = OpenTelemetry
 
 ### What is OpenTelemetry?
 
-OpenTelemetry is an open-source, CNCF-backed observability framework for distributed tracing, metrics, and logs.
+CNCF-backed, vendor-neutral framework for **traces, metrics, and logs** with a single SDK and wire protocol (OTLP).
 
-### Why OpenTelemetry for xrpld?
+### Why OTel for xrpld?
 
-- **End-to-End Transaction Visibility**: Track transactions from submission → consensus → ledger inclusion
-- **Cross-Node Correlation**: Follow requests across multiple independent nodes using a unique `trace_id`
-- **Consensus Round Analysis**: Understand timing and behavior across validators
-- **Incident Debugging**: Correlate events across distributed nodes during issues
+- **End-to-end TX visibility** — submission → consensus → ledger inclusion
+- **Cross-node correlation** — shared `trace_id` stitches hops without a central coordinator
+- **Consensus round analysis** — phase timing across validators
+- **Incident debugging** — correlated traces, metrics, logs for one query
 
 ```mermaid
 flowchart LR
@@ -27,221 +29,130 @@ flowchart LR
     style D fill:#e65100,stroke:#bf360c,color:#fff
 ```
 
-**Reading the diagram:**
-
-- **Node A (blue, leftmost)**: The originating node that first receives the transaction and assigns a new `trace_id: abc123`; this ID becomes the correlation key for the entire distributed trace.
-- **Node B and Node C (green, middle)**: Relay and validation nodes — each creates its own span but carries the same `trace_id`, so their work is linked to the original submission without any central coordinator.
-- **Node D (orange, rightmost)**: The final node that applies the transaction to the ledger; the trace now spans the full lifecycle from submission to ledger inclusion.
-- **Left-to-right flow**: The horizontal progression shows the real-world message path — a transaction hops from node to node, and the shared `trace_id` stitches all hops into a single queryable trace.
-
-> **Trace ID: abc123** — All nodes share the same trace, enabling cross-node correlation.
+> One trace, four nodes, full lifecycle.
 
 ---
 
-## Slide 2: OpenTelemetry vs Open Source Alternatives
+## Slide 2: Old Stack vs New OTel Stack
 
-> **CNCF** = Cloud Native Computing Foundation
+### Side-by-Side
 
-| Feature             | OpenTelemetry    | Jaeger           | Zipkin             | SkyWalking | Pinpoint   | Prometheus |
-| ------------------- | ---------------- | ---------------- | ------------------ | ---------- | ---------- | ---------- |
-| **Tracing**         | YES              | YES              | YES                | YES        | YES        | NO         |
-| **Metrics**         | YES              | NO               | NO                 | YES        | YES        | YES        |
-| **Logs**            | YES              | NO               | NO                 | YES        | NO         | NO         |
-| **C++ SDK**         | YES Official     | YES (Deprecated) | YES (Unmaintained) | NO         | NO         | YES        |
-| **Vendor Neutral**  | YES Primary goal | NO               | NO                 | NO         | NO         | NO         |
-| **Instrumentation** | Manual + Auto    | Manual           | Manual             | Auto-first | Auto-first | Manual     |
-| **Backend**         | Any (exporters)  | Self             | Self               | Self       | Self       | Self       |
-| **CNCF Status**     | Incubating       | Graduated        | NO                 | Incubating | NO         | Graduated  |
+| Aspect                    | Before (StatsD + Debug Logs)                                                      | After (OTel: Traces + Metrics + Logs)                        |
+| ------------------------- | --------------------------------------------------------------------------------- | ------------------------------------------------------------ |
+| **Metrics**               | Beast Insight → StatsD UDP → Graphite                                             | `MetricsRegistry` → OTLP/HTTP → Prometheus                   |
+| **Metric inventory**      | **~250 metric series** at runtime (28 registrations × overlay traffic categories) | **23 native instruments** × dimensions + RED via spanmetrics |
+| **Logs**                  | `beast::Journal` → `debug.log` (grep / tail)                                      | Journal → filelog tail → Loki (structured, queryable)        |
+| **Traces**                | None                                                                              | Telemetry SDK → OTLP → Tempo (cross-node)                    |
+| **Correlation**           | Timestamp + grep across files                                                     | Shared `trace_id` across all 3 signals                       |
+| **Format**                | Counter/gauge names; free-form log lines                                          | OTLP protobuf; structured records                            |
+| **Backend choice**        | Locked to StatsD daemon + log files                                               | Vendor-neutral via Collector exporters                       |
+| **Cross-node view**       | ❌ Not possible                                                                   | ✅ Native via trace context propagation                      |
+| **Histogram p50/p95/p99** | ❌ Counters/gauges only                                                           | ✅ Native histograms + spanmetrics                           |
 
-> **Why OpenTelemetry?** It's the only actively maintained, full-featured C++ option with vendor neutrality — allowing export to Tempo, Prometheus, Grafana, or any commercial backend without changing instrumentation.
+### Legacy StatsD Metric Series (~250 total)
 
----
-
-## Slide 3: Adoption Scope — Traces Only (Current Plan)
-
-OpenTelemetry supports three signal types: **Traces**, **Metrics**, and **Logs**. xrpld already captures metrics (StatsD via Beast Insight) and logs (Journal/PerfLog). The question is: how much of OTel do we adopt?
-
-> **Scenario A**: Add distributed tracing. Keep StatsD for metrics and Journal for logs.
-
-```mermaid
-flowchart LR
-    subgraph xrpld["xrpld Process"]
-        direction TB
-        OTel["OTel SDK<br/>(Traces)"]
-        Insight["Beast Insight<br/>(StatsD Metrics)"]
-        Journal["Journal + PerfLog<br/>(Logging)"]
-    end
-
-    OTel -->|"OTLP"| Collector["OTel Collector"]
-    Insight -->|"UDP"| StatsD["StatsD Server"]
-    Journal -->|"File I/O"| LogFile["perf.log / debug.log"]
-
-    Collector --> Tempo["Tempo"]
-    StatsD --> Graphite["Graphite / Grafana"]
-    LogFile --> Loki["Loki (optional)"]
-
-    style xrpld fill:#424242,stroke:#212121,color:#fff
-    style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff
-    style Insight fill:#1565c0,stroke:#0d47a1,color:#fff
-    style Journal fill:#e65100,stroke:#bf360c,color:#fff
-    style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff
-```
-
-| Aspect                         | Details                                                                                                         |
-| ------------------------------ | --------------------------------------------------------------------------------------------------------------- |
-| **What changes for operators** | Deploy OTel Collector + trace backend. Existing StatsD and log pipelines stay as-is.                            |
-| **Codebase impact**            | New `Telemetry` module (~1500 LOC). Beast Insight and Journal untouched.                                        |
-| **New capabilities**           | Cross-node trace correlation, span-based debugging, request lifecycle visibility.                               |
-| **What we still can't do**     | Correlate metrics with specific traces natively. StatsD metrics remain fire-and-forget with no trace exemplars. |
-| **Maintenance burden**         | Three separate observability systems to maintain (OTel + StatsD + Journal).                                     |
-| **Risk**                       | Lowest — additive change, no existing systems disturbed.                                                        |
-
----
-
-## Slide 4: Future Adoption — Metrics & Logs via OTel
-
-### Scenario B: + OTel Metrics (Replace StatsD)
-
-> Migrate StatsD to OTel Metrics API, exposing Prometheus-compatible metrics. Remove Beast Insight.
-
-```mermaid
-flowchart LR
-    subgraph xrpld["xrpld Process"]
-        direction TB
-        OTel["OTel SDK<br/>(Traces + Metrics)"]
-        Journal["Journal + PerfLog<br/>(Logging)"]
-    end
-
-    OTel -->|"OTLP"| Collector["OTel Collector"]
-    Journal -->|"File I/O"| LogFile["perf.log / debug.log"]
-
-    Collector --> Tempo["Tempo<br/>(Traces)"]
-    Collector --> Prom["Prometheus<br/>(Metrics)"]
-    LogFile --> Loki["Loki (optional)"]
-
-    style xrpld fill:#424242,stroke:#212121,color:#fff
-    style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff
-    style Journal fill:#e65100,stroke:#bf360c,color:#fff
-    style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff
-```
-
-- **Better metrics?** Yes — Prometheus gives native histograms (p50/p95/p99), multi-dimensional labels, and exemplars linking metric spikes to traces.
-- **Codebase**: Remove `Beast::Insight` + `StatsDCollector` (~2000 LOC). Single SDK for traces and metrics.
-- **Operator effort**: Rewrite dashboards from StatsD/Graphite queries to PromQL. Run both in parallel during transition.
-- **Risk**: Medium — operators must migrate monitoring infrastructure.
-
-### Scenario C: + OTel Logs (Full Stack)
-
-> Also replace Journal logging with OTel Logs API. Single SDK for everything.
-
-```mermaid
-flowchart LR
-    subgraph xrpld["xrpld Process"]
-        OTel["OTel SDK<br/>(Traces + Metrics + Logs)"]
-    end
-
-    OTel -->|"OTLP"| Collector["OTel Collector"]
-
-    Collector --> Tempo["Tempo<br/>(Traces)"]
-    Collector --> Prom["Prometheus<br/>(Metrics)"]
-    Collector --> Loki["Loki / Elastic<br/>(Logs)"]
-
-    style xrpld fill:#424242,stroke:#212121,color:#fff
-    style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff
-    style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff
-```
-
-- **Structured logging**: OTel Logs API outputs structured records with `trace_id`, `span_id`, severity, and attributes by design.
-- **Full correlation**: Every log line carries `trace_id`. Click trace → see logs. Click metric spike → see trace → see logs.
-- **Codebase**: Remove Beast Insight (~2000 LOC) + simplify Journal/PerfLog (~3000 LOC). One dependency instead of three.
-- **Risk**: Highest — `beast::Journal` is deeply embedded in every component. Large refactor. OTel C++ Logs API is newer (stable since v1.11, less battle-tested).
-
-### Recommendation
-
-```mermaid
-flowchart LR
-    A["Phase 1<br/><b>Traces Only</b><br/>(Current Plan)"] --> B["Phase 2<br/><b>+ Metrics</b><br/>(Replace StatsD)"] --> C["Phase 3<br/><b>+ Logs</b><br/>(Full OTel)"]
-
-    style A fill:#2e7d32,stroke:#1b5e20,color:#fff
-    style B fill:#1565c0,stroke:#0d47a1,color:#fff
-    style C fill:#e65100,stroke:#bf360c,color:#fff
-```
-
-| Phase                | Signal    | Strategy                                                       | Risk   |
-| -------------------- | --------- | -------------------------------------------------------------- | ------ |
-| **Phase 1** (now)    | Traces    | Add OTel traces. Keep StatsD and Journal. Prove value.         | Low    |
-| **Phase 2** (future) | + Metrics | Migrate StatsD → Prometheus via OTel. Remove Beast Insight.    | Medium |
-| **Phase 3** (future) | + Logs    | Adopt OTel Logs API. Align with structured logging initiative. | High   |
-
-> **Key Takeaway**: Start with traces (unique value, lowest risk), then incrementally adopt metrics and logs as the OTel infrastructure proves itself.
-
----
-
-## Slide 5: Comparison with xrpld's Existing Solutions
-
-### Current Observability Stack
-
-| Aspect                | PerfLog (JSON)        | StatsD (Metrics)      | OpenTelemetry (NEW)         |
-| --------------------- | --------------------- | --------------------- | --------------------------- |
-| **Type**              | Logging               | Metrics               | Distributed Tracing         |
-| **Scope**             | Single node           | Single node           | **Cross-node**              |
-| **Data**              | JSON log entries      | Counters, gauges      | Spans with context          |
-| **Correlation**       | By timestamp          | By metric name        | By `trace_id`               |
-| **Overhead**          | Low (file I/O)        | Low (UDP)             | Low-Medium (configurable)   |
-| **Question Answered** | "What happened here?" | "How many? How fast?" | **"What was the journey?"** |
+| Category                    | Series   | Notes                                                                               |
+| --------------------------- | -------- | ----------------------------------------------------------------------------------- |
+| **Overlay traffic gauges**  | ~224     | 56 `TrafficCount::category` enum × 4 gauges (`Bytes_{In,Out}`, `Messages_{In,Out}`) |
+| **Peer Finder**             | 2        | `Active_{In,Out}bound_Peers`                                                        |
+| **State Accounting**        | 10       | `{Disconnected,Connected,Syncing,Tracking,Full}_{duration,transitions}`             |
+| **Ledger**                  | 4        | `Validated/Published_Ledger_Age`, `mismatch`, `ledger_fetches`                      |
+| **RPC / Pathfinding**       | 5        | `requests`, `size`, `time`, `pathfind_{fast,full}`                                  |
+| **JobQueue / IO / Disconn** | 3        | `job_count`, `ios_latency`, `Peer_Disconnects`                                      |
+| **Total**                   | **~248** | 28 `make_*` call sites; series count balloons via overlay-category fan-out          |
 
 ### Use Case Matrix
 
-| Scenario                         | PerfLog | StatsD | OpenTelemetry |
-| -------------------------------- | ------- | ------ | ------------- |
-| "How many TXs per second?"       | ❌      | ✅     | ❌            |
-| "Why was this specific TX slow?" | ⚠️      | ❌     | ✅            |
-| "Which node delayed consensus?"  | ❌      | ❌     | ✅            |
-| "Show TX journey across 5 nodes" | ❌      | ❌     | ✅            |
+| Scenario                           | StatsD | Debug Logs | OTel Traces | OTel Metrics | OTel Logs |
+| ---------------------------------- | ------ | ---------- | ----------- | ------------ | --------- |
+| "TXs per second?"                  | ✅     | ❌         | ❌          | ✅           | ❌        |
+| "Why was this specific TX slow?"   | ❌     | ⚠️         | ✅          | ❌           | ⚠️        |
+| "Which node delayed consensus?"    | ❌     | ❌         | ✅          | ❌           | ❌        |
+| "TX journey across 5 nodes"        | ❌     | ❌         | ✅          | ❌           | ❌        |
+| "Validator error at 14:02"         | ❌     | ✅         | ⚠️          | ❌           | ✅        |
+| "Reproduce rare assertion / crash" | ❌     | ✅         | ❌          | ❌           | ✅        |
+| "p99 RPC latency by method"        | ⚠️     | ❌         | ⚠️          | ✅           | ❌        |
 
-> **Key Insight**: In the **traces-only** approach (Phase 1), OpenTelemetry **complements** existing systems. In future phases, OTel metrics and logs could **replace** StatsD and Journal respectively — see Slides 3-4 for the full adoption roadmap.
+> Old stack: 2 signals, no correlation, single node. New stack: 3 signals, `trace_id` everywhere, cross-node native.
 
 ---
 
-## Slide 6: Architecture
+## Slide 3: OTel vs Open-Source Alternatives
 
-> **OTLP** = OpenTelemetry Protocol | **WS** = WebSocket
+| Feature             | OpenTelemetry   | Jaeger        | Zipkin          | SkyWalking | Pinpoint   | Prometheus |
+| ------------------- | --------------- | ------------- | --------------- | ---------- | ---------- | ---------- |
+| **Tracing**         | ✅              | ✅            | ✅              | ✅         | ✅         | ❌         |
+| **Metrics**         | ✅              | ❌            | ❌              | ✅         | ✅         | ✅         |
+| **Logs**            | ✅              | ❌            | ❌              | ✅         | ❌         | ❌         |
+| **C++ SDK**         | ✅ Official     | ⚠️ Deprecated | ⚠️ Unmaintained | ❌         | ❌         | ✅         |
+| **Vendor neutral**  | ✅ Primary goal | ❌            | ❌              | ❌         | ❌         | ❌         |
+| **Instrumentation** | Manual + Auto   | Manual        | Manual          | Auto-first | Auto-first | Manual     |
+| **Backend**         | Any (exporters) | Self          | Self            | Self       | Self       | Self       |
+| **CNCF Status**     | Incubating      | Graduated     | —               | Incubating | —          | Graduated  |
 
-### High-Level Integration Architecture
+> Only actively maintained, full-signal C++ option. Backend-agnostic — Tempo/Prometheus/Loki/Elastic/commercial all work without code change.
+
+---
+
+## Slide 4: Architecture (Current)
+
+> **OTLP** = OpenTelemetry Protocol over HTTP/gRPC
 
 ```mermaid
 flowchart TB
     subgraph xrpld["xrpld Node"]
-        subgraph services["Core Services"]
-            direction LR
-            RPC["RPC Server<br/>(HTTP/WS)"] ~~~ Overlay["Overlay<br/>(P2P Network)"] ~~~ Consensus["Consensus<br/>(RCLConsensus)"]
-        end
-
-        Telemetry["Telemetry Module<br/>(OpenTelemetry SDK)"]
-
-        services --> Telemetry
+        direction TB
+        Surfaces["RPC · TX · Consensus · Peer · Ledger · Job"]
+        SDK["Telemetry SDK + MetricsRegistry"]
+        Journal["beast::Journal → debug.log<br/>(trace_id/span_id injected)"]
+        Surfaces --> SDK
+        Surfaces --> Journal
     end
 
-    Telemetry -->|OTLP/gRPC| Collector["OTel Collector"]
+    SDK -->|"OTLP/HTTP :4318<br/>traces + metrics"| Collector["OTel Collector"]
+    Journal -->|"filelog tail"| Collector
 
-    Collector --> Tempo["Grafana Tempo"]
-    Collector --> Elastic["Elastic APM"]
+    Collector --> Tempo["Tempo<br/>(traces)"]
+    Collector --> Prom["Prometheus<br/>(metrics)"]
+    Collector --> Loki["Loki<br/>(logs)"]
+
+    Tempo --> Grafana["Grafana<br/>(15 dashboards)"]
+    Prom --> Grafana
+    Loki --> Grafana
 
     style xrpld fill:#424242,stroke:#212121,color:#fff
-    style services fill:#1565c0,stroke:#0d47a1,color:#fff
-    style Telemetry fill:#2e7d32,stroke:#1b5e20,color:#fff
+    style SDK fill:#2e7d32,stroke:#1b5e20,color:#fff
+    style Journal fill:#1565c0,stroke:#0d47a1,color:#fff
     style Collector fill:#e65100,stroke:#bf360c,color:#fff
+    style Grafana fill:#4a148c,stroke:#2e0d57,color:#fff
 ```
 
-**Reading the diagram:**
+| Component              | Role                                                |
+| ---------------------- | --------------------------------------------------- |
+| Telemetry SDK          | Span creation, trace context, OTLP traces export    |
+| MetricsRegistry        | RPC/job/peer/consensus counters, gauges, histograms |
+| beast::Journal filelog | `debug.log` tailed by Collector, parsed → Loki      |
+| OTel Collector         | Receive OTLP + filelog; route to Tempo/Prom/Loki    |
+| Spanmetrics connector  | Derives RED metrics from spans (Prometheus)         |
 
-- **Core Services (blue, top)**: RPC Server, Overlay, and Consensus are the three primary components that generate trace data — they represent the entry points for client requests, peer messages, and consensus rounds respectively.
-- **Telemetry Module (green, middle)**: The OpenTelemetry SDK sits below the core services and receives span data from all three; it acts as a single collection point within the xrpld process.
-- **OTel Collector (orange, center)**: An external process that receives spans over OTLP/gRPC from the Telemetry Module; it decouples xrpld from backend choices and handles batching, sampling, and routing.
-- **Backends (bottom row)**: Tempo and Elastic APM are interchangeable — the Collector fans out to any combination, so operators can switch backends without modifying xrpld code.
-- **Top-to-bottom flow**: Data flows from instrumented code down through the SDK, out over the network to the Collector, and finally into storage/visualization backends.
+---
 
-### Context Propagation
+## Slide 5: Signal Coverage
+
+| Surface            | Traces (Spans)                                                  | Metrics (OTLP)                                 | Logs (Journal Partition)       |
+| ------------------ | --------------------------------------------------------------- | ---------------------------------------------- | ------------------------------ |
+| **RPC**            | `rpc.request` + handler spans                                   | request count, latency p50/p95/p99, error rate | `RPC*`                         |
+| **Transactions**   | `tx.receive`, `tx.validate`, `tx.relay`, `tx.apply`             | TX/sec by result, fee escalation gauges        | `TxQ`, `LedgerMaster`          |
+| **Consensus**      | `consensus.round`, `proposal.send/recv`, `validation.send/recv` | round duration, phase histograms, mode gauge   | `Consensus`, `LedgerConsensus` |
+| **Peer / Overlay** | `peer.send`, `peer.receive` per message type                    | peer count, bytes/sec by msg type, suppression | `Overlay`, `PeerImp`           |
+| **Ledger**         | `ledger.close`, `ledger.apply`                                  | close time, TX count, ledger index gauge       | `LedgerMaster`                 |
+| **Job Queue**      | (sampled per type)                                              | queue depth, queue/run duration histograms     | `JobQueue`                     |
+
+> ~30 distinct span kinds, ~80 metric series, structured logs from 50+ partitions.
+
+---
+
+## Slide 6: Context Propagation
 
 ```mermaid
 sequenceDiagram
@@ -250,423 +161,275 @@ sequenceDiagram
     participant NodeB as Node B
 
     Client->>NodeA: Submit TX (no context)
-    Note over NodeA: Creates trace_id: abc123<br/>span: tx.receive
-    NodeA->>NodeB: Relay TX<br/>(traceparent: abc123)
-    Note over NodeB: Links to trace_id: abc123<br/>span: tx.relay
+    Note over NodeA: Create trace_id: abc123<br/>span: tx.receive
+    NodeA->>NodeB: Relay TX (TraceContext field, ~29B)
+    Note over NodeB: Link trace_id: abc123<br/>span: tx.relay (parent: A)
 ```
 
-- **HTTP/RPC**: W3C Trace Context headers (`traceparent`)
-- **P2P Messages**: Protocol Buffer extension fields
+| Carrier               | Mechanism                                  |
+| --------------------- | ------------------------------------------ |
+| HTTP / WebSocket RPC  | W3C `traceparent` header                   |
+| P2P protobuf          | `TraceContext` extension field per message |
+| Internal job dispatch | Thread-local context + `SpanGuard`         |
+
+| Field         | Size      | Description                           |
+| ------------- | --------- | ------------------------------------- |
+| `trace_id`    | 16 bytes  | Trace correlation key                 |
+| `span_id`     | 8 bytes   | Parent span on receiver               |
+| `trace_flags` | 1 byte    | Sampling decision                     |
+| `trace_state` | 0-4 bytes | Optional vendor data                  |
+| **Total**     | **~29 B** | Per traced P2P message (~1-6% of msg) |
 
 ---
 
-## Slide 7: Implementation Plan
+## Slide 7: Performance Overhead
 
-### 5-Phase Rollout (9 Weeks)
+| Metric            | Overhead   | Driver                                              |
+| ----------------- | ---------- | --------------------------------------------------- |
+| **CPU**           | 1-3%       | ~4 μs/TX span work (~2% at 25 TPS baseline)         |
+| **Memory**        | ~10 MB     | SDK statics + worker stack + 2048-span export queue |
+| **Network**       | 10-50 KB/s | OTLP export + 29 B P2P context per traced msg       |
+| **Latency (p99)** | <2%        | TX path dominates; RPC and consensus negligible     |
 
-> **Note**: Dates shown are relative to project start, not calendar dates.
+### Kill Switches
+
+1. `enabled=0` in `xrpld.cfg` → instant disable, no restart
+2. Build with `XRPL_ENABLE_TELEMETRY=OFF` → zero overhead (no-op stubs)
+3. Reduce `sampling_ratio` → linear export reduction
+
+> Derivations and per-component cost tables: see [03-implementation-strategy.md §3.5.4](./03-implementation-strategy.md#354-performance-data-sources).
+
+---
+
+## Slide 8: Sampling — Head vs Tail
+
+|                          | Head Sampling                     | Tail Sampling                          |
+| ------------------------ | --------------------------------- | -------------------------------------- |
+| **Where**                | Inside xrpld (SDK)                | OTel Collector (external)              |
+| **Decision time**        | Trace start (random coin flip)    | Trace end (after all spans buffered)   |
+| **Knows trace content?** | No                                | Yes — error, latency, span kind        |
+| **xrpld overhead**       | Lowest (drop = no-op)             | Higher (export 100%)                   |
+| **Captures all errors?** | No                                | **Yes** (status_code policy)           |
+| **Captures slow ops?**   | No                                | **Yes** (latency policy)               |
+| **Config**               | `xrpld.cfg`: `sampling_ratio=0.1` | `tail_sampling` processor in collector |
+| **Best for**             | Steady-state high volume          | Anomaly + error retention              |
+
+### Recommended Layered Strategy
+
+```mermaid
+flowchart LR
+    xrpld["xrpld<br/>sampling_ratio=1.0<br/>(export all)"] -->|"100%"| col["Collector<br/>tail_sampling:<br/>errors + slow + 10% random"]
+    col -->|"~15-20% kept"| tempo["Tempo storage"]
+
+    style xrpld fill:#424242,stroke:#212121,color:#fff
+    style col fill:#1565c0,stroke:#0d47a1,color:#fff
+    style tempo fill:#2e7d32,stroke:#1b5e20,color:#fff
+```
+
+> If Collector resource pressure: drop `sampling_ratio` to 0.5 — still enough trace volume for tail decisions.
+
+---
+
+## Slide 9: Data Collection & Privacy
+
+### Collected (operational metadata)
+
+| Category    | Attributes                                                           |
+| ----------- | -------------------------------------------------------------------- |
+| Transaction | `tx.hash`, `tx.type`, `tx.result`, `tx.fee`, `ledger_index`          |
+| Consensus   | `round`, `phase`, `mode`, `proposers`, `duration_ms`                 |
+| RPC         | `command`, `version`, `status`, `duration_ms`                        |
+| Peer        | `peer.id` (public key), `latency_ms`, `message.type`, `message.size` |
+| Ledger      | `ledger.hash`, `ledger.index`, `close_time`, `tx_count`              |
+| Job         | `job.type`, `queue_ms`, `worker`                                     |
+
+### NOT Collected (hard exclusions)
+
+> ❌ Private keys · ❌ Account balances · ❌ Transaction amounts · ❌ Raw payloads · ❌ Personal data · ⚙️ IP addresses (configurable)
+
+### Privacy Mechanisms
+
+| Mechanism              | Description                                               |
+| ---------------------- | --------------------------------------------------------- |
+| Account hashing        | `xrpl.tx.account` hashed at Collector before storage      |
+| Configurable redaction | Sensitive attributes excluded via Collector config        |
+| Sampling               | 10% default reduces exposure                              |
+| Local control          | Operator owns Collector → backend pipeline                |
+| No raw payloads        | Span attributes are metadata only, never message contents |
+
+> Principle: telemetry records **operational metadata** — never financial or personal content.
+
+---
+
+## Slide 10: Implementation Timeline
 
 ```mermaid
 gantt
-    title Implementation Timeline
+    title OpenTelemetry Rollout
     dateFormat  YYYY-MM-DD
     axisFormat  Week %W
 
-    section Phase 1
-    Core Infrastructure    :p1, 2024-01-01, 2w
+    section Done
+    Phase 1 Core Infra        :done, p1, 2024-01-01, 2w
+    Phase 2 RPC Tracing       :done, p2, after p1, 2w
+    Phase 3 TX Tracing        :done, p3, after p2, 2w
+    Phase 4 Consensus         :done, p4, after p3, 2w
+    Phase 5 Docs/Deploy       :done, p5, after p4, 1w
+    Phase 6 StatsD Bridge     :done, p6, after p5, 1w
+    Phase 7 Native OTel Metrics :done, p7, after p6, 2w
+    Phase 8 Log-Trace Correlation :done, p8, after p7, 1w
+    Phase 9 Metric Gap Fill   :active, p9, after p8, 2w
 
-    section Phase 2
-    RPC Tracing           :p2, after p1, 2w
-
-    section Phase 3
-    Transaction Tracing   :p3, after p2, 2w
-
-    section Phase 4
-    Consensus Tracing     :p4, after p3, 2w
-
-    section Phase 5
-    Documentation         :p5, after p4, 1w
+    section Future
+    Phase 10 Workload Validation :p10, after p9, 2w
+    Phase 11 3rd-Party Pipelines :p11, after p10, 3w
 ```
 
-### Phase Details
-
-| Phase | Focus               | Key Deliverables                             | Effort  |
-| ----- | ------------------- | -------------------------------------------- | ------- |
-| 1     | Core Infrastructure | SDK integration, Telemetry interface, Config | 10 days |
-| 2     | RPC Tracing         | HTTP context extraction, Handler spans       | 10 days |
-| 3     | Transaction Tracing | Protobuf context, P2P relay propagation      | 10 days |
-| 4     | Consensus Tracing   | Round spans, Proposal/validation tracing     | 10 days |
-| 5     | Documentation       | Runbook, Dashboards, Training                | 7 days  |
-
-**Total Effort**: ~47 developer-days (2 developers)
-
-> **Future Phases** (not in current scope): After traces are stable, OTel metrics can replace StatsD (~3 weeks), and OTel logs can replace Journal (~4 weeks, aligned with structured logging initiative). See Slides 3-4 for the full adoption roadmap.
+| Phase | Focus                                       | Status  |
+| ----- | ------------------------------------------- | ------- |
+| 1     | SDK integration, Telemetry, Config          | ✅ Done |
+| 2     | RPC handler spans, HTTP context             | ✅ Done |
+| 3     | TX spans, P2P protobuf context              | ✅ Done |
+| 4     | Consensus rounds, proposal/validation       | ✅ Done |
+| 5     | Runbook, dashboards, deployment             | ✅ Done |
+| 6     | StatsD bridge (interim)                     | ✅ Done |
+| 7     | Native OTel metrics (replace Beast Insight) | ✅ Done |
+| 8     | Log-trace correlation (Loki)                | ✅ Done |
+| 9     | Internal metric gap fill                    | ✅ Done |
 
 ---
 
-## Slide 8: Performance Overhead
+## Slide 11: Current State — What Shipped
 
-> **OTLP** = OpenTelemetry Protocol
+### By Signal
 
-### Estimated System Impact
+| Signal      | Backend    | Status | Notes                                                    |
+| ----------- | ---------- | ------ | -------------------------------------------------------- |
+| **Traces**  | Tempo      | ✅     | All 6 surfaces instrumented; cross-node propagation live |
+| **Metrics** | Prometheus | ✅     | Native OTLP; Beast Insight retired                       |
+| **Logs**    | Loki       | ✅     | filelog tailing `debug.log`; `trace_id` injected         |
 
-| Metric            | Overhead   | Notes                                            |
-| ----------------- | ---------- | ------------------------------------------------ |
-| **CPU**           | 1-3%       | Span creation and attribute setting              |
-| **Memory**        | ~10 MB     | SDK statics + batch buffer + worker thread stack |
-| **Network**       | 10-50 KB/s | Compressed OTLP export to collector              |
-| **Latency (p99)** | <2%        | With proper sampling configuration               |
+### By Surface
 
-#### How We Arrived at These Numbers
+| Surface        | Spans Live | Metrics Live | Notes                                               |
+| -------------- | ---------- | ------------ | --------------------------------------------------- |
+| RPC            | ✅         | ✅           | Handler + pathfinding + TxQ                         |
+| Transactions   | ✅         | ✅           | Receive, validate, relay, apply                     |
+| Consensus      | ✅         | ✅           | Round + proposal/validation send+receive (Phase 4a) |
+| Peer / Overlay | ✅         | ✅           | Per-msg-type send/receive                           |
+| Ledger         | ✅         | ✅           | Close + apply                                       |
+| Job Queue      | ✅         | ✅           | Queue depth + duration histograms                   |
 
-**Assumptions (XRPL mainnet baseline)**:
+### Stack Live
 
-| Parameter                 | Value                  | Source                                                                                              |
-| ------------------------- | ---------------------- | --------------------------------------------------------------------------------------------------- |
-| Transaction throughput    | ~25 TPS (peaks to ~50) | Mainnet average                                                                                     |
-| Default peers per node    | 21                     | `peerfinder/detail/Tuning.h` (`defaultMaxPeers`)                                                    |
-| Consensus round frequency | ~1 round / 3-4 seconds | `ConsensusParms.h` (`ledgerMIN_CONSENSUS=1950ms`)                                                   |
-| Proposers per round       | ~20-35                 | Mainnet UNL size                                                                                    |
-| P2P message rate          | ~160 msgs/sec          | See message breakdown below                                                                         |
-| Avg TX processing time    | ~200 μs                | Profiled baseline                                                                                   |
-| Single span creation cost | 500-1000 ns            | OTel C++ SDK benchmarks (see [3.5.4](./03-implementation-strategy.md#354-performance-data-sources)) |
-
-**P2P message breakdown** (per node, mainnet):
-
-| Message Type  | Rate         | Derivation                                                            |
-| ------------- | ------------ | --------------------------------------------------------------------- |
-| TMTransaction | ~100/sec     | ~25 TPS × ~4 relay hops per TX, deduplicated by HashRouter            |
-| TMValidation  | ~50/sec      | ~35 validators × ~1 validation/3s round ≈ ~12/sec, plus relay fan-out |
-| TMProposeSet  | ~10/sec      | ~35 proposers / 3s round ≈ ~12/round, clustered in establish phase    |
-| **Total**     | **~160/sec** | **Only traced message types counted**                                 |
-
-**CPU (1-3%) — Calculation**:
-
-Per-transaction tracing cost breakdown:
-
-| Operation                                       | Cost        | Notes                                      |
-| ----------------------------------------------- | ----------- | ------------------------------------------ |
-| `tx.receive` span (create + end + 4 attributes) | ~1400 ns    | ~1000ns create + ~200ns end + 4×50ns attrs |
-| `tx.validate` span                              | ~1200 ns    | ~1000ns create + ~200ns for 2 attributes   |
-| `tx.relay` span                                 | ~1200 ns    | ~1000ns create + ~200ns for 2 attributes   |
-| Context injection into P2P message              | ~200 ns     | Serialize trace_id + span_id into protobuf |
-| **Total per TX**                                | **~4.0 μs** |                                            |
-
-> **CPU overhead**: 4.0 μs / 200 μs baseline = **~2.0% per transaction**. Under high load with consensus + RPC spans overlapping, reaches ~3%. Consensus itself adds only ~36 μs per 3-second round (~0.001%), so the TX path dominates. On production server hardware (3+ GHz Xeon), span creation drops to ~500-600 ns, bringing per-TX cost to ~2.6 μs (~1.3%). See [Section 3.5.4](./03-implementation-strategy.md#354-performance-data-sources) for benchmark sources.
-
-**Memory (~10 MB) — Calculation**:
-
-| Component                                     | Size               | Notes                                 |
-| --------------------------------------------- | ------------------ | ------------------------------------- |
-| TracerProvider + Exporter (gRPC channel init) | ~320 KB            | Allocated once at startup             |
-| BatchSpanProcessor (circular buffer)          | ~16 KB             | 2049 × 8-byte AtomicUniquePtr entries |
-| BatchSpanProcessor (worker thread stack)      | ~8 MB              | Default Linux thread stack size       |
-| Active spans (in-flight, max ~1000)           | ~500-800 KB        | ~500-800 bytes/span × 1000 concurrent |
-| Export queue (batch buffer, max 2048 spans)   | ~1 MB              | ~500 bytes/span × 2048 queue depth    |
-| Thread-local context storage (~100 threads)   | ~6.4 KB            | ~64 bytes/thread                      |
-| **Total**                                     | **~10 MB ceiling** |                                       |
-
-> Memory plateaus once the export queue fills — the `max_queue_size=2048` config bounds growth.
-> The worker thread stack (~8 MB) dominates the static footprint but is virtual memory; actual RSS
-> depends on stack usage (typically much less). Active spans are larger than originally estimated
-> (~500-800 bytes) because the OTel SDK `Span` object includes a mutex (~40 bytes), `SpanData`
-> recordable (~250 bytes base), and `std::map`-based attribute storage (~200-500 bytes for 3-5
-> string attributes). See [Section 3.5.4](./03-implementation-strategy.md#354-performance-data-sources) for source references.
-
-**Network (10-50 KB/s) — Calculation**:
-
-Two sources of network overhead:
-
-**(A) OTLP span export to Collector:**
-
-| Sampling Rate              | Effective Spans/sec | Avg Span Size (compressed) | Bandwidth    |
-| -------------------------- | ------------------- | -------------------------- | ------------ |
-| 100% (dev only)            | ~500                | ~500 bytes                 | ~250 KB/s    |
-| **10% (recommended prod)** | **~50**             | **~500 bytes**             | **~25 KB/s** |
-| 1% (minimal)               | ~5                  | ~500 bytes                 | ~2.5 KB/s    |
-
-> The ~500 spans/sec at 100% comes from: ~100 TX spans + ~160 P2P context spans + ~23 consensus spans/round + ~50 RPC spans = ~500/sec. OTLP protobuf with gzip compression yields ~500 bytes/span average.
-
-**(B) P2P trace context overhead** (added to existing messages, always-on regardless of sampling):
-
-| Message Type  | Rate     | Context Size | Bandwidth     |
-| ------------- | -------- | ------------ | ------------- |
-| TMTransaction | ~100/sec | 29 bytes     | ~2.9 KB/s     |
-| TMValidation  | ~50/sec  | 29 bytes     | ~1.5 KB/s     |
-| TMProposeSet  | ~10/sec  | 29 bytes     | ~0.3 KB/s     |
-| **Total P2P** |          |              | **~4.7 KB/s** |
-
-> **Combined**: 25 KB/s (OTLP export at 10%) + 5 KB/s (P2P context) ≈ **~30 KB/s typical**. The 10-50 KB/s range covers 10-20% sampling under normal to peak mainnet load.
-
-**Latency (<2%) — Calculation**:
-
-| Path                           | Tracing Cost | Baseline | Overhead |
-| ------------------------------ | ------------ | -------- | -------- |
-| Fast RPC (e.g., `server_info`) | 2.75 μs      | ~1 ms    | 0.275%   |
-| Slow RPC (e.g., `path_find`)   | 2.75 μs      | ~100 ms  | 0.003%   |
-| Transaction processing         | 4.0 μs       | ~200 μs  | 2.0%     |
-| Consensus round                | 36 μs        | ~3 sec   | 0.001%   |
-
-> At p99, even the worst case (TX processing at 2.0%) is within the 1-3% range. RPC and consensus overhead are negligible. On production hardware, TX overhead drops to ~1.3%.
-
-### Per-Message Overhead (Context Propagation)
-
-Each P2P message carries trace context with the following overhead:
-
-| Field         | Size          | Description                               |
-| ------------- | ------------- | ----------------------------------------- |
-| `trace_id`    | 16 bytes      | Unique identifier for the entire trace    |
-| `span_id`     | 8 bytes       | Current span (becomes parent on receiver) |
-| `trace_flags` | 1 byte        | Sampling decision flags                   |
-| `trace_state` | 0-4 bytes     | Optional vendor-specific data             |
-| **Total**     | **~29 bytes** | **Added per traced P2P message**          |
-
-```mermaid
-flowchart LR
-    subgraph msg["P2P Message with Trace Context"]
-        A["Original Message<br/>(variable size)"] --> B["+ TraceContext<br/>(~29 bytes)"]
-    end
-
-    subgraph breakdown["Context Breakdown"]
-        C["trace_id<br/>16 bytes"]
-        D["span_id<br/>8 bytes"]
-        E["flags<br/>1 byte"]
-        F["state<br/>0-4 bytes"]
-    end
-
-    B --> breakdown
-
-    style A fill:#424242,stroke:#212121,color:#fff
-    style B fill:#2e7d32,stroke:#1b5e20,color:#fff
-    style C fill:#1565c0,stroke:#0d47a1,color:#fff
-    style D fill:#1565c0,stroke:#0d47a1,color:#fff
-    style E fill:#e65100,stroke:#bf360c,color:#fff
-    style F fill:#4a148c,stroke:#2e0d57,color:#fff
-```
-
-**Reading the diagram:**
-
-- **Original Message (gray, left)**: The existing P2P message payload of variable size — this is unchanged; trace context is appended, never modifying the original data.
-- **+ TraceContext (green, right of message)**: The additional 29-byte context block attached to each traced message; the arrow from the original message shows it is a pure addition.
-- **Context Breakdown (right subgraph)**: The four fields — `trace_id` (16 bytes), `span_id` (8 bytes), `flags` (1 byte), and `state` (0-4 bytes) — show exactly what is added and their individual sizes.
-- **Color coding**: Blue fields (`trace_id`, `span_id`) are the core identifiers required for trace correlation; orange (`flags`) controls sampling decisions; purple (`state`) is optional vendor data typically omitted.
-
-> **Note**: 29 bytes represents ~1-6% overhead depending on message size (500B simple TX to 5KB proposal), which is acceptable for the observability benefits provided.
-
-### Mitigation Strategies
-
-```mermaid
-flowchart LR
-    A["Head Sampling<br/>10% default"] --> B["Tail Sampling<br/>Keep errors/slow"] --> C["Batch Export<br/>Reduce I/O"] --> D["Conditional Compile<br/>XRPL_ENABLE_TELEMETRY"]
-
-    style A fill:#1565c0,stroke:#0d47a1,color:#fff
-    style B fill:#2e7d32,stroke:#1b5e20,color:#fff
-    style C fill:#e65100,stroke:#bf360c,color:#fff
-    style D fill:#4a148c,stroke:#2e0d57,color:#fff
-```
-
-> For a detailed explanation of head vs. tail sampling, see Slide 9.
-
-### Kill Switches (Rollback Options)
-
-1. **Config Disable**: Set `enabled=0` in config → instant disable, no restart needed for sampling
-2. **Rebuild**: Compile with `XRPL_ENABLE_TELEMETRY=OFF` → zero overhead (no-op)
-3. **Full Revert**: Clean separation allows easy commit reversion
+| Component                  | Version |
+| -------------------------- | ------- |
+| OTel Collector (contrib)   | 0.121.0 |
+| Grafana Tempo              | 2.7.2   |
+| Grafana Loki               | 3.4.2   |
+| Prometheus                 | latest  |
+| Grafana                    | 11.5.2  |
+| **Dashboards provisioned** | **15**  |
 
 ---
 
-## Slide 9: Sampling Strategies — Head vs. Tail
+## Slide 12: Future Phases
 
-> Sampling controls **which traces are recorded and exported**. Without sampling, every operation generates a trace — at 500+ spans/sec, this overwhelms storage and network. Sampling lets you keep the signal, discard the noise.
+### Phase 10 — Synthetic Workload Validation
 
-### Head Sampling (Decision at Start)
+| Aspect      | Detail                                                             |
+| ----------- | ------------------------------------------------------------------ |
+| Goal        | Drive instrumented surfaces under reproducible load                |
+| Why         | Validate dashboards, catch regressions, measure overhead at scale  |
+| Deliverable | Workload generator + assertion suite (RPC/TX/peer churn scenarios) |
+| Effort      | ~2 weeks                                                           |
 
-The sampling decision is made **when a trace begins**, before any work is done. A random number is generated; if it falls within the configured ratio, the entire trace is recorded. Otherwise, the trace is silently dropped.
+### Phase 11 — Admin-RPC Receiver (`xrpl_*` metrics)
+
+| Aspect      | Detail                                                                                                                                        |
+| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
+| Goal        | Custom Go OTel Collector receiver polls xrpld admin RPC, emits `xrpl_*` Prometheus metrics                                                    |
+| Why         | Admin-RPC-only data has no native export — every consumer reinvents JSON-RPC polling                                                          |
+| Scope       | `validators` (UNL, listed keys), `feature` (amendments), `peers` (per-peer detail), `amm_info`, `book_offers`, `fee` (detail tiers)           |
+| Excluded    | `server_info` / `get_counts` basics — Phase 9 (#6513) already ships `xrpld_server_info` + 14 gauges/histograms natively from in-process state |
+| Deliverable | Go receiver plugin + custom Collector binary + 4 Grafana dashboards (UNL, amendments, AMM, DEX) + Prometheus alerts                           |
+| Effort      | ~3 weeks                                                                                                                                      |
 
 ```mermaid
 flowchart LR
-    A["New Request<br/>Arrives"] --> B{"Random < 10%?"}
-    B -->|"Yes (1 in 10)"| C["Record Entire Trace<br/>(all spans)"]
-    B -->|"No (9 in 10)"| D["Drop Entire Trace<br/>(zero overhead)"]
+    rpc["xrpld admin RPC<br/>(validators, feature, peers,<br/>amm_info, book_offers, fee)"] -->|JSON-RPC poll| recv["Custom Go receiver<br/>(in Collector)"]
+    recv -->|xrpl_* metrics| prom["Prometheus"]
+    prom --> graf["Grafana dashboards"]
 
-    style C fill:#2e7d32,stroke:#1b5e20,color:#fff
-    style D fill:#c62828,stroke:#8c2809,color:#fff
-    style B fill:#1565c0,stroke:#0d47a1,color:#fff
+    style rpc fill:#2e7d32,stroke:#1b5e20,color:#fff
+    style recv fill:#1565c0,stroke:#0d47a1,color:#fff
+    style prom fill:#e65100,stroke:#bf360c,color:#fff
+    style graf fill:#6a1b9a,stroke:#4a148c,color:#fff
 ```
 
-| Aspect                        | Details                                                                                                                                                                                                  |
-| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Where it runs**             | Inside xrpld (SDK-level). Configured via `sampling_ratio` in `xrpld.cfg`.                                                                                                                                |
-| **When the decision happens** | At trace creation time — before the first span is even populated.                                                                                                                                        |
-| **How it works**              | `sampling_ratio=0.1` means each trace has a 10% probability of being recorded. Dropped traces incur near-zero overhead (no spans created, no attributes set, no export).                                 |
-| **Propagation**               | Once a trace is sampled, the `trace_flags` field (1 byte in the context header) tells downstream nodes to also sample it. Unsampled traces propagate `trace_flags=0`, so downstream nodes skip them too. |
-| **Pros**                      | Lowest overhead. Simple to configure. Predictable resource usage.                                                                                                                                        |
-| **Cons**                      | **Blind** — it doesn't know if the trace will be interesting. A rare error or slow consensus round has only a 10% chance of being captured.                                                              |
-| **Best for**                  | High-volume, steady-state traffic where most traces look similar (e.g., routine RPC requests).                                                                                                           |
-
-**xrpld configuration**:
-
-```ini
-[telemetry]
-# Record 10% of traces (recommended for production)
-sampling_ratio=0.1
-```
-
-### Tail Sampling (Decision at End)
-
-The sampling decision is made **after the trace completes**, based on its actual content — was it slow? Did it error? Was it a consensus round? This requires buffering complete traces before deciding.
-
-```mermaid
-flowchart TB
-    A["All Traces<br/>Buffered (100%)"] --> B["OTel Collector<br/>Evaluates Rules"]
-
-    B --> C{"Error?"}
-    C -->|Yes| K["KEEP"]
-
-    C -->|No| D{"Slow?<br/>(>5s consensus,<br/>>1s RPC)"}
-    D -->|Yes| K
-
-    D -->|No| E{"Random < 10%?"}
-    E -->|Yes| K
-    E -->|No| F["DROP"]
-
-    style K fill:#2e7d32,stroke:#1b5e20,color:#fff
-    style F fill:#c62828,stroke:#8c2809,color:#fff
-    style B fill:#1565c0,stroke:#0d47a1,color:#fff
-    style C fill:#e65100,stroke:#bf360c,color:#fff
-    style D fill:#e65100,stroke:#bf360c,color:#fff
-    style E fill:#4a148c,stroke:#2e0d57,color:#fff
-```
-
-| Aspect                        | Details                                                                                                                                                                                                 |
-| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Where it runs**             | In the **OTel Collector** (external process), not inside xrpld. xrpld exports 100% of traces; the Collector decides what to keep.                                                                       |
-| **When the decision happens** | After the Collector has received all spans for a trace (waits `decision_wait=10s` for stragglers).                                                                                                      |
-| **How it works**              | Policy rules evaluate the completed trace: keep all errors, keep slow operations above a threshold, keep all consensus rounds, then probabilistically sample the rest at 10%.                           |
-| **Pros**                      | **Never misses important traces**. Errors, slow requests, and consensus anomalies are always captured regardless of probability.                                                                        |
-| **Cons**                      | Higher resource usage — xrpld must export 100% of spans to the Collector, which buffers them in memory before deciding. The Collector needs more RAM (configured via `num_traces` and `decision_wait`). |
-| **Best for**                  | Production troubleshooting where you can't afford to miss errors or anomalies.                                                                                                                          |
-
-**Collector configuration** (tail sampling rules for xrpld):
-
-```yaml
-processors:
-  tail_sampling:
-    decision_wait: 10s # Wait for all spans in a trace
-    num_traces: 100000 # Buffer up to 100K concurrent traces
-    policies:
-      - name: errors # Always keep error traces
-        type: status_code
-        status_code: { status_codes: [ERROR] }
-
-      - name: slow-consensus # Keep consensus rounds >5s
-        type: latency
-        latency: { threshold_ms: 5000 }
-
-      - name: slow-rpc # Keep slow RPC requests >1s
-        type: latency
-        latency: { threshold_ms: 1000 }
-
-      - name: probabilistic # Sample 10% of everything else
-        type: probabilistic
-        probabilistic: { sampling_percentage: 10 }
-```
-
-### Head vs. Tail — Side-by-Side
-
-|                               | Head Sampling                            | Tail Sampling                                    |
-| ----------------------------- | ---------------------------------------- | ------------------------------------------------ |
-| **Decision point**            | Trace start (inside xrpld)               | Trace end (in OTel Collector)                    |
-| **Knows trace content?**      | No (random coin flip)                    | Yes (evaluates completed trace)                  |
-| **Overhead on xrpld**         | Lowest (dropped traces = no-op)          | Higher (must export 100% to Collector)           |
-| **Collector resource usage**  | Low (receives only sampled traces)       | Higher (buffers all traces before deciding)      |
-| **Captures all errors?**      | No (only if trace was randomly selected) | **Yes** (error policy catches them)              |
-| **Captures slow operations?** | No (random)                              | **Yes** (latency policy catches them)            |
-| **Configuration**             | `xrpld.cfg`: `sampling_ratio=0.1`        | `otel-collector.yaml`: `tail_sampling` processor |
-| **Best for**                  | High-throughput steady-state             | Troubleshooting & anomaly detection              |
-
-### Recommended Strategy for xrpld
-
-Use **both** in a layered approach:
-
-```mermaid
-flowchart LR
-    subgraph xrpld["xrpld (Head Sampling)"]
-        HS["sampling_ratio=1.0<br/>(export everything)"]
-    end
-
-    subgraph collector["OTel Collector (Tail Sampling)"]
-        TS["Keep: errors + slow + 10% random<br/>Drop: routine traces"]
-    end
-
-    subgraph storage["Backend Storage"]
-        ST["Only interesting traces<br/>stored long-term"]
-    end
-
-    xrpld -->|"100% of spans"| collector -->|"~15-20% kept"| storage
-
-    style xrpld fill:#424242,stroke:#212121,color:#fff
-    style collector fill:#1565c0,stroke:#0d47a1,color:#fff
-    style storage fill:#2e7d32,stroke:#1b5e20,color:#fff
-```
-
-> **Why this works**: xrpld exports everything (no blind drops), the Collector applies intelligent filtering (keep errors/slow/anomalies, sample the rest), and only ~15-20% of traces reach storage. If Collector resource usage becomes a concern, add head sampling at `sampling_ratio=0.5` to halve the export volume while still giving the Collector enough data for good tail-sampling decisions.
+> Phase 11 fills the gap above Phase 9 — data only reachable via admin RPC, not via in-process metric callbacks.
 
 ---
 
-## Slide 10: Data Collection & Privacy
+## Slide 11: External Dashboard Parity (Phase 7+)
 
-### What Data is Collected
+### Bridging Community Monitoring into Native OTel
 
-| Category        | Attributes Collected                                                                 | Purpose                     |
-| --------------- | ------------------------------------------------------------------------------------ | --------------------------- |
-| **Transaction** | `tx.hash`, `tx.type`, `tx.result`, `tx.fee`, `ledger_index`                          | Trace transaction lifecycle |
-| **Consensus**   | `round`, `phase`, `mode`, `proposers` (count of proposing validators), `duration_ms` | Analyze consensus timing    |
-| **RPC**         | `command`, `version`, `status`, `duration_ms`                                        | Monitor RPC performance     |
-| **Peer**        | `peer.id`(public key), `latency_ms`, `message.type`, `message.size`                  | Network topology analysis   |
-| **Ledger**      | `ledger.hash`, `ledger.index`, `close_time`, `tx_count`                              | Ledger progression tracking |
-| **Job**         | `job.type`, `queue_ms`, `worker`                                                     | JobQueue performance        |
+The community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard) provides 86 metrics for validator operators. We integrated the 29 missing metrics natively into the OTel pipeline.
 
-### What is NOT Collected (Privacy Guarantees)
+### New Metric Categories
 
 ```mermaid
-flowchart LR
-    subgraph notCollected["❌ NOT Collected"]
-        direction LR
-        A["Private Keys"] ~~~ B["Account Balances"] ~~~ C["Transaction Amounts"]
+graph LR
+    subgraph "New Observable Gauges"
+        VH["Validator Health<br/>amendment_blocked, UNL expiry,<br/>quorum"]
+        PQ["Peer Quality<br/>P90 latency, insane peers,<br/>version awareness"]
+        LE["Ledger Economy<br/>fees, reserves, tx rate,<br/>ledger age"]
+        ST["State Tracking<br/>state value 0-6,<br/>time in state"]
+        VA["Validation Agreement<br/>1h/24h agreement %,<br/>agreements, misses"]
     end
 
-    subgraph alsoNot["❌ Also Excluded"]
-        direction LR
-        D["IP Addresses<br/>(configurable)"] ~~~ E["Personal Data"] ~~~ F["Raw TX Payloads"]
+    subgraph "Counters"
+        C1["ledgers_closed_total"]
+        C2["validations_sent_total"]
+        C3["state_changes_total"]
     end
 
-    style A fill:#c62828,stroke:#8c2809,color:#fff
-    style B fill:#c62828,stroke:#8c2809,color:#fff
-    style C fill:#c62828,stroke:#8c2809,color:#fff
-    style D fill:#c62828,stroke:#8c2809,color:#fff
-    style E fill:#c62828,stroke:#8c2809,color:#fff
-    style F fill:#c62828,stroke:#8c2809,color:#fff
+    style VH fill:#1565c0,color:#fff
+    style PQ fill:#2e7d32,color:#fff
+    style LE fill:#e65100,color:#fff
+    style ST fill:#6a1b9a,color:#fff
+    style VA fill:#c62828,color:#fff
+    style C1 fill:#37474f,color:#fff
+    style C2 fill:#37474f,color:#fff
+    style C3 fill:#37474f,color:#fff
 ```
 
-**Reading the diagram:**
+### ValidationTracker — Agreement Computation
 
-- **NOT Collected (top row, red)**: Private Keys, Account Balances, and Transaction Amounts are explicitly excluded — these are financial/security-sensitive fields that telemetry never touches.
-- **Also Excluded (bottom row, red)**: IP Addresses (configurable per deployment), Personal Data, and Raw TX Payloads are also excluded — these protect operator and user privacy.
-- **All-red styling**: Every box is styled in red to visually reinforce that these are hard exclusions, not optional — the telemetry system has no code path to collect any of these fields.
-- **Two-row layout**: The split between "NOT Collected" and "Also Excluded" distinguishes between financial data (top) and operational/personal data (bottom), making the privacy boundaries clear to auditors.
+```mermaid
+sequenceDiagram
+    participant C as RCLConsensus
+    participant VT as ValidationTracker
+    participant MR as MetricsRegistry
+    participant P as Prometheus
 
-### Privacy Protection Mechanisms
+    C->>VT: recordOurValidation(hash, seq)
+    Note over VT: Stores pending event
+    C->>VT: recordNetworkValidation(hash, seq)
+    Note over VT: Marks network validated
+    MR->>VT: reconcile() [every 10s]
+    Note over VT: After 8s grace period:<br/>both validated → agreed<br/>only one → missed<br/>5min late repair window
+    MR->>P: Export agreement_pct_1h/24h
+```
 
-| Mechanism                  | Description                                                   |
-| -------------------------- | ------------------------------------------------------------- |
-| **Account Hashing**        | `xrpl.tx.account` is hashed at collector level before storage |
-| **Configurable Redaction** | Sensitive fields can be excluded via config                   |
-| **Sampling**               | Only 10% of traces recorded by default (reduces exposure)     |
-| **Local Control**          | Node operators control what gets exported                     |
-| **No Raw Payloads**        | Transaction content is never recorded, only metadata          |
+### New Grafana Dashboards
 
-> **Key Principle**: Telemetry collects **operational metadata** (timing, counts, hashes) — never **sensitive content** (keys, balances, amounts).
+| Dashboard        | Key Panels                                          |
+| ---------------- | --------------------------------------------------- |
+| Validator Health | Agreement %, amendment blocked, quorum, state value |
+| Peer Quality     | P90 latency, version awareness, upgrade recommended |
+| Ledger Economy   | Base fee, reserves, ledger age, transaction rate    |
 
 ---
 

cmake/XrplCore.cmake

@@ -78,6 +78,13 @@ include(target_link_modules)
 # Level 01
 add_module(xrpl beast)
 target_link_libraries(xrpl.libxrpl.beast PUBLIC xrpl.imports.main)
+# OTelCollector in beast/insight uses OTel Metrics SDK when telemetry is enabled.
+if(telemetry)
+    target_link_libraries(
+        xrpl.libxrpl.beast
+        PUBLIC opentelemetry-cpp::opentelemetry-cpp
+    )
+endif()
 
 include(GitInfo)
 add_module(xrpl git)

cmake/XrplDocs.cmake

@@ -27,8 +27,12 @@ file(
     src/*.cpp
     src/*.md
     Builds/*.md
-    *.md
 )
+# Add only top-level .md files (README, CONTRIBUTING, etc.) without
+# recursing into dot-directories like .claude/ whose files are not
+# valid Doxygen/CMake sources.
+file(GLOB doxygen_top_md CONFIGURE_DEPENDS "*.md")
+list(APPEND doxygen_input ${doxygen_top_md})
 list(APPEND doxygen_input external/README.md)
 set(dependencies "${doxygen_input}" "${doxyfile}")
 

conanfile.py

@@ -54,7 +54,11 @@ class Xrpl(ConanFile):
         "rocksdb": True,
         "shared": False,
         "static": True,
-        "telemetry": True,
+        # OTel-overhead baseline branch: telemetry compiled OUT so a perf-iac
+        # comparison run (this branch as baseline vs phase-10 as on-demand)
+        # measures the full linked-in + hot-path cost of telemetry. Do not
+        # merge this flip into a feature branch.
+        "telemetry": False,
         "tests": False,
         "unity": False,
         "xrpld": False,

cspell.config.yaml

@@ -95,6 +95,8 @@ words:
   - dcmake
   - dearmor
   - dedented
+  - Dedup
+  - dedup
   - deleteme
   - demultiplexer
   - deserializaton
@@ -108,6 +110,7 @@ words:
   - enabled
   - enablerepo
   - endmacro
+  - EOCFG
   - exceptioned
   - EXPECT_STREQ
   - exfiltration
@@ -160,6 +163,7 @@ words:
   - libxrpl
   - llection
   - LOCALGOOD
+  - logql
   - logwstream
   - lseq
   - lsmf
@@ -200,6 +204,7 @@ words:
   - nixfmt
   - nixos
   - nixpkgs
+  - NETOP
   - NOLINT
   - NOLINTNEXTLINE
   - nonxrp
@@ -222,9 +227,12 @@ words:
   - permdex
   - perminute
   - permissioned
+  - pgrep
+  - pkill
   - pimpl
   - pointee
   - populator
+  - pratik
   - preauth
   - preauthorization
   - preauthorize
@@ -242,6 +250,7 @@ words:
   - Raphson
   - reparent
   - replayer
+  - reqps
   - rerere
   - retriable
   - RIPD
@@ -352,5 +361,5 @@ words:
   - xxhasher
   - xychart
   - zpages
-  - pratik
-  - dedup
+  - ripplex
+  - mseconds

docker/telemetry/.gitignore

@@ -0,0 +1,2 @@
+# Runtime data generated by xrpld and telemetry stack
+data/

docker/telemetry/TESTING.md

@@ -0,0 +1,641 @@
+# OpenTelemetry Integration Testing Guide
+
+This document describes how to verify the xrpld OpenTelemetry telemetry
+pipeline end-to-end, from span generation through the observability stack
+(otel-collector, Tempo, Prometheus, Grafana).
+
+---
+
+## Prerequisites
+
+### Build xrpld with telemetry
+
+```bash
+conan install . --build=missing -o telemetry=True
+cmake --preset default -Dtelemetry=ON
+cmake --build --preset default --target xrpld
+```
+
+The binary is at `.build/xrpld`.
+
+### Required tools
+
+- **Docker** with `docker compose` (v2)
+- **curl**
+- **jq** (JSON processor)
+
+### Verify binary
+
+```bash
+.build/xrpld --version
+```
+
+---
+
+## Test 1: Single-Node Standalone (Quick Verification)
+
+This test verifies RPC and transaction spans in standalone mode. Consensus
+spans will not fire because standalone mode does not run consensus.
+
+### Step 1: Start the observability stack
+
+```bash
+docker compose -f docker/telemetry/docker-compose.yml up -d
+```
+
+Wait for services to be ready:
+
+```bash
+# otel-collector health
+curl -sf http://localhost:13133/ && echo "collector ready"
+
+# Tempo readiness
+curl -sf http://localhost:3200/ready >/dev/null && echo "tempo ready"
+```
+
+### Step 2: Start xrpld in standalone mode
+
+```bash
+.build/xrpld --conf docker/telemetry/xrpld-telemetry.cfg -a --start
+```
+
+Wait a few seconds for the node to initialize.
+
+### Step 3: Exercise RPC spans
+
+```bash
+# server_info
+curl -s http://localhost:5005 \
+    -d '{"method":"server_info"}' | jq .result.info.server_state
+
+# server_state
+curl -s http://localhost:5005 \
+    -d '{"method":"server_state"}' | jq .result.state.server_state
+
+# ledger
+curl -s http://localhost:5005 \
+    -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' |
+    jq .result.ledger_current_index
+```
+
+### Step 4: Submit a transaction
+
+Close the ledger first (required in standalone mode):
+
+```bash
+curl -s http://localhost:5005 -d '{"method":"ledger_accept"}'
+```
+
+Submit a Payment from the genesis account:
+
+```bash
+curl -s http://localhost:5005 -d '{
+  "method": "submit",
+  "params": [{
+    "secret": "snoPBrXtMeMyMHUVTgbuqAfg1SUTb",
+    "tx_json": {
+      "TransactionType": "Payment",
+      "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh",
+      "Destination": "rPMh7Pi9ct699iZUTWzJaUMR1o42VEfGqF",
+      "Amount": "10000000"
+    }
+  }]
+}' | jq .result.engine_result
+```
+
+Expected result: `"tesSUCCESS"`.
+
+Close the ledger again to finalize:
+
+```bash
+curl -s http://localhost:5005 -d '{"method":"ledger_accept"}'
+```
+
+### Step 5: Verify traces in Tempo
+
+Wait 5 seconds for the batch export, then:
+
+```bash
+TEMPO="http://localhost:3200"
+
+# Check xrpld service is registered
+curl -s "$TEMPO/api/v2/search/tag/resource.service.name/values" | jq '.tagValues[].value'
+
+# Check RPC spans
+curl -s "$TEMPO/api/search" \
+    --data-urlencode 'q={resource.service.name="xrpld" && name="rpc.http_request"}' \
+    --data-urlencode 'limit=5' | jq '.traces | length'
+
+curl -s "$TEMPO/api/search" \
+    --data-urlencode 'q={resource.service.name="xrpld" && name="rpc.process"}' \
+    --data-urlencode 'limit=5' | jq '.traces | length'
+
+curl -s "$TEMPO/api/search" \
+    --data-urlencode 'q={resource.service.name="xrpld" && name="rpc.command.server_info"}' \
+    --data-urlencode 'limit=5' | jq '.traces | length'
+
+# Check transaction spans
+curl -s "$TEMPO/api/search" \
+    --data-urlencode 'q={resource.service.name="xrpld" && name="tx.process"}' \
+    --data-urlencode 'limit=5' | jq '.traces | length'
+```
+
+Or open Grafana Explore with Tempo datasource: http://localhost:3000
+
+### Step 6: Teardown
+
+```bash
+# Kill xrpld (Ctrl+C or)
+kill $(pgrep -f 'xrpld.*xrpld-telemetry')
+
+# Stop observability stack
+docker compose -f docker/telemetry/docker-compose.yml down
+
+# Clean xrpld data
+rm -rf data/
+```
+
+### Expected spans (standalone mode)
+
+| Span Name                   | Expected | Notes                         |
+| --------------------------- | -------- | ----------------------------- |
+| `rpc.http_request`          | Yes      | Every HTTP RPC call           |
+| `rpc.process`               | Yes      | Every RPC processing          |
+| `rpc.command.server_info`   | Yes      | server_info RPC               |
+| `rpc.command.server_state`  | Yes      | server_state RPC              |
+| `rpc.command.ledger`        | Yes      | ledger RPC                    |
+| `rpc.command.submit`        | Yes      | submit RPC                    |
+| `rpc.command.ledger_accept` | Yes      | ledger_accept RPC             |
+| `tx.process`                | Yes      | Transaction submission        |
+| `tx.receive`                | No       | No peers in standalone        |
+| `consensus.*`               | No       | Consensus disabled standalone |
+
+---
+
+## Test 2: 6-Node Consensus Network (Full Verification)
+
+This test verifies ALL span categories including consensus and peer
+transaction relay, using a 6-node validator network.
+
+### Automated
+
+Run the integration test script:
+
+```bash
+bash docker/telemetry/integration-test.sh
+```
+
+The script will:
+
+1. Start the observability stack
+2. Generate 6 validator key pairs
+3. Create config files for each node
+4. Start all 6 nodes
+5. Wait for consensus ("proposing" state)
+6. Exercise RPC, submit transactions
+7. Verify all span categories in Tempo
+8. Verify spanmetrics in Prometheus
+9. Print results and leave the stack running
+
+### Manual
+
+If you prefer to run the steps manually:
+
+#### Step 1: Start observability stack
+
+```bash
+docker compose -f docker/telemetry/docker-compose.yml up -d
+```
+
+#### Step 2: Generate validator keys
+
+Start a temporary standalone xrpld:
+
+```bash
+.build/xrpld --conf docker/telemetry/xrpld-telemetry.cfg -a --start &
+TEMP_PID=$!
+sleep 5
+```
+
+Generate 6 key pairs:
+
+```bash
+for i in $(seq 1 6); do
+    curl -s http://localhost:5005 \
+        -d '{"method":"validation_create"}' | jq '.result'
+done
+```
+
+Record the `validation_seed` and `validation_public_key` for each.
+Kill the temporary node:
+
+```bash
+kill $TEMP_PID
+rm -rf data/
+```
+
+#### Step 3: Create node configs
+
+For each node (1-6), create a config file. Template:
+
+```ini
+[server]
+port_rpc
+port_peer
+
+[port_rpc]
+port = {5004 + node_number}
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[port_peer]
+port = {51234 + node_number}
+ip = 0.0.0.0
+protocol = peer
+
+[node_db]
+type=NuDB
+path=/tmp/xrpld-integration/node{N}/nudb
+online_delete=256
+
+[database_path]
+/tmp/xrpld-integration/node{N}/db
+
+[debug_logfile]
+/tmp/xrpld-integration/node{N}/debug.log
+
+[validation_seed]
+{seed from step 2}
+
+[validators_file]
+/tmp/xrpld-integration/validators.txt
+
+[ips_fixed]
+127.0.0.1 51235
+127.0.0.1 51236
+127.0.0.1 51237
+127.0.0.1 51238
+127.0.0.1 51239
+127.0.0.1 51240
+
+[peer_private]
+1
+
+[telemetry]
+enabled=1
+endpoint=http://localhost:4318/v1/traces
+sampling_ratio=1.0
+batch_size=512
+batch_delay_ms=2000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=0
+trace_ledger=1
+
+[rpc_startup]
+{ "command": "log_level", "severity": "warning" }
+
+[ssl_verify]
+0
+```
+
+#### Step 4: Create validators.txt
+
+```ini
+[validators]
+{public_key_1}
+{public_key_2}
+{public_key_3}
+{public_key_4}
+{public_key_5}
+{public_key_6}
+```
+
+#### Step 5: Start all 6 nodes
+
+```bash
+for i in $(seq 1 6); do
+    .build/xrpld --conf /tmp/xrpld-integration/node$i/xrpld.cfg --start &
+    echo $! >/tmp/xrpld-integration/node$i/xrpld.pid
+done
+```
+
+#### Step 6: Wait for consensus
+
+Poll each node until `server_state` = `"proposing"`:
+
+```bash
+for port in 5005 5006 5007 5008 5009 5010; do
+    while true; do
+        state=$(curl -s http://localhost:$port \
+            -d '{"method":"server_info"}' |
+            jq -r '.result.info.server_state')
+        echo "Port $port: $state"
+        [ "$state" = "proposing" ] && break
+        sleep 5
+    done
+done
+```
+
+#### Step 7: Exercise RPC and submit transaction
+
+```bash
+# RPC calls
+curl -s http://localhost:5005 -d '{"method":"server_info"}'
+curl -s http://localhost:5005 -d '{"method":"server_state"}'
+curl -s http://localhost:5005 -d '{"method":"ledger","params":[{"ledger_index":"current"}]}'
+
+# Submit transaction
+curl -s http://localhost:5005 -d '{
+  "method": "submit",
+  "params": [{
+    "secret": "snoPBrXtMeMyMHUVTgbuqAfg1SUTb",
+    "tx_json": {
+      "TransactionType": "Payment",
+      "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh",
+      "Destination": "rPMh7Pi9ct699iZUTWzJaUMR1o42VEfGqF",
+      "Amount": "10000000"
+    }
+  }]
+}'
+```
+
+Wait 15 seconds for consensus and batch export.
+
+#### Step 8: Verify in Tempo
+
+See the "Verification Queries" section below.
+
+---
+
+## Expected Span Catalog
+
+All 16 production span names instrumented across Phases 2-5:
+
+| Span Name                   | Source File       | Phase | Key Attributes                                                                           | How to Trigger            |
+| --------------------------- | ----------------- | ----- | ---------------------------------------------------------------------------------------- | ------------------------- |
+| `rpc.http_request`          | ServerHandler.cpp | 2     | --                                                                                       | Any HTTP RPC call         |
+| `rpc.ws_upgrade`            | ServerHandler.cpp | 2     | --                                                                                       | WebSocket upgrade         |
+| `rpc.ws_message`            | ServerHandler.cpp | 2     | --                                                                                       | WebSocket RPC message     |
+| `rpc.process`               | ServerHandler.cpp | 2     | --                                                                                       | RPC processing            |
+| `rpc.command.<name>`        | RPCHandler.cpp    | 2     | `command`, `version`, `rpc_role`                                                         | Any RPC command           |
+| `tx.process`                | NetworkOPs.cpp    | 3     | `xrpl.tx.hash`, `local`, `path`                                                          | Submit transaction        |
+| `tx.receive`                | PeerImp.cpp       | 3     | `xrpl.peer.id`                                                                           | Peer relays transaction   |
+| `consensus.proposal.send`   | RCLConsensus.cpp  | 4     | `xrpl.consensus.round`                                                                   | Consensus proposing phase |
+| `consensus.ledger_close`    | RCLConsensus.cpp  | 4     | `xrpl.consensus.ledger.seq`, `xrpl.consensus.mode`                                       | Ledger close event        |
+| `consensus.accept`          | RCLConsensus.cpp  | 4     | `xrpl.consensus.proposers`, `xrpl.consensus.round_time_ms`                               | Ledger accepted           |
+| `consensus.validation.send` | RCLConsensus.cpp  | 4     | `xrpl.consensus.ledger.seq`, `xrpl.consensus.proposing`                                  | Validation sent           |
+| `consensus.accept.apply`    | RCLConsensus.cpp  | 4     | `xrpl.consensus.close_time`, `close_time_correct`, `close_resolution_ms`, `state`        | Ledger apply + close time |
+| `tx.apply`                  | BuildLedger.cpp   | 5     | `xrpl.ledger.tx_count`, `xrpl.ledger.tx_failed`                                          | Ledger close (tx set)     |
+| `ledger.build`              | BuildLedger.cpp   | 5     | `xrpl.ledger.seq`, `xrpl.ledger.close_time`, `close_time_correct`, `close_resolution_ms` | Ledger build              |
+| `ledger.validate`           | LedgerMaster.cpp  | 5     | `xrpl.ledger.seq`, `xrpl.ledger.validations`                                             | Ledger validated          |
+| `ledger.store`              | LedgerMaster.cpp  | 5     | `xrpl.ledger.seq`                                                                        | Ledger stored             |
+| `peer.proposal.receive`     | PeerImp.cpp       | 5     | `xrpl.peer.id`, `proposal_trusted`                                                       | Peer sends proposal       |
+| `peer.validation.receive`   | PeerImp.cpp       | 5     | `xrpl.peer.id`, `validation_trusted`                                                     | Peer sends validation     |
+
+---
+
+## Verification Queries
+
+### Tempo API
+
+Base URL: `http://localhost:3200`
+
+```bash
+TEMPO="http://localhost:3200"
+
+# List all services
+curl -s "$TEMPO/api/v2/search/tag/resource.service.name/values" | jq '.tagValues[].value'
+
+# Query traces by operation
+for op in "rpc.http_request" "rpc.ws_upgrade" "rpc.ws_message" "rpc.process" \
+    "rpc.command.server_info" "rpc.command.server_state" "rpc.command.ledger" \
+    "tx.process" "tx.receive" "tx.apply" \
+    "consensus.proposal.send" "consensus.ledger_close" \
+    "consensus.accept" "consensus.accept.apply" \
+    "consensus.validation.send" \
+    "ledger.build" "ledger.validate" "ledger.store" \
+    "peer.proposal.receive" "peer.validation.receive"; do
+    count=$(curl -s "$TEMPO/api/search" \
+        --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \
+        --data-urlencode "limit=5" |
+        jq '.traces | length')
+    printf "%-35s %s traces\n" "$op" "$count"
+done
+```
+
+### Prometheus API
+
+Base URL: `http://localhost:9090`
+
+```bash
+PROM="http://localhost:9090"
+
+# Span call counts (from spanmetrics connector)
+curl -s "$PROM/api/v1/query?query=traces_span_metrics_calls_total" |
+    jq '.data.result[] | {span: .metric.span_name, count: .value[1]}'
+
+# Latency histogram
+curl -s "$PROM/api/v1/query?query=traces_span_metrics_duration_milliseconds_count" |
+    jq '.data.result[] | {span: .metric.span_name, count: .value[1]}'
+
+# RPC calls by command
+curl -s "$PROM/api/v1/query?query=traces_span_metrics_calls_total{span_name=~\"rpc.command.*\"}" |
+    jq '.data.result[] | {command: .metric["command"], count: .value[1]}'
+```
+
+### Grafana
+
+Open http://localhost:3000 (anonymous admin access enabled).
+
+Pre-configured dashboards:
+
+- **RPC Performance**: Request rates, latency percentiles by command, top commands, WebSocket rate
+- **Transaction Overview**: Transaction processing rates, apply duration, peer relay, failed tx rate
+- **Consensus Health**: Consensus round duration, proposer counts, mode tracking, accept heatmap
+- **Ledger Operations**: Build/validate/store rates and durations, TX apply metrics
+- **Peer Network**: Proposal/validation receive rates, trusted vs untrusted breakdown (requires `trace_peer=1`)
+
+Pre-configured datasources:
+
+- **Tempo**: Trace data at `http://tempo:3200`
+- **Prometheus**: Metrics at `http://prometheus:9090`
+- **Loki**: Log data at `http://loki:3100` (via Grafana Explore)
+
+---
+
+## Test 3: Log-Trace Correlation (Phase 8)
+
+Phase 8 injects `trace_id` and `span_id` into xrpld's log output when
+a log line is emitted within an active OTel span. This test verifies the
+end-to-end log-trace correlation pipeline.
+
+### Step 1: Verify trace_id in log output
+
+After running Test 1 or Test 2 (which generate RPC spans), check the
+xrpld debug.log for trace context:
+
+```bash
+grep 'trace_id=[a-f0-9]\{32\} span_id=[a-f0-9]\{16\}' /path/to/debug.log
+```
+
+Expected: log lines with `trace_id=<32hex> span_id=<16hex>` between the
+severity code and the message. Example:
+
+```
+2024-01-15T10:30:45.123Z RPCHandler:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Calling server_info
+```
+
+Lines emitted outside of an active span (background tasks, startup) will
+NOT have trace context — this is expected.
+
+### Step 2: Cross-check trace_id in Tempo
+
+Extract a `trace_id` from the log and verify it exists in Tempo:
+
+```bash
+TRACE_ID=$(grep -o 'trace_id=[a-f0-9]\{32\}' /path/to/debug.log | head -1 | cut -d= -f2)
+echo "Checking trace: $TRACE_ID"
+curl -s "http://localhost:3200/api/traces/$TRACE_ID" | jq '.batches | length'
+```
+
+Expected result: `> 0` (the trace exists in Tempo).
+
+### Step 3: Verify Loki log ingestion
+
+The OTel Collector's filelog receiver tails xrpld's debug.log and
+exports parsed entries to Loki. Verify Loki has received entries:
+
+```bash
+# Query Loki for any xrpld logs
+curl -sG "http://localhost:3100/loki/api/v1/query" \
+    --data-urlencode 'query={job="xrpld"}' \
+    --data-urlencode 'limit=5' | jq '.data.result | length'
+```
+
+Expected: > 0 results.
+
+### Step 4: Verify Grafana Tempo-to-Loki correlation
+
+1. Open Grafana at http://localhost:3000
+2. Navigate to **Explore** -> select **Tempo** datasource
+3. Search for a trace (e.g., operation `rpc.command.server_info`)
+4. Click **"Logs for this trace"** in the trace detail view
+5. Verify that Loki log lines appear, filtered by the trace's `trace_id`
+
+### Step 5: Verify Grafana Loki-to-Tempo correlation
+
+1. In Grafana **Explore**, select **Loki** datasource
+2. Query: `{job="xrpld"} |= "trace_id="`
+3. In the log results, click the **TraceID** derived field link
+4. Verify it navigates to the full trace in Tempo
+
+### Expected results
+
+| Check                          | Expected                                 |
+| ------------------------------ | ---------------------------------------- |
+| `trace_id=` in debug.log       | Present in log lines within active spans |
+| `span_id=` in debug.log        | Present alongside trace_id               |
+| Logs without active span       | No trace_id/span_id fields               |
+| trace_id in Tempo              | Matches a valid trace                    |
+| Loki log ingestion             | Logs visible via LogQL                   |
+| Tempo -> Loki "Logs for trace" | Shows correlated log lines               |
+| Loki -> Tempo TraceID link     | Navigates to correct trace               |
+
+---
+
+## Troubleshooting
+
+### No traces in Tempo
+
+1. Check otel-collector logs:
+   ```bash
+   docker compose -f docker/telemetry/docker-compose.yml logs otel-collector
+   ```
+2. Verify xrpld telemetry config has `enabled=1` and correct endpoint
+3. Check that otel-collector port 4318 is accessible:
+   ```bash
+   curl -sf http://localhost:4318 && echo "reachable"
+   ```
+4. Increase `batch_delay_ms` or decrease `batch_size` in xrpld config
+
+### Nodes not reaching "proposing" state
+
+1. Check that all peer ports (51235-51240) are not in use:
+   ```bash
+   for p in 51235 51236 51237 51238 51239 51240; do
+       ss -tlnp | grep ":$p " && echo "port $p in use"
+   done
+   ```
+2. Verify `[ips_fixed]` lists all 6 peer ports
+3. Verify `validators.txt` has all 6 public keys
+4. Check node debug logs: `tail -50 /tmp/xrpld-integration/node1/debug.log`
+5. Ensure `[peer_private]` is set to `1` (prevents reaching out to public network)
+
+### Transaction not processing
+
+1. Verify genesis account exists:
+   ```bash
+   curl -s http://localhost:5005 \
+       -d '{"method":"account_info","params":[{"account":"rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"}]}' |
+       jq .result.account_data.Balance
+   ```
+2. Check submit response for error codes
+3. In standalone mode, remember to call `ledger_accept` after submitting
+
+### No trace_id in log output (Phase 8)
+
+1. Verify xrpld was built with `telemetry=ON` (`-Dtelemetry=ON` in CMake)
+2. Verify `enabled=1` in the `[telemetry]` config section
+3. Log lines only contain trace context when emitted inside an active span.
+   Background logs (startup, periodic tasks outside spans) will not have
+   `trace_id`/`span_id`.
+4. Ensure the trace category is enabled (e.g., `trace_rpc=1` for RPC logs)
+
+### No logs in Loki (Phase 8)
+
+1. Verify the log file mount in docker-compose.yml:
+   ```yaml
+   volumes:
+     - /tmp/xrpld-integration:/var/log/rippled:ro
+   ```
+2. Check OTel Collector logs for filelog receiver errors:
+   ```bash
+   docker compose -f docker/telemetry/docker-compose.yml logs otel-collector | grep -i "filelog\|loki\|error"
+   ```
+3. Verify Loki is running:
+   ```bash
+   curl -s http://localhost:3100/ready
+   ```
+4. Verify the filelog receiver glob pattern matches your log files:
+   The default pattern is `/var/log/rippled/*/debug.log`
+
+### Grafana trace-log links not working (Phase 8)
+
+1. Verify `tracesToLogs` is configured in the Tempo datasource provisioning
+   (`docker/telemetry/grafana/provisioning/datasources/tempo.yaml`)
+2. Verify `derivedFields` is configured in the Loki datasource provisioning
+   (`docker/telemetry/grafana/provisioning/datasources/loki.yaml`)
+3. Restart Grafana after changing provisioning files:
+   ```bash
+   docker compose -f docker/telemetry/docker-compose.yml restart grafana
+   ```
+
+### Spanmetrics not appearing in Prometheus
+
+1. Verify otel-collector config has `spanmetrics` connector
+2. Check that the metrics pipeline is configured:
+   ```yaml
+   service:
+     pipelines:
+       metrics:
+         receivers: [spanmetrics]
+         exporters: [prometheus]
+   ```
+3. Verify Prometheus can reach collector:
+   ```bash
+   curl -s http://localhost:9090/api/v1/targets | jq '.data.activeTargets'
+   ```

docker/telemetry/docker-compose.workload.yaml

@@ -0,0 +1,101 @@
+# Docker Compose workload harness for Phase 10 telemetry validation.
+#
+# Runs a 5-node validator cluster with full OTel telemetry stack:
+#   - 5 rippled validator nodes (consensus network)
+#   - OTel Collector (traces + native OTLP metrics)
+#   - Tempo (trace backend + search API)
+#   - Prometheus (metrics)
+#   - Loki (log aggregation for log-trace correlation)
+#   - Grafana (dashboards + trace/log exploration)
+#
+# Usage:
+#   # Start the harness (requires pre-built xrpld image or mount binary):
+#   docker compose -f docker/telemetry/docker-compose.workload.yaml up -d
+#
+#   # Or use the orchestrator:
+#   docker/telemetry/workload/run-full-validation.sh
+#
+# Prerequisites:
+#   - xrpld binary built with -DXRPL_ENABLE_TELEMETRY=ON
+#   - Validator keys generated via generate-validator-keys.sh
+#   - Node configs generated by run-full-validation.sh
+#
+# Note: No Docker healthchecks are defined here. The orchestrator script
+# (run-full-validation.sh) polls each service endpoint directly from the
+# host, which avoids issues with missing curl/wget in container images.
+
+services:
+  # ---------------------------------------------------------------------------
+  # Telemetry Backend Stack
+  # ---------------------------------------------------------------------------
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    ports:
+      - "4317:4317" # OTLP gRPC
+      - "4318:4318" # OTLP HTTP (traces + beast::insight metrics)
+      - "8889:8889" # Prometheus metrics endpoint
+      - "13133:13133" # Health check
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
+      # Mount the validation workdir so filelog receiver can tail node logs.
+      - /tmp/xrpld-validation:/var/log/rippled:ro
+    depends_on:
+      - tempo
+    networks:
+      - workload-net
+
+  tempo:
+    image: grafana/tempo:2.7.2
+    command: ["-config.file=/etc/tempo.yaml"]
+    ports:
+      - "3200:3200" # Tempo HTTP API
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml:ro
+      - tempo-data:/var/tempo
+    networks:
+      - workload-net
+
+  prometheus:
+    image: prom/prometheus:latest
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+    depends_on:
+      - otel-collector
+    networks:
+      - workload-net
+
+  loki:
+    image: grafana/loki:3.4.2
+    ports:
+      - "3100:3100" # Loki HTTP API
+    command: ["-config.file=/etc/loki/local-config.yaml"]
+    networks:
+      - workload-net
+
+  grafana:
+    image: grafana/grafana:latest
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    depends_on:
+      - tempo
+      - prometheus
+      - loki
+    networks:
+      - workload-net
+
+volumes:
+  tempo-data:
+
+networks:
+  workload-net:
+    driver: bridge

docker/telemetry/docker-compose.yml

@@ -2,12 +2,15 @@
 #
 # Provides services for local development:
 #   - otel-collector: receives OTLP traces from xrpld, batches and
-#     forwards them to Tempo. Listens on ports 4317 (gRPC)
-#     and 4318 (HTTP).
+#     forwards them to Tempo. Also tails xrpld log files
+#     via filelog receiver and exports to Loki. Listens on ports
+#     4317 (gRPC) and 4318 (HTTP).
 #   - tempo: Grafana Tempo tracing backend, queryable via Grafana Explore
 #     on port 3000. Recommended for production (S3/GCS storage, TraceQL).
-#   - grafana: dashboards on port 3000, pre-configured with Tempo
-#     datasource.
+#   - loki: Grafana Loki log aggregation backend for centralized log
+#     ingestion and log-trace correlation (Phase 8).
+#   - grafana: dashboards on port 3000, pre-configured with Tempo,
+#     Prometheus, and Loki datasources.
 #
 # Usage:
 #   docker compose -f docker/telemetry/docker-compose.yml up -d
@@ -24,14 +27,24 @@ services:
     image: otel/opentelemetry-collector-contrib:0.121.0
     command: ["--config=/etc/otel-collector-config.yaml"]
     ports:
-      - "4317:4317" # OTLP gRPC receiver
-      - "4318:4318" # OTLP HTTP receiver (xrpld sends traces here)
-      - "13133:13133" # Health check endpoint
+      - "4317:4317" # OTLP gRPC
+      - "4318:4318" # OTLP HTTP (traces + native OTel metrics)
+      - "8889:8889" # Prometheus metrics (spanmetrics + OTLP)
+      # StatsD UDP port removed — beast::insight now uses native OTLP.
+      # Uncomment if using server=statsd fallback:
+      # - "8125:8125/udp"
     volumes:
       # Mount collector pipeline config (receivers → processors → exporters)
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
+      # Phase 8: Mount rippled log directories for filelog receiver.
+      # User-run xrpld writes to /home/pratik/xrpld-logs/<network>/debug.log.
+      # Integration test still writes to /tmp/xrpld-integration/.
+      # Both are mounted read-only so the collector can tail debug.log files.
+      - /home/pratik/xrpld-logs:/var/log/rippled:ro
+      - /tmp/xrpld-integration:/var/log/rippled-integration:ro
     depends_on:
       - tempo
+      - loki
     networks:
       - xrpld-telemetry
 
@@ -50,6 +63,32 @@ services:
     networks:
       - xrpld-telemetry
 
+  # Phase 8: Grafana Loki for centralized log ingestion and log-trace
+  # correlation. Loki 3.x supports native OTLP ingestion, so the OTel
+  # Collector exports via otlphttp to Loki's /otlp endpoint.
+  # Query logs via Grafana Explore -> Loki at http://localhost:3000.
+  loki:
+    image: grafana/loki:3.4.2
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+    volumes:
+      - loki-data:/loki
+    networks:
+      - xrpld-telemetry
+
+  prometheus:
+    image: prom/prometheus:latest
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    depends_on:
+      - otel-collector
+    networks:
+      - xrpld-telemetry
+
   # Grafana: visualization UI with Tempo pre-configured as a datasource.
   # Anonymous admin access enabled for local development convenience.
   grafana:
@@ -57,21 +96,40 @@ services:
     environment:
       - GF_AUTH_ANONYMOUS_ENABLED=true # No login required for local dev
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin # Full access without auth
+      # Remote image rendering: point Grafana at the renderer container.
+      # These belong on the grafana service (the server delegates renders);
+      # the callback URL is how the renderer fetches the panel from grafana.
+      - GF_RENDERING_SERVER_URL=http://renderer:8081/render
+      - GF_RENDERING_CALLBACK_URL=http://grafana:3000/
     ports:
       - "3000:3000" # Grafana web UI
     volumes:
       # Auto-provision Tempo datasource and search filters on startup
       - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
     depends_on:
       - tempo
+      - prometheus
+      - loki
+      - renderer
     networks:
       - xrpld-telemetry
 
+  # Grafana image renderer: a sidecar that renders panels/dashboards to PNG
+  # for image export and alerting. Grafana calls it at http://renderer:8081.
+  renderer:
+    image: grafana/grafana-image-renderer:latest
+    ports:
+      - "8081:8081" # Renderer HTTP endpoint (called by grafana)
+    networks:
+      - xrpld-telemetry
 # Named volume for Tempo trace storage (WAL and compacted blocks).
 # Data persists across container restarts. Remove with:
 #   docker compose -f docker/telemetry/docker-compose.yml down -v
 volumes:
   tempo-data:
+  prometheus-data:
+  loki-data:
 
 # Isolated bridge network so services communicate by container name
 # (e.g., the collector reaches Tempo at http://tempo:4317).

docker/telemetry/grafana/dashboards/ledger-operations.json

@@ -0,0 +1,340 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Ledger Build Rate",
+      "description": "Rate at which new ledgers are being built. The ledger.build span (BuildLedger.cpp) wraps the entire buildLedgerImpl() function which creates a new ledger from a parent, applies transactions, flushes SHAMap nodes, and sets the accepted state. Should match the consensus close rate (~0.25/sec on mainnet with ~4s rounds).",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"ledger.build\"}[5m]))",
+          "legendFormat": "Builds / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Ledger Build Duration",
+      "description": "p95 duration of ledger builds. Measures the full buildLedgerImpl() call including transaction application, SHAMap flushing, and ledger acceptance. The span records xrpl.ledger.seq as an attribute. Long build times indicate expensive transaction sets or I/O pressure from SHAMap flushes.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=\"ledger.build\"}[5m])))",
+          "legendFormat": "P95 Build Duration [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Ledger Validation Rate",
+      "description": "Rate at which ledgers pass the validation threshold and are accepted as fully validated. The ledger.validate span (LedgerMaster.cpp) fires in checkAccept() only after the ledger receives sufficient trusted validations (>= quorum). Records xrpl.ledger.seq and validations (the number of validations received).",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"ledger.validate\"}[5m]))",
+          "legendFormat": "Validations / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Ledger Build Duration Heatmap",
+      "description": "Heatmap showing the distribution of ledger.build durations across histogram buckets over time. Each cell represents the count of ledger builds that fell into that duration bucket in a 5m window. Useful for spotting occasional slow ledger builds that may not appear in percentile charts.",
+      "type": "heatmap",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "yAxis": {
+          "axisLabel": "Duration (ms)"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum(increase(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=\"ledger.build\"}[5m])) by (le)",
+          "legendFormat": "{{le}}",
+          "format": "heatmap"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Transaction Apply Duration",
+      "description": "p95 duration of applying the consensus transaction set during ledger building. The tx.apply span (BuildLedger.cpp) wraps applyTransactions() which iterates through the CanonicalTXSet with multiple retry passes. Records tx_count (successful) and tx_failed (failed) as attributes.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=\"tx.apply\"}[5m])))",
+          "legendFormat": "P95 tx.apply [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Transaction Apply Rate",
+      "description": "Rate of tx.apply span invocations, reflecting how frequently the transaction application phase runs during ledger building. Each ledger build triggers one tx.apply call. Should closely match the ledger build rate.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"tx.apply\"}[5m]))",
+          "legendFormat": "tx.apply / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Ledger Store Rate",
+      "description": "Rate at which ledgers are stored into the ledger history. The ledger.store span (LedgerMaster.cpp) wraps storeLedger() which inserts the ledger into the LedgerHistory cache. Records xrpl.ledger.seq. Should match the ledger build rate under normal operation.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"ledger.store\"}[5m]))",
+          "legendFormat": "Stores / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Build vs Close Duration",
+      "description": "Compares p95 durations of ledger.build (the actual ledger construction in BuildLedger.cpp) vs consensus.ledger_close (the consensus close event in RCLConsensus.cpp). Build time is a subset of close time. A large gap between them indicates overhead in the consensus pipeline outside of ledger construction itself.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=\"ledger.build\"}[5m])))",
+          "legendFormat": "P95 ledger.build [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=\"consensus.ledger_close\"}[5m])))",
+          "legendFormat": "P95 consensus.ledger_close [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "ledger"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)",
+        "type": "query",
+        "query": "label_values(traces_span_metrics_calls_total, exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Ledger Operations",
+  "uid": "xrpld-ledger-ops",
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/peer-network.json

@@ -0,0 +1,320 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "Requires trace_peer=1 in the [telemetry] config section.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Peer Proposal Receive Rate",
+      "description": "Rate of consensus proposals received from network peers. The peer.proposal.receive span (PeerImp.cpp) fires in onMessage(TMProposeSet) for each incoming proposal. Records xrpl.peer.id (sending peer) and proposal_trusted (whether the proposer is in our UNL). Requires trace_peer=1 in the telemetry config.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"peer.proposal.receive\"}[5m]))",
+          "legendFormat": "Proposals Received / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Proposals / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Peer Validation Receive Rate",
+      "description": "Rate of ledger validations received from network peers. The peer.validation.receive span (PeerImp.cpp) fires in onMessage(TMValidation) for each incoming validation message. Records xrpl.peer.id (sending peer) and validation_trusted (whether the validator is trusted). Requires trace_peer=1 in the telemetry config.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"peer.validation.receive\"}[5m]))",
+          "legendFormat": "Validations Received / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Validations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Proposals Trusted vs Untrusted",
+      "description": "Pie chart showing the ratio of proposals received from trusted validators (in our UNL) vs untrusted validators. Grouped by the proposal_trusted span attribute (true/false). A healthy node connected to a well-configured UNL should see a significant portion of trusted proposals. Note: proposals that fail early validation may not have the trusted attribute set.",
+      "type": "piechart",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (proposal_trusted, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[5m]))",
+          "legendFormat": "Trusted = {{proposal_trusted}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Validations Trusted vs Untrusted",
+      "description": "Pie chart showing the ratio of validations received from trusted validators (in our UNL) vs untrusted validators. Grouped by the validation_trusted span attribute (true/false). Monitoring this helps detect if the node is receiving validations from the expected set of trusted validators. Note: validations that fail early checks may not have the trusted attribute set.",
+      "type": "piechart",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (validation_trusted, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[5m]))",
+          "legendFormat": "Trusted = {{validation_trusted}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Reduce-Relay Peer Selection",
+      "description": "Transaction reduce-relay efficiency: peers selected as relay sources vs suppressed, plus peers with the feature disabled. A high suppressed:selected ratio proves reduce-relay is saving bandwidth; a high not_enabled count means stale peers force full relay.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_reduce_relay_metrics{metric=\"selected_peers\",exported_instance=~\"$node\"}",
+          "legendFormat": "Selected [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_reduce_relay_metrics{metric=\"suppressed_peers\",exported_instance=~\"$node\"}",
+          "legendFormat": "Suppressed [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_reduce_relay_metrics{metric=\"not_enabled_peers\",exported_instance=~\"$node\"}",
+          "legendFormat": "Not Enabled [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Peer Count",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Reduce-Relay Missing-Tx Frequency",
+      "description": "Frequency of on-demand transaction fetches triggered when a peer is missing a relayed transaction. A rising value means the suppression is too aggressive and the on-demand fetch path is growing.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_reduce_relay_metrics{metric=\"missing_tx_freq\",exported_instance=~\"$node\"}",
+          "legendFormat": "Missing Tx Freq [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Frequency",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "peer"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)",
+        "type": "query",
+        "query": "label_values(traces_span_metrics_calls_total, exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "proposal_trusted",
+        "label": "Proposal Trusted",
+        "description": "Filter by proposal trust status (true = from trusted validator)",
+        "type": "query",
+        "query": "label_values(traces_span_metrics_calls_total{span_name=\"peer.proposal.receive\"}, proposal_trusted)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "validation_trusted",
+        "label": "Validation Trusted",
+        "description": "Filter by validation trust status (true = from trusted validator)",
+        "type": "query",
+        "query": "label_values(traces_span_metrics_calls_total{span_name=\"peer.validation.receive\"}, validation_trusted)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Peer Network",
+  "uid": "xrpld-peer-net",
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/rpc-performance.json

@@ -0,0 +1,473 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "RPC Request Rate by Command",
+      "description": "Per-second rate of RPC command executions, broken down by command name (e.g. server_info, submit). Calculated as rate(traces_span_metrics_calls_total{span_name=~\"rpc.command.*\"}) over a 5m window, grouped by the command span attribute.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (command, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m]))",
+          "legendFormat": "{{command}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "custom": {
+            "axisLabel": "Requests / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Latency P95 by Command",
+      "description": "95th percentile response time for each RPC command. Computed from the spanmetrics duration histogram using histogram_quantile(0.95) over rpc.command.* spans, grouped by command. High values indicate slow commands that may need optimization.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, command, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])))",
+          "legendFormat": "P95 {{command}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Latency (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Error Rate",
+      "description": "Percentage of RPC commands that completed with an error status, per command. Calculated as (error calls / total calls) * 100, where errors have status_code=STATUS_CODE_ERROR. Thresholds: green < 1%, yellow 1-5%, red > 5%.",
+      "type": "bargauge",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (command, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", command=~\"$command\", span_name=~\"rpc.command.*\", status_code=\"STATUS_CODE_ERROR\"}[5m])) / sum by (command, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])) * 100",
+          "legendFormat": "{{command}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 5
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Latency Heatmap",
+      "description": "Distribution of RPC command response times across histogram buckets. Shows the density of requests at each latency level over time. Each cell represents the count of requests that fell into that duration bucket in a 5m window. Useful for spotting bimodal latency patterns.",
+      "type": "heatmap",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "yAxis": {
+          "axisLabel": "Duration (ms)",
+          "unit": "ms"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum(increase(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])) by (le)",
+          "legendFormat": "{{le}}",
+          "format": "heatmap"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        }
+      }
+    },
+    {
+      "title": "Overall RPC Throughput",
+      "description": "Aggregate RPC throughput showing two layers of the request pipeline. rpc.http_request is the outer HTTP handler (ServerHandler.cpp) that accepts incoming connections. rpc.process is the inner processing layer (ServerHandler.cpp) that parses and dispatches. A gap between the two indicates requests being queued or rejected before processing.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", command=~\"$command\", span_name=\"rpc.http_request\"}[5m]))",
+          "legendFormat": "rpc.http_request / Sec [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", command=~\"$command\", span_name=\"rpc.process\"}[5m]))",
+          "legendFormat": "rpc.process / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "custom": {
+            "axisLabel": "Requests / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Success vs Error",
+      "description": "Aggregate rate of successful vs failed RPC commands across all command types. Success = status_code UNSET (OpenTelemetry default for OK spans). Error = status_code STATUS_CODE_ERROR. A sustained error rate warrants investigation via per-command breakdown above.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", command=~\"$command\", span_name=~\"rpc.command.*\", status_code=\"STATUS_CODE_UNSET\"}[5m]))",
+          "legendFormat": "Success [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", command=~\"$command\", span_name=~\"rpc.command.*\", status_code=\"STATUS_CODE_ERROR\"}[5m]))",
+          "legendFormat": "Error [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Commands / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Top Commands by Volume",
+      "description": "Top 10 most frequently called RPC commands by total invocation count over the last 5 minutes. Uses topk(10, increase(calls_total)) to rank commands. Helps identify the hottest API endpoints driving load on the node.",
+      "type": "bargauge",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "topk(10, sum by (command, exported_instance) (increase(traces_span_metrics_calls_total{exported_instance=~\"$node\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])))",
+          "legendFormat": "{{command}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "WebSocket Message Rate",
+      "description": "Rate of incoming WebSocket RPC messages processed by the server. Sourced from the rpc.ws_message span (ServerHandler.cpp). Only active when clients connect via WebSocket instead of HTTP. Zero is normal if only HTTP RPC is in use.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", command=~\"$command\", span_name=\"rpc.ws_message\"}[5m]))",
+          "legendFormat": "WS Messages / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Resource Cost by Command",
+      "description": "RPC commands grouped by load_type (resource cost category). High-cost categories like exception_rpc or malformed_rpc indicate problematic clients.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "lastNotNull"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (load_type) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=~\"rpc.command.*\", load_type!=\"\"}[5m]))",
+          "legendFormat": "{{load_type}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Requests / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Batch vs Single RPC Requests",
+      "description": "Rate of batch RPC requests vs single requests. High batch rate may indicate bulk automation clients.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"rpc.process\", is_batch=\"true\"}[5m]))",
+          "legendFormat": "Batch [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"rpc.process\", is_batch=\"false\"}[5m]))",
+          "legendFormat": "Single [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Requests / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "rpc"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)",
+        "type": "query",
+        "query": "label_values(traces_span_metrics_calls_total, exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "command",
+        "label": "RPC Command",
+        "description": "Filter by RPC command name (e.g., server_info, submit)",
+        "type": "query",
+        "query": "label_values(traces_span_metrics_calls_total{span_name=~\"rpc.command.*\"}, command)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "RPC Performance",
+  "uid": "xrpld-rpc-perf",
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/system-ledger-data-sync.json

@@ -0,0 +1,528 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "Ledger data exchange and object fetch traffic from beast::insight System Metrics. Covers ledger sync, node data retrieval, and transaction set exchange. Requires [insight] server=otel in rippled config.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Ledger Data Exchange (Bytes In)",
+      "description": "Inbound bytes for ledger data sub-categories. 'ledger_data' = aggregated ledger data, sub-types include Transaction_Set_candidate (proposed tx sets), Transaction_Node (tx tree nodes), and Account_State_Node (state tree nodes). High Account_State_Node traffic indicates state sync; high Transaction_Set_candidate indicates consensus catch-up. Sourced from TrafficCount.h ledger_data_* categories.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_data_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Ledger Data Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_data_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Ledger Data Share [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_data_Transaction_Set_candidate_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Set Candidate Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_data_Transaction_Set_candidate_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Set Candidate Share [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_data_Transaction_Node_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Node Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_data_Transaction_Node_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Node Share [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_data_Account_State_Node_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Account State Node Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_data_Account_State_Node_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Account State Node Share [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes In",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Ledger Share/Get Traffic (Bytes)",
+      "description": "Legacy ledger share and get traffic by sub-type. These are the older ledger fetch protocol categories (as opposed to ledger_data_* which is the newer protocol). Sub-types: Transaction_Set_candidate, Transaction_node, Account_State_node, plus aggregate ledger_share and ledger_get. Sourced from TrafficCount.h ledger_* categories.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Ledger Share In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Ledger Get In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_Transaction_Set_candidate_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Set Candidate Share [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_Transaction_Set_candidate_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Set Candidate Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_Transaction_node_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Node Share [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_Transaction_node_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Node Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_Account_State_node_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Account State Share [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_ledger_Account_State_node_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Account State Get [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes In",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "GetObject Traffic by Type (Bytes In)",
+      "description": "Object fetch traffic by object type. GetObject is the protocol for fetching specific SHAMap nodes. Types: Ledger (full ledger headers), Transaction (individual txs), Transaction_node (tx tree nodes), Account_State_node (state tree nodes), CAS (Content Addressable Storage objects), Fetch_Pack (batch fetch during catch-up), Transactions (bulk tx fetch). High Fetch_Pack traffic indicates a node is catching up. Sourced from TrafficCount.h getobject_* categories.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Ledger_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Ledger Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Ledger_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Ledger Share [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Transaction_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Transaction Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Transaction_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Transaction Share [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Transaction_node_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Node Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Transaction_node_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Node Share [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Account_State_node_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Account State Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Account_State_node_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Account State Share [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes In",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "GetObject Aggregate & Special Types (Bytes In)",
+      "description": "Aggregate getobject traffic plus special categories: CAS (Content Addressable Storage) for SHAMap node fetch, Fetch_Pack for bulk batch downloads during catch-up, Transactions for bulk tx fetch, and the aggregate getobject_get/getobject_share totals. Sourced from TrafficCount.h getobject_* categories.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_CAS_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "CAS Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_CAS_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "CAS Share [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Fetch_Pack_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Fetch Pack Share [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Fetch_Pack_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Fetch Pack Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Transactions_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Transactions Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Aggregate Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Aggregate Share [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes In",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "GetObject Messages by Type",
+      "description": "Message counts for object fetch operations. Shows how many individual fetch requests and responses are exchanged per type. High message counts with low byte counts indicate small object fetches; the inverse indicates large batch transfers. Sourced from TrafficCount.h getobject_* categories.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Ledger_get_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Ledger Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Transaction_get_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Transaction Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Transaction_node_get_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Node Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Account_State_node_get_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Account State Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_CAS_get_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "CAS Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Fetch_Pack_get_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Fetch Pack Get [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_getobject_Transactions_get_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Transactions Get [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages In",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Overlay Traffic Heatmap (All Categories, Bytes In)",
+      "description": "Bar gauge showing all overlay traffic categories ranked by inbound bytes. Provides a complete at-a-glance view of which protocol message types consume the most bandwidth across all 57+ traffic categories. Sourced from all TrafficCount.h categories via wildcard match.",
+      "type": "bargauge",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "topk(20, {exported_instance=~\"$node\", __name__=~\"xrpld_.*_Bytes_In\", __name__!~\"xrpld_total_.*\"})",
+          "legendFormat": "{{__name__}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1048576
+              },
+              {
+                "color": "red",
+                "value": 104857600
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "statsd", "ledger", "sync"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(xrpld_ledger_data_get_Bytes_In, exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Ledger Data & Sync (System Metrics)",
+  "uid": "xrpld-system-ledger-sync",
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/system-network-traffic.json

@@ -0,0 +1,806 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "Network traffic and peer metrics from beast::insight System Metrics. Requires [insight] server=otel in rippled config.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Active Peers",
+      "description": "Number of active inbound and outbound peer connections. Sourced from Peer_Finder.Active_Inbound_Peers and Peer_Finder.Active_Outbound_Peers gauges (PeerfinderManager.cpp). A healthy mainnet node typically has 10-21 outbound and 0-85 inbound peers depending on configuration.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_Peer_Finder_Active_Inbound_Peers{exported_instance=~\"$node\"}",
+          "legendFormat": "Inbound Peers [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_Peer_Finder_Active_Outbound_Peers{exported_instance=~\"$node\"}",
+          "legendFormat": "Outbound Peers [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Peers",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Peer Disconnects",
+      "description": "Cumulative count of peer disconnections. Sourced from the Overlay.Peer_Disconnects gauge (OverlayImpl.h). A rising trend indicates network instability, aggressive peer management, or resource exhaustion causing connection drops.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_Overlay_Peer_Disconnects{exported_instance=~\"$node\"}",
+          "legendFormat": "Disconnects [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Disconnects",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Total Network Bytes",
+      "description": "Total bytes sent and received across all peer connections. Sourced from the total.Bytes_In and total.Bytes_Out traffic category gauges (OverlayImpl.h). Provides a high-level view of network bandwidth consumption.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_total_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Bytes In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_total_Bytes_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Bytes Out [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Total Network Messages",
+      "description": "Total messages sent and received across all peer connections. Sourced from the total.Messages_In and total.Messages_Out traffic category gauges (OverlayImpl.h). Shows the overall message throughput of the overlay network.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_total_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Messages In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_total_Messages_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Messages Out [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Transaction Traffic",
+      "description": "Bytes and messages for transaction-related overlay traffic. Includes the transactions traffic category (OverlayImpl/TrafficCount.h). Spikes indicate high transaction volume on the network or transaction flooding.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_transactions_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Messages In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_transactions_Messages_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Messages Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_transactions_duplicate_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "TX Duplicate In [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Proposal Traffic",
+      "description": "Messages for consensus proposal overlay traffic. Includes proposals, proposals_untrusted, and proposals_duplicate categories (TrafficCount.h). High untrusted or duplicate counts may indicate UNL misconfiguration or network spam.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_proposals_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Proposals In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_proposals_Messages_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Proposals Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_proposals_untrusted_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Untrusted In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_proposals_duplicate_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Duplicate In [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Validation Traffic",
+      "description": "Messages for validation overlay traffic. Includes validations, validations_untrusted, and validations_duplicate categories (TrafficCount.h). Monitoring trusted vs untrusted validation traffic helps detect UNL health issues.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validations_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Validations In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validations_Messages_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Validations Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validations_untrusted_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Untrusted In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validations_duplicate_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Duplicate In [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Overlay Traffic by Category (Bytes In)",
+      "description": "Top traffic categories by inbound bytes. Includes all 57 overlay traffic categories from TrafficCount.h. Shows which protocol message types consume the most bandwidth. Categories include transactions, proposals, validations, ledger data, getobject, and overlay overhead.",
+      "type": "bargauge",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "topk(10, {exported_instance=~\"$node\", __name__=~\"xrpld_.*_Bytes_In\", __name__!~\"xrpld_total_.*\"})",
+          "legendFormat": "{{__name__}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_transactions_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Transactions"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_proposals_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Proposals"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_validations_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Validations"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_overhead_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Overhead"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_overhead_overlay_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Overhead Overlay"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ping_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Ping"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_status_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Status"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_getObject_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Get Object"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_haveTxSet_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Have Tx Set"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ledgerData_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Ledger Data"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ledger_share_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Ledger Share"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ledger_data_get_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Ledger Data Get"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ledger_data_share_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Ledger Data Share"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ledger_data_Account_State_Node_get_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Account State Node Get"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ledger_data_Account_State_Node_share_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Account State Node Share"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ledger_data_Transaction_Node_get_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Transaction Node Get"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ledger_data_Transaction_Node_share_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Transaction Node Share"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ledger_data_Transaction_Set_candidate_get_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Tx Set Candidate Get"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ledger_Account_State_node_share_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Account State Node Share (Legacy)"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ledger_Transaction_Set_candidate_share_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Tx Set Candidate Share"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_ledger_Transaction_node_share_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Transaction Node Share (Legacy)"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "xrpld_set_get_Bytes_In"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Set Get"
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "title": "Duplicate Traffic (Wasted Bandwidth)",
+      "description": "Rate of duplicate overlay traffic across transaction, proposal, and validation categories. Duplicate messages are messages the node has already seen and discards. High duplicate rates indicate inefficient message routing or network topology issues causing redundant relays.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_transactions_duplicate_Bytes_In{exported_instance=~\"$node\"}[5m])",
+          "legendFormat": "TX Duplicate In"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_transactions_duplicate_Bytes_Out{exported_instance=~\"$node\"}[5m])",
+          "legendFormat": "TX Duplicate Out"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_proposals_duplicate_Bytes_In{exported_instance=~\"$node\"}[5m])",
+          "legendFormat": "Proposals Duplicate In"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_proposals_duplicate_Bytes_Out{exported_instance=~\"$node\"}[5m])",
+          "legendFormat": "Proposals Duplicate Out"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_validations_duplicate_Bytes_In{exported_instance=~\"$node\"}[5m])",
+          "legendFormat": "Validations Duplicate In"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_validations_duplicate_Bytes_Out{exported_instance=~\"$node\"}[5m])",
+          "legendFormat": "Validations Duplicate Out"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps",
+          "custom": {
+            "axisLabel": "Throughput",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "All Traffic Categories (Detail)",
+      "description": "Top 15 traffic categories by inbound byte rate, excluding the total aggregate. Provides a detailed timeseries view of which overlay message types are consuming the most bandwidth over time. Complements the bar gauge snapshot view in the Overlay Traffic panel.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "topk(15, rate({__name__=~\"xrpld_.*_Bytes_In\", __name__!~\"xrpld_total_{exported_instance=~\"$node\"}.*\"}[5m]))",
+          "legendFormat": "{{__name__}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps",
+          "custom": {
+            "axisLabel": "Throughput",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "statsd", "network"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(xrpld_Peer_Finder_Active_Inbound_Peers, exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Network Traffic (System Metrics)",
+  "uid": "xrpld-system-network",
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json

@@ -0,0 +1,588 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "Detailed overlay traffic breakdown for categories not covered by the main Network Traffic dashboard. Includes squelch, overhead, validator lists, object fetch, ledger sync, and protocol negotiation traffic. Requires [insight] server=otel in rippled config.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Squelch Traffic (Messages)",
+      "description": "Squelch-related overlay messages. Squelch is the peer traffic management protocol that suppresses redundant message forwarding. 'squelch' = squelch control messages, 'squelch_suppressed' = messages suppressed by squelch, 'squelch_ignored' = squelch directives that were ignored. High suppressed counts indicate effective bandwidth savings; high ignored counts may indicate misconfigured peers. Sourced from TrafficCount.h squelch categories.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_squelch_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Squelch In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_squelch_Messages_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Squelch Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_squelch_suppressed_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Suppressed In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_squelch_suppressed_Messages_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Suppressed Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_squelch_ignored_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Ignored In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_squelch_ignored_Messages_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Ignored Out [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Overhead Traffic Breakdown (Bytes)",
+      "description": "Overlay protocol overhead by sub-category. 'overhead' = base protocol overhead (ping, status, etc.), 'overhead_cluster' = intra-cluster communication overhead, 'overhead_manifest' = validator manifest distribution overhead. High cluster overhead may indicate frequent cluster state syncs; high manifest overhead occurs during UNL changes. Sourced from TrafficCount.h overhead categories.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_overhead_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Base Overhead In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_overhead_Bytes_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Base Overhead Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_overhead_cluster_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Cluster In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_overhead_cluster_Bytes_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Cluster Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_overhead_manifest_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Manifest In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_overhead_manifest_Bytes_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Manifest Out [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Validator List Traffic",
+      "description": "Validator list (UNL) distribution traffic. Validator lists are exchanged when peers share their trusted validator configurations. Spikes occur during UNL updates or when new peers connect. Sourced from TrafficCount.h validator_lists category.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validator_lists_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Bytes In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validator_lists_Bytes_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Bytes Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validator_lists_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Messages In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validator_lists_Messages_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Messages Out [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Count",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/Bytes/"
+            },
+            "properties": [
+              {
+                "id": "custom.axisPlacement",
+                "value": "right"
+              },
+              {
+                "id": "unit",
+                "value": "decbytes"
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "title": "Set Get/Share Traffic (Bytes)",
+      "description": "Transaction set get and share traffic. 'set_get' = requests to fetch transaction sets (sent during ledger close), 'set_share' = responses sharing transaction sets. High set_get traffic indicates peers frequently requesting missing transaction sets, which may signal sync delays. Sourced from TrafficCount.h set_get/set_share categories.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_set_get_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Set Get In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_set_get_Bytes_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Set Get Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_set_share_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Set Share In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_set_share_Bytes_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Set Share Out [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Have/Requested Transactions (Messages)",
+      "description": "Transaction availability protocol messages. 'have_transactions' = advertisements that a peer has specific transactions available, 'requested_transactions' = explicit requests for transaction data. A high ratio of requested to have may indicate peers are behind on transaction propagation. Sourced from TrafficCount.h have_transactions/requested_transactions categories.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_have_transactions_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Have TX In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_have_transactions_Messages_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Have TX Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_requested_transactions_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Requested TX In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_requested_transactions_Messages_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Requested TX Out [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Unknown / Unclassified Traffic",
+      "description": "Traffic that does not match any known overlay message category. Non-zero values may indicate protocol version mismatches, corrupted messages, or new message types not yet classified. Sourced from TrafficCount.h unknown category.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_unknown_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Unknown Bytes In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_unknown_Bytes_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Unknown Bytes Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_unknown_Messages_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Unknown Messages In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_unknown_Messages_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Unknown Messages Out [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Count",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/Bytes/"
+            },
+            "properties": [
+              {
+                "id": "custom.axisPlacement",
+                "value": "right"
+              },
+              {
+                "id": "unit",
+                "value": "decbytes"
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "title": "Proof Path Traffic",
+      "description": "Proof path request/response traffic for ledger state proof exchange. Used by peers to verify specific ledger entries without downloading the full ledger. High request volume may indicate peers validating state during catch-up. Sourced from TrafficCount.h proof_path_request/proof_path_response categories.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_proof_path_request_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Request Bytes In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_proof_path_request_Bytes_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Request Bytes Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_proof_path_response_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Response Bytes In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_proof_path_response_Bytes_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Response Bytes Out [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Replay Delta Traffic",
+      "description": "Replay delta request/response traffic for ledger replay protocol. Used during catch-up to efficiently replay ledger state changes. Sourced from TrafficCount.h replay_delta_request/replay_delta_response categories.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_replay_delta_request_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Request Bytes In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_replay_delta_request_Bytes_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Request Bytes Out [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_replay_delta_response_Bytes_In{exported_instance=~\"$node\"}",
+          "legendFormat": "Response Bytes In [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_replay_delta_response_Bytes_Out{exported_instance=~\"$node\"}",
+          "legendFormat": "Response Bytes Out [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "statsd", "overlay", "network"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(xrpld_squelch_Messages_In, exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Overlay Traffic Detail (System Metrics)",
+  "uid": "xrpld-system-overlay-detail",
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json

@@ -0,0 +1,605 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "RPC and pathfinding metrics from beast::insight System Metrics. Requires [insight] server=otel in rippled config.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "RPC Request Rate (System Metrics)",
+      "description": "Rate of RPC requests as counted by the beast::insight counter. Sourced from rpc.requests (ServerHandler.cpp) which increments on every HTTP and WebSocket RPC request. Compare with the span-based rpc.request rate in the RPC Performance dashboard for cross-validation.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_rpc_requests_total{exported_instance=~\"$node\"}[5m])",
+          "legendFormat": "Requests / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Response Time (System Metrics)",
+      "description": "P95 of RPC response time from the beast::insight timer. Sourced from the rpc.time event (ServerHandler.cpp) which records elapsed milliseconds for each RPC response. This measures the full HTTP handler time, not just command execution. Compare with span-based rpc.request duration.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, exported_instance) (rate(xrpld_rpc_time_milliseconds_bucket{exported_instance=~\"$node\"}[5m])))",
+          "legendFormat": "P95 Response Time [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Latency (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Response Size",
+      "description": "P95 of RPC response payload size in bytes. Sourced from the rpc.size event (ServerHandler.cpp) which records the byte length of each RPC JSON response. Large responses may indicate expensive queries (e.g. account_tx with many results) or API misuse.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, exported_instance) (rate(xrpld_rpc_size_milliseconds_bucket{exported_instance=~\"$node\"}[5m])))",
+          "legendFormat": "P95 Response Size [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Size (Bytes)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Response Time Distribution",
+      "description": "Distribution of RPC response times from the beast::insight timer showing P90, P95, and P99 quantiles. Sourced from the rpc.time event (ServerHandler.cpp). Useful for detecting bimodal latency or long-tail requests.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.9, sum by (le, exported_instance) (rate(xrpld_rpc_time_milliseconds_bucket{exported_instance=~\"$node\"}[5m])))",
+          "legendFormat": "P90 [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, exported_instance) (rate(xrpld_rpc_time_milliseconds_bucket{exported_instance=~\"$node\"}[5m])))",
+          "legendFormat": "P95 [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le, exported_instance) (rate(xrpld_rpc_time_milliseconds_bucket{exported_instance=~\"$node\"}[5m])))",
+          "legendFormat": "P99 [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Latency (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Pathfinding Fast Duration",
+      "description": "P95 of fast pathfinding execution time. Sourced from the pathfind_fast event (PathRequests.h) which records the duration of the fast pathfinding algorithm. Fast pathfinding uses a simplified search that trades accuracy for speed.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, exported_instance) (rate(xrpld_pathfind_fast_milliseconds_bucket{exported_instance=~\"$node\"}[5m])))",
+          "legendFormat": "P95 Fast Pathfind [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Pathfinding Full Duration",
+      "description": "P95 of full pathfinding execution time. Sourced from the pathfind_full event (PathRequests.h) which records the duration of the exhaustive pathfinding search. Full pathfinding is more expensive and can take significantly longer than fast mode.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, exported_instance) (rate(xrpld_pathfind_full_milliseconds_bucket{exported_instance=~\"$node\"}[5m])))",
+          "legendFormat": "P95 Full Pathfind [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Resource Warnings Rate",
+      "description": "Rate of resource warning events from the Resource Manager. Sourced from the warn meter (Logic.h) which increments when a consumer (peer or RPC client) exceeds the warning threshold for resource usage. A rising rate indicates aggressive clients that may need throttling. NOTE: This panel will show no data until the |m -> |c fix is applied in System MetricsCollector.cpp (Phase 6 Task 6.1).",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_warn_total{exported_instance=~\"$node\"}[5m])",
+          "legendFormat": "Warnings / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.1
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Resource Drops Rate",
+      "description": "Rate of resource drop events from the Resource Manager. Sourced from the drop meter (Logic.h) which increments when a consumer is disconnected or blocked due to excessive resource usage. Non-zero values mean the node is actively rejecting abusive connections. NOTE: This panel will show no data until the |m -> |c fix is applied in System MetricsCollector.cpp (Phase 6 Task 6.1).",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_drop_total{exported_instance=~\"$node\"}[5m])",
+          "legendFormat": "Drops / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.01
+              },
+              {
+                "color": "red",
+                "value": 0.1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "gRPC Request Rate by Method (Spans)",
+      "description": "Per-method gRPC call rate derived from the grpc.{Method} spans (GRPCServer.cpp). Covers the gRPC API used by reporting/Clio. Populated only when the node serves gRPC traffic.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (method, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", method=~\"$grpc_method\", span_name=~\"grpc\\\\..*\"}[5m]))",
+          "legendFormat": "{{method}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Calls / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "gRPC Latency P95 by Method (Spans)",
+      "description": "p95 latency per gRPC method from grpc.{Method} span durations. Identifies slow gRPC read paths.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, method, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", method=~\"$grpc_method\", span_name=~\"grpc\\\\..*\"}[5m])))",
+          "legendFormat": "{{method}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "gRPC Error Rate by Status (Spans)",
+      "description": "Rate of gRPC spans broken down by grpc_status (success/error/resource_exhausted/failed_precondition). A rising error or resource_exhausted rate indicates gRPC clients hitting limits.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 40
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (grpc_status, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=~\"grpc\\\\..*\", grpc_status!=\"\"}[5m]))",
+          "legendFormat": "{{grpc_status}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Calls / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Pathfinding Compute Duration (Spans)",
+      "description": "p95 of the pathfind.compute span, the per-request path computation. Complements the StatsD pathfind_fast/full timers with span-level visibility. Populated under pathfinding (book/path) RPC load.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 40
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=\"pathfind.compute\"}[5m])))",
+          "legendFormat": "P95 Compute [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Pathfinding Request & Discovery Rate (Spans)",
+      "description": "Rate of pathfind.request (client path requests) and pathfind.discover (path-discovery passes) spans. Shows pathfinding demand and the discovery cost driver for subscription-heavy nodes.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 48
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"pathfind.request\"}[5m]))",
+          "legendFormat": "Requests / Sec [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"pathfind.discover\"}[5m]))",
+          "legendFormat": "Discoveries / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "statsd", "rpc", "pathfinding"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(xrpld_rpc_requests_total, exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "grpc_method",
+        "label": "gRPC Method",
+        "description": "Filter by gRPC method (GetLedger, GetLedgerData, GetLedgerDiff, GetLedgerEntry)",
+        "type": "query",
+        "query": "label_values(traces_span_metrics_calls_total{span_name=~\"grpc\\\\..*\"}, method)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "RPC & Pathfinding (System Metrics)",
+  "uid": "xrpld-system-rpc",
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/transaction-overview.json

@@ -0,0 +1,994 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Transaction Apply Failed Rate",
+      "description": "Rate of tx.apply spans completing with error status, indicating transaction application failures during ledger building. The span records tx_failed as an attribute. Thresholds: green < 0.1/sec, yellow 0.1-1/sec, red > 1/sec. Some failures are normal (e.g. conflicting offers) but sustained high rates may indicate issues.",
+      "type": "stat",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"tx.apply\", status_code=\"STATUS_CODE_ERROR\"}[5m]))",
+          "legendFormat": "Failed / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.1
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "id": 1
+    },
+    {
+      "title": "Transaction Processing Latency by Type",
+      "description": "Per-transaction-type processing latency (p95). Filter with $tx_type variable above.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max", "lastNotNull"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, tx_type, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m])))",
+          "legendFormat": "{{tx_type}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Latency (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 2
+    },
+    {
+      "title": "Transaction Rate by Type",
+      "description": "Transaction processing rate broken down by tx_type (Payment, OfferCreate, AMMDeposit, etc.). Requires tx_type dimension in spanmetrics.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 12
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "lastNotNull"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (tx_type, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m]))",
+          "legendFormat": "{{tx_type}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "TX / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 3
+    },
+    {
+      "title": "Transaction Results by Type",
+      "description": "Transaction result codes (ter_result) broken down by tx_type. Shows which transaction types fail most often.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 20
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "lastNotNull"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (tx_type, ter_result, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"tx.process\", tx_type=~\"$tx_type\", ter_result=~\"$ter_result\", ter_result!=\"tesSUCCESS\"}[5m]))",
+          "legendFormat": "{{tx_type}} [{{ter_result}}, {{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Failed TX / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 4
+    },
+    {
+      "title": "Transaction Receive vs Suppressed",
+      "description": "Total rate of raw transaction messages received from peers (tx.receive span from PeerImp.cpp). This fires before deduplication via the HashRouter, so the difference between tx.receive and tx.process reflects suppressed duplicate transactions.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 28
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (suppressed, exported_instance) (rate(traces_span_metrics_calls_total{span_name=\"tx.receive\", tx_type=~\"$tx_type\", exported_instance=~\"$node\"}[$__rate_interval]))",
+          "legendFormat": "Suppressed [{{suppressed}}, {{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Transactions / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 5
+    },
+    {
+      "title": "Transaction Processing Rate",
+      "description": "Rate of transactions entering the processing pipeline. tx.process (NetworkOPs.cpp) fires when a transaction is submitted locally or received from a peer and enters processTransaction(). tx.receive (PeerImp.cpp) fires when a raw transaction message arrives from a peer before deduplication.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 28
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m]))",
+          "legendFormat": "tx.process / Sec [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"tx.receive\", tx_type=~\"$tx_type\"}[5m]))",
+          "legendFormat": "tx.receive / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Transactions / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 6
+    },
+    {
+      "title": "Transaction Path Distribution",
+      "description": "Breakdown of transactions by origin path. The local attribute indicates whether the transaction was submitted locally (true) or received from a peer (false). Helps understand the ratio of locally-originated vs relayed transactions.",
+      "type": "piechart",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 36
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (local, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", local=~\"$tx_origin\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m]))",
+          "legendFormat": "Local [{{local}}, {{exported_instance}}]"
+        }
+      ],
+      "id": 7
+    },
+    {
+      "title": "Transaction Processing Duration Heatmap",
+      "description": "Heatmap showing the distribution of tx.process span durations across histogram buckets over time. Each cell represents the count of transactions that completed within that latency bucket in a 5m window. Reveals whether processing times are consistent or exhibit multi-modal patterns.",
+      "type": "heatmap",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 36
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        },
+        "yAxis": {
+          "axisLabel": "Duration (ms)",
+          "unit": "ms"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum(increase(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m])) by (le)",
+          "legendFormat": "{{le}}",
+          "format": "heatmap"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        }
+      },
+      "id": 8
+    },
+    {
+      "title": "Peer Transaction Receive Rate",
+      "description": "Rate of transaction messages received from network peers. Sourced from the tx.receive span (PeerImp.cpp) which fires in the onMessage(TMTransaction) handler. High rates may indicate network-wide transaction volume spikes or peer flooding.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 44
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"tx.receive\", tx_type=~\"$tx_type\"}[5m]))",
+          "legendFormat": "tx.receive / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Transactions / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 9
+    },
+    {
+      "title": "Transactor Duration by Type (p95)",
+      "description": "Per-transactor execution time (tx.transactor span). Shows which transaction types are most expensive to execute.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 52
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, tx_type, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=\"tx.transactor\", tx_type=~\"$tx_type\"}[5m])))",
+          "legendFormat": "{{tx_type}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 10
+    },
+    {
+      "title": "Tx Apply Pipeline Rate by Stage",
+      "description": "Span rate for each apply-pipeline stage (preflight, preclaim, apply). A drop between stages shows where transactions are filtered out. Requires the stage dimension in spanmetrics.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 60
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "label_replace(label_replace(label_replace(sum by (stage, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", stage=~\"$stage\"}[5m])), \"stage\", \"Preflight\", \"stage\", \"preflight\"), \"stage\", \"Preclaim\", \"stage\", \"preclaim\"), \"stage\", \"Apply\", \"stage\", \"apply\")",
+          "legendFormat": "{{stage}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Spans / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 11
+    },
+    {
+      "title": "Tx Apply Pipeline Latency by Stage (p95)",
+      "description": "95th-percentile duration of each apply-pipeline stage. Isolates which stage (preflight, preclaim, apply) dominates transaction processing time.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 68
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "label_replace(label_replace(label_replace(histogram_quantile(0.95, sum by (le, stage, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", stage=~\"$stage\"}[5m]))), \"stage\", \"Preflight\", \"stage\", \"preflight\"), \"stage\", \"Preclaim\", \"stage\", \"preclaim\"), \"stage\", \"Apply\", \"stage\", \"apply\")",
+          "legendFormat": "{{stage}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 12
+    },
+    {
+      "title": "Tx Apply Pipeline Failure Rate by Stage",
+      "description": "Rate of apply-pipeline spans whose ter_result is not tesSUCCESS, split by stage. Shows whether failures concentrate in preflight, preclaim, or apply. Filters on ter_result rather than span status because a failing ter code completes the span normally; only thrown exceptions set an error status.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 76
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "label_replace(label_replace(label_replace(sum by (stage, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", stage=~\"$stage\", ter_result!~\"tesSUCCESS|\"}[5m])), \"stage\", \"Preflight\", \"stage\", \"preflight\"), \"stage\", \"Preclaim\", \"stage\", \"preclaim\"), \"stage\", \"Apply\", \"stage\", \"apply\")",
+          "legendFormat": "{{stage}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Failed Spans / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 13
+    },
+    {
+      "title": "Tx Apply Pipeline Latency by Type and Stage (p95)",
+      "description": "95th-percentile duration broken down by both transaction type and apply-pipeline stage. Shows, for each transaction type, which stage (preflight, preclaim, apply) dominates its latency. Higher cardinality than the by-stage view; filter with the $tx_type and $stage variables.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 84
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "label_replace(label_replace(label_replace(histogram_quantile(0.95, sum by (le, tx_type, stage, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", tx_type=~\"$tx_type\", stage=~\"$stage\"}[5m]))), \"stage\", \"Preflight\", \"stage\", \"preflight\"), \"stage\", \"Preclaim\", \"stage\", \"preclaim\"), \"stage\", \"Apply\", \"stage\", \"apply\")",
+          "legendFormat": "{{tx_type}} [{{stage}}, {{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 14
+    },
+    {
+      "title": "Transaction Apply Duration per Ledger",
+      "description": "p95 latency of applying the consensus transaction set to a new ledger. The tx.apply span (BuildLedger.cpp) wraps the applyTransactions() function that iterates through the CanonicalTXSet and applies each transaction to the OpenView. Long durations indicate heavy transaction sets or expensive transaction processing.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 92
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=\"tx.apply\"}[5m])))",
+          "legendFormat": "tx.apply [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Latency (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 15
+    },
+    {
+      "title": "TxQ Enqueue Rate by Transaction Type",
+      "description": "Rate of txq.enqueue spans broken down by transaction type (tx_type). Shows what share of inbound demand is Payment vs OfferCreate vs other transactors, and how the mix shifts as the queue fills. A spam burst of one type is a leading indicator of fee escalation.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 100
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (tx_type, exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"txq.enqueue\"}[5m]))",
+          "legendFormat": "{{tx_type}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Enqueues / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 16
+    },
+    {
+      "title": "TxQ Accept Status",
+      "description": "TxQ accept outcomes: applied (included in ledger), failed (removed), retried (kept for next round).",
+      "type": "piechart",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 100
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "values": ["value", "percent"]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "maxHeight": 500
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (txq_status, exported_instance) (increase(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"txq.accept_tx\", txq_status=~\"$txq_status\"}[5m]))",
+          "legendFormat": "{{txq_status}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "id": 17
+    },
+    {
+      "title": "Queue Bypass Ratio (Direct Apply vs Enqueue)",
+      "description": "Ratio of transactions that applied directly to the open ledger (txq.apply_direct) versus those that had to be queued (txq.enqueue). A falling bypass ratio is the cleanest single signal the network has entered sustained fee escalation.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 108
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"txq.apply_direct\"}[5m])) / clamp_min(sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"txq.apply_direct\"}[5m])) + sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"txq.enqueue\"}[5m])), 1)",
+          "legendFormat": "Direct-Apply Fraction [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "custom": {
+            "axisLabel": "Bypass Fraction",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 18
+    },
+    {
+      "title": "Queue Accept (Drain) Duration per Ledger",
+      "description": "p95 duration of the txq.accept span, which drains queued transactions into a newly closed ledger. Rising drain time signals queue pressure at ledger close.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 108
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, exported_instance) (rate(traces_span_metrics_duration_milliseconds_bucket{exported_instance=~\"$node\", span_name=\"txq.accept\"}[5m])))",
+          "legendFormat": "Drain [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 19
+    },
+    {
+      "title": "Queue Cleanup Rate (Expired Entries)",
+      "description": "Rate of txq.cleanup spans, which remove expired transactions from the queue each ledger. A rising rate means submitters under-bid the escalating fee and abandoned their transactions \u2014 a demand-frustration signal distinct from acceptance throughput.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 116
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 500
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(traces_span_metrics_calls_total{exported_instance=~\"$node\", span_name=\"txq.cleanup\"}[5m]))",
+          "legendFormat": "Cleanups / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Cleanups / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      },
+      "id": 20
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "transactions"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)",
+        "type": "query",
+        "query": "label_values(traces_span_metrics_calls_total, exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "tx_origin",
+        "label": "TX Origin",
+        "description": "Filter by transaction origin (true = local submit, false = peer relay)",
+        "type": "query",
+        "query": "label_values(traces_span_metrics_calls_total{span_name=\"tx.process\"}, local)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "tx_type",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus"
+        },
+        "query": "label_values(traces_span_metrics_calls_total{span_name=\"tx.process\", tx_type!=\"\"}, tx_type)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "sort": 1,
+        "label": "TX Type"
+      },
+      {
+        "name": "ter_result",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus"
+        },
+        "query": "label_values(traces_span_metrics_calls_total{span_name=\"tx.process\", ter_result!=\"\"}, ter_result)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "sort": 1,
+        "label": "Result Code"
+      },
+      {
+        "name": "txq_status",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus"
+        },
+        "query": "label_values(traces_span_metrics_calls_total{span_name=\"txq.accept_tx\", txq_status!=\"\"}, txq_status)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "sort": 1,
+        "label": "Queue Status"
+      },
+      {
+        "name": "stage",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus"
+        },
+        "query": "label_values(traces_span_metrics_calls_total{span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", stage!=\"\"}, stage)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "sort": 1,
+        "label": "Apply Stage"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Transaction Overview",
+  "uid": "xrpld-transactions",
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/xrpld-fee-market.json

@@ -0,0 +1,494 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "Fee market dynamics: TxQ depth/capacity, fee escalation levels, and load factor breakdown. Sourced from OTel MetricsRegistry observable gauges (Phase 9).",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Transaction Queue Depth",
+      "description": "Current number of transactions waiting in the queue vs. maximum capacity. Sourced from MetricsRegistry txq_metrics observable gauge with metric=txq_count and metric=txq_max_size.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_txq_metrics{exported_instance=~\"$node\", metric=\"txq_count\"}",
+          "legendFormat": "Queue Depth [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_txq_metrics{exported_instance=~\"$node\", metric=\"txq_max_size\"}",
+          "legendFormat": "Max Capacity [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Transactions",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Transactions Per Ledger",
+      "description": "Transactions in the current open ledger vs. expected per-ledger count. Sourced from txq_metrics with metric=txq_in_ledger and metric=txq_per_ledger.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_txq_metrics{exported_instance=~\"$node\", metric=\"txq_in_ledger\"}",
+          "legendFormat": "In Ledger [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_txq_metrics{exported_instance=~\"$node\", metric=\"txq_per_ledger\"}",
+          "legendFormat": "Expected Per Ledger [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Transactions",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Fee Escalation Levels",
+      "description": "Fee levels that control transaction queue admission. Reference fee level is the baseline; open ledger fee level triggers escalation. Sourced from txq_metrics observable gauge.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_txq_metrics{exported_instance=~\"$node\", metric=\"txq_reference_fee_level\"}",
+          "legendFormat": "Reference Fee Level [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_txq_metrics{exported_instance=~\"$node\", metric=\"txq_min_processing_fee_level\"}",
+          "legendFormat": "Min Processing Fee Level [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_txq_metrics{exported_instance=~\"$node\", metric=\"txq_med_fee_level\"}",
+          "legendFormat": "Median Fee Level [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_txq_metrics{exported_instance=~\"$node\", metric=\"txq_open_ledger_fee_level\"}",
+          "legendFormat": "Open Ledger Fee Level [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Fee Level",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 5,
+            "scaleDistribution": {
+              "type": "log",
+              "log": 2
+            }
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Load Factor Breakdown",
+      "description": "Decomposed load factor components: server (max of local, net, cluster), fee escalation, fee queue, and combined. Values are unitless multipliers where 1.0 = no load. Sourced from load_factor_metrics observable gauge.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_load_factor_metrics{exported_instance=~\"$node\", metric=\"load_factor\"}",
+          "legendFormat": "Combined Load Factor [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_load_factor_metrics{exported_instance=~\"$node\", metric=\"load_factor_server\"}",
+          "legendFormat": "Server [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_load_factor_metrics{exported_instance=~\"$node\", metric=\"load_factor_fee_escalation\"}",
+          "legendFormat": "Fee Escalation [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_load_factor_metrics{exported_instance=~\"$node\", metric=\"load_factor_fee_queue\"}",
+          "legendFormat": "Fee Queue [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Multiplier",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 5
+          },
+          "color": {
+            "mode": "palette-classic"
+          },
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 2
+              },
+              {
+                "color": "red",
+                "value": 10
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Load Factor Components",
+      "description": "Individual load factor contributors: local server load, network load, and cluster load. Only differ from 1.0 under load conditions. Sourced from load_factor_metrics observable gauge.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_load_factor_metrics{exported_instance=~\"$node\", metric=\"load_factor_local\"}",
+          "legendFormat": "Local [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_load_factor_metrics{exported_instance=~\"$node\", metric=\"load_factor_net\"}",
+          "legendFormat": "Network [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_load_factor_metrics{exported_instance=~\"$node\", metric=\"load_factor_cluster\"}",
+          "legendFormat": "Cluster [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Multiplier",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 5
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Load Factor Attribution (Stacked Components)",
+      "description": "Stacked contribution of each load-factor component (fee escalation, queue, local, net, cluster) to the effective transaction cost. Shows WHICH component is driving the fee at any moment, which the aggregate load_factor hides.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_load_factor_metrics{metric=\"load_factor_fee_escalation\",exported_instance=~\"$node\"}",
+          "legendFormat": "Fee Escalation [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_load_factor_metrics{metric=\"load_factor_fee_queue\",exported_instance=~\"$node\"}",
+          "legendFormat": "Fee Queue [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_load_factor_metrics{metric=\"load_factor_local\",exported_instance=~\"$node\"}",
+          "legendFormat": "Local [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_load_factor_metrics{metric=\"load_factor_net\",exported_instance=~\"$node\"}",
+          "legendFormat": "Net [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_load_factor_metrics{metric=\"load_factor_cluster\",exported_instance=~\"$node\"}",
+          "legendFormat": "Cluster [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Load Factor Multiplier",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3,
+            "stacking": {
+              "mode": "normal",
+              "group": "A"
+            },
+            "fillOpacity": 30
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Queue Abandonment Rate (Expired)",
+      "description": "Rate of transactions expired out of the queue (LastLedgerSequence passed). Rising expiry means submitters under-bid the escalating fee and gave up \u2014 a demand-frustration signal.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(xrpld_txq_expired_total{exported_instance=~\"$node\"}[5m]))",
+          "legendFormat": "Expired / Sec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Expired / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Queue Admission Rejections (Dropped)",
+      "description": "Rate of transactions refused admission to the queue, by reason. queue_full means the queue is at capacity \u2014 admission-control backpressure distinct from expiry and from job-queue overflow.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (reason, exported_instance) (rate(xrpld_txq_dropped_total{exported_instance=~\"$node\"}[5m]))",
+          "legendFormat": "{{reason}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Dropped / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "fee-market"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Fee Market & TxQ",
+  "uid": "xrpld-fee-market",
+  "version": 1,
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/xrpld-job-queue.json

@@ -0,0 +1,512 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "Job queue analysis: per-job-type throughput rates, queue wait times, and execution times. Sourced from OTel MetricsRegistry synchronous counters and histograms (Phase 9).",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Current Job Latency (p99 Gauge)",
+      "description": "At-a-glance p99 job queue wait and execution time over the last 5 minutes. Green < 100ms, yellow 100ms-1s, red > 1s.",
+      "type": "gauge",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "orientation": "auto",
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(xrpld_job_queued_duration_us_bucket{exported_instance=~\"$node\", job_type=~\"$job_type\"}[5m])))",
+          "legendFormat": "p99 Wait"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(xrpld_job_running_duration_us_bucket{exported_instance=~\"$node\", job_type=~\"$job_type\"}[5m])))",
+          "legendFormat": "p99 Exec"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "\u00b5s",
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 100000
+              },
+              {
+                "color": "red",
+                "value": 1000000
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "id": 1
+    },
+    {
+      "title": "Job Throughput Rate (Per Second)",
+      "description": "Rate of jobs queued, started, and finished across all job types. Computed as rate() over the OTel counter values. High queue rates with low finish rates indicate backlog.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(xrpld_job_queued_total{exported_instance=~\"$node\", job_type=~\"$job_type\"}[5m]))",
+          "legendFormat": "Queued/s [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(xrpld_job_started_total{exported_instance=~\"$node\", job_type=~\"$job_type\"}[5m]))",
+          "legendFormat": "Started/s [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(xrpld_job_finished_total{exported_instance=~\"$node\", job_type=~\"$job_type\"}[5m]))",
+          "legendFormat": "Finished/s [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10,
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 2
+    },
+    {
+      "title": "Per-Job-Type Queued Rate",
+      "description": "Rate of jobs queued broken down by job_type label. Identifies which job types contribute most to queue activity.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "topk(10, rate(xrpld_job_queued_total{exported_instance=~\"$node\", job_type=~\"$job_type\"}[5m]))",
+          "legendFormat": "{{job_type}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 5,
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 3
+    },
+    {
+      "title": "Per-Job-Type Finish Rate",
+      "description": "Rate of jobs completing broken down by job_type. Compare with queued rate to identify backlog per type.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "topk(10, rate(xrpld_job_finished_total{exported_instance=~\"$node\", job_type=~\"$job_type\"}[5m]))",
+          "legendFormat": "{{job_type}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 5,
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 4
+    },
+    {
+      "title": "Job Queue Wait Time",
+      "description": "Job queue wait time distribution (p75 typical, p99 tail). How long jobs sit in the queue before a worker picks them up.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.75, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{exported_instance=~\"$node\", job_type=~\"$job_type\"}[5m])))",
+          "legendFormat": "p75 Wait [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{exported_instance=~\"$node\", job_type=~\"$job_type\"}[5m])))",
+          "legendFormat": "p99 Wait [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "\u00b5s",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 5,
+            "axisLabel": "Duration (\u03bcs)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 5
+    },
+    {
+      "title": "Job Execution Time",
+      "description": "Job execution time distribution (p75 typical, p99 tail). How long jobs run once started.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.75, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{exported_instance=~\"$node\", job_type=~\"$job_type\"}[5m])))",
+          "legendFormat": "p75 Exec [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{exported_instance=~\"$node\", job_type=~\"$job_type\"}[5m])))",
+          "legendFormat": "p99 Exec [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "\u00b5s",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 5,
+            "axisLabel": "Duration (\u03bcs)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 6
+    },
+    {
+      "title": "Per-Job-Type Execution Time (p99)",
+      "description": "Top 10 slowest job types by p99 execution time.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "topk(10, histogram_quantile(0.99, sum by (le, job_type, exported_instance) (rate(xrpld_job_running_duration_us_bucket{exported_instance=~\"$node\", job_type=~\"$job_type\"}[5m]))))",
+          "legendFormat": "{{job_type}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "\u00b5s",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 5,
+            "axisLabel": "Duration (\u03bcs)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 7
+    },
+    {
+      "title": "Transaction Overflow Rate",
+      "description": "Rate of job queue transaction overflows per minute. Overflows occur when the job queue's transaction limit is exceeded, causing transactions to be dropped. Non-zero values indicate the node is under heavy transaction load.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 40
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_jq_trans_overflow_total{exported_instance=~\"$node\"}[5m]) * 60",
+          "legendFormat": "Overflows/min [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 10
+              }
+            ]
+          },
+          "custom": {
+            "axisLabel": "Overflows / Min",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10,
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 8
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "job-queue"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "job_type",
+        "label": "Job Type",
+        "description": "Filter by job type",
+        "type": "query",
+        "query": "label_values(xrpld_job_queued_total, job_type)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Job Queue Analysis",
+  "uid": "xrpld-job-queue",
+  "version": 1,
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/xrpld-peer-quality.json

@@ -0,0 +1,398 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "Peer network quality metrics: latency, divergence, version distribution, upgrade recommendations, disconnects, and connection direction balance. Requires push_metrics.py or equivalent OTel metric source.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "P90 Peer Latency",
+      "description": "90th percentile peer-to-peer latency in milliseconds over time. High latency indicates network congestion or geographically distant peers.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_peer_quality{metric=\"peer_latency_p90_ms\",exported_instance=~\"$node\"}",
+          "legendFormat": "P90 Latency [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Latency (ms)",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10,
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          },
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 200
+              },
+              {
+                "color": "red",
+                "value": 500
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Insane/Diverged Peers",
+      "description": "Count of peers whose ledger state is considered insane or diverged from the network. Non-zero values suggest those peers are misbehaving or on a fork.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_peer_quality{metric=\"peers_insane_count\",exported_instance=~\"$node\"}",
+          "legendFormat": "Insane Peers [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 3
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Higher Version Peers %",
+      "description": "Percentage of connected peers running a higher rippled version. A high percentage suggests this node should be upgraded.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_peer_quality{metric=\"peers_higher_version_pct\",exported_instance=~\"$node\"}",
+          "legendFormat": "Higher Version % [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 30
+              },
+              {
+                "color": "red",
+                "value": 60
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Upgrade Recommended",
+      "description": "Whether an upgrade is recommended based on peer version analysis (1=recommended, 0=not needed). Triggered when a majority of peers run a newer version.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_peer_quality{metric=\"upgrade_recommended\",exported_instance=~\"$node\"}",
+          "legendFormat": "Upgrade [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": {
+                  "text": "No",
+                  "color": "green"
+                }
+              }
+            },
+            {
+              "type": "value",
+              "options": {
+                "1": {
+                  "text": "Yes",
+                  "color": "red"
+                }
+              }
+            }
+          ],
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Resource Disconnects",
+      "description": "Cumulative peer disconnections due to resource limit violations over time. Rising values indicate aggressive or misbehaving peers being dropped.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 10,
+        "x": 6,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_server_info{metric=\"peer_disconnects_resources\",exported_instance=~\"$node\"}",
+          "legendFormat": "Disconnects [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Disconnects",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10,
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Inbound vs Outbound Peers",
+      "description": "Comparison of active inbound and outbound peer connections. A healthy node should have a balanced mix. All-inbound may indicate NAT/firewall issues preventing outbound connections.",
+      "type": "bargauge",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 8
+      },
+      "options": {
+        "orientation": "horizontal",
+        "displayMode": "gradient",
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_Peer_Finder_Active_Inbound_Peers{exported_instance=~\"$node\"}",
+          "legendFormat": "Inbound [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_Peer_Finder_Active_Outbound_Peers{exported_instance=~\"$node\"}",
+          "legendFormat": "Outbound [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "Inbound.*"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "mode": "fixed",
+                  "fixedColor": "blue"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "Outbound.*"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "mode": "fixed",
+                  "fixedColor": "orange"
+                }
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "peer", "network"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(xrpld_peer_quality, exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Peer Quality",
+  "uid": "xrpld-peer-quality",
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/xrpld-rpc-perf-otel.json

@@ -0,0 +1,452 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "Per-RPC-method performance: call rates, error rates, and latency distributions. Sourced from OTel MetricsRegistry synchronous counters and histograms (Phase 9).",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "RPC Call Rate (All Methods)",
+      "description": "Aggregate rate of RPC calls started, finished, and errored across all methods. Computed as rate() over OTel counters.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(xrpld_rpc_method_started_total{exported_instance=~\"$node\", method=~\"$method\"}[5m]))",
+          "legendFormat": "Started/s [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(xrpld_rpc_method_finished_total{exported_instance=~\"$node\", method=~\"$method\"}[5m]))",
+          "legendFormat": "Finished/s [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "sum by (exported_instance) (rate(xrpld_rpc_method_errored_total{exported_instance=~\"$node\", method=~\"$method\"}[5m]))",
+          "legendFormat": "Errored/s [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10,
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Per-Method Call Rate (Top 10)",
+      "description": "Per-method RPC call rate, showing the 10 most active methods. Useful for identifying hot paths.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "topk(10, rate(xrpld_rpc_method_started_total{exported_instance=~\"$node\", method=~\"$method\"}[5m]))",
+          "legendFormat": "{{method}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 5,
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Per-Method Error Rate (Top 10)",
+      "description": "Per-method RPC error rate. Non-zero values warrant investigation. Common culprits: invalid parameters, resource exhaustion.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "topk(10, rate(xrpld_rpc_method_errored_total{exported_instance=~\"$node\", method=~\"$method\"}[5m]))",
+          "legendFormat": "{{method}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 5,
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Latency - All Methods",
+      "description": "RPC method latency distribution (p75 typical, p99 tail) across all methods.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.75, sum by (le, exported_instance) (rate(xrpld_rpc_method_duration_us_bucket{exported_instance=~\"$node\", method=~\"$method\"}[5m])))",
+          "legendFormat": "p75 [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le, exported_instance) (rate(xrpld_rpc_method_duration_us_bucket{exported_instance=~\"$node\", method=~\"$method\"}[5m])))",
+          "legendFormat": "p99 [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "\u00b5s",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 5,
+            "axisLabel": "Duration (\u03bcs)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Per-Method Latency (p99, Top 10 Slowest)",
+      "description": "Top 10 slowest RPC methods by p99 latency.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "topk(10, histogram_quantile(0.99, sum by (le, method, exported_instance) (rate(xrpld_rpc_method_duration_us_bucket{exported_instance=~\"$node\", method=~\"$method\"}[5m]))))",
+          "legendFormat": "{{method}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "\u00b5s",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 5,
+            "axisLabel": "Duration (\u03bcs)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Error Ratio by Method",
+      "description": "Error ratio (errors / total started) per method. Values above 0.05 (5%) warrant investigation.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "topk(10, rate(xrpld_rpc_method_errored_total{exported_instance=~\"$node\", method=~\"$method\"}[5m]) / (rate(xrpld_rpc_method_started_total{exported_instance=~\"$node\", method=~\"$method\"}[5m]) > 0))",
+          "legendFormat": "{{method}} [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1,
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 5,
+            "axisLabel": "Error Ratio",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          },
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.05
+              },
+              {
+                "color": "red",
+                "value": 0.25
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Current RPC Latency (p99 Gauge)",
+      "description": "At-a-glance p99 RPC method latency over the last 5 minutes. Green < 100ms, yellow 100ms-1s, red > 1s.",
+      "type": "gauge",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "orientation": "auto",
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(xrpld_rpc_method_duration_us_bucket{exported_instance=~\"$node\", method=~\"$method\"}[5m])))",
+          "legendFormat": "p99 Latency"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "\u00b5s",
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 100000
+              },
+              {
+                "color": "red",
+                "value": 1000000
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "rpc"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "method",
+        "label": "RPC Method",
+        "description": "Filter by RPC method",
+        "type": "query",
+        "query": "label_values(xrpld_rpc_method_started_total, method)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "RPC Performance (OTel)",
+  "uid": "xrpld-rpc-perf-otel",
+  "version": 1,
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/xrpld-validator-health.json

@@ -0,0 +1,951 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "Validator health metrics: agreement rates, validation counts, amendment status, UNL expiry, server state tracking, and ledger close rates. Requires push_metrics.py or equivalent OTel metric source.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Validation Agreement",
+      "type": "row",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "collapsed": false,
+      "panels": []
+    },
+    {
+      "title": "Agreement % (1h)",
+      "description": "Validation agreement percentage over the last 1 hour. Values below 80% indicate the validator is frequently disagreeing with the network consensus.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 0,
+        "y": 1
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validation_agreement{metric=\"agreement_pct_1h\",exported_instance=~\"$node\"}",
+          "legendFormat": "Agreement 1h [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 80
+              },
+              {
+                "color": "green",
+                "value": 95
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Agreement % (24h)",
+      "description": "Validation agreement percentage over the last 24 hours. A sustained value below 90% may indicate configuration drift or network partition.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 6,
+        "y": 1
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validation_agreement{metric=\"agreement_pct_24h\",exported_instance=~\"$node\"}",
+          "legendFormat": "Agreement 24h [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 80
+              },
+              {
+                "color": "green",
+                "value": 95
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Agreements vs Missed (1h)",
+      "description": "Comparison of successful agreements and missed validations over 1 hour. High missed count indicates the validator is not participating in consensus rounds.",
+      "type": "bargauge",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 12,
+        "y": 1
+      },
+      "options": {
+        "orientation": "horizontal",
+        "displayMode": "gradient",
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validation_agreement{metric=\"agreements_1h\",exported_instance=~\"$node\"}",
+          "legendFormat": "Agreements 1h [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validation_agreement{metric=\"missed_1h\",exported_instance=~\"$node\"}",
+          "legendFormat": "Missed 1h [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "Missed.*"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "mode": "fixed",
+                  "fixedColor": "red"
+                }
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "title": "Agreements vs Missed (24h)",
+      "description": "Comparison of successful agreements and missed validations over 24 hours. Provides a longer-term view of validator reliability.",
+      "type": "bargauge",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 18,
+        "y": 1
+      },
+      "options": {
+        "orientation": "horizontal",
+        "displayMode": "gradient",
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validation_agreement{metric=\"agreements_24h\",exported_instance=~\"$node\"}",
+          "legendFormat": "Agreements 24h [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validation_agreement{metric=\"missed_24h\",exported_instance=~\"$node\"}",
+          "legendFormat": "Missed 24h [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "Missed.*"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "mode": "fixed",
+                  "fixedColor": "red"
+                }
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "title": "Validation Rates",
+      "type": "row",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 9
+      },
+      "collapsed": false,
+      "panels": []
+    },
+    {
+      "title": "Validation Rate",
+      "description": "Rate of validations sent per minute (5m average). Indicates how actively this node is producing validations. Expected ~10-12/min on mainnet (one per ledger close).",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 0,
+        "y": 10
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_validations_sent_total{exported_instance=~\"$node\"}[5m]) * 60",
+          "legendFormat": "Sent/min [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 5
+              },
+              {
+                "color": "green",
+                "value": 8
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Validations Checked Rate",
+      "description": "Rate of validations checked (received from peers) per minute (5m average). Indicates how many validations from the network are being processed.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 6,
+        "y": 10
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_validations_checked_total{exported_instance=~\"$node\"}[5m]) * 60",
+          "legendFormat": "Checked/min [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Amendment Blocked",
+      "description": "Whether the node is amendment-blocked (1=blocked, 0=normal). An amendment-blocked node cannot validate and needs a software upgrade.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 12,
+        "y": 10
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validator_health{metric=\"amendment_blocked\",exported_instance=~\"$node\"}",
+          "legendFormat": "Blocked [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": {
+                  "text": "OK",
+                  "color": "green"
+                }
+              }
+            },
+            {
+              "type": "value",
+              "options": {
+                "1": {
+                  "text": "BLOCKED",
+                  "color": "red"
+                }
+              }
+            }
+          ],
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "UNL Expiry (days)",
+      "description": "Days until the UNL (Unique Node List) expires. A value below 7 requires attention to renew the UNL before it expires and the validator stops trusting peers.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 18,
+        "y": 10
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validator_health{metric=\"unl_expiry_days\",exported_instance=~\"$node\"}",
+          "legendFormat": "UNL Expiry [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 7
+              },
+              {
+                "color": "green",
+                "value": 30
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "UNL Blocked",
+      "description": "Whether the node's UNL (Unique Node List) is blocked (1=blocked, 0=normal). A UNL-blocked node cannot determine validator trust and may stop participating in consensus.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 0,
+        "y": 18
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validator_health{metric=\"unl_blocked\",exported_instance=~\"$node\"}",
+          "legendFormat": "UNL Blocked [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": {
+                  "text": "OK",
+                  "color": "green"
+                }
+              }
+            },
+            {
+              "type": "value",
+              "options": {
+                "1": {
+                  "text": "BLOCKED",
+                  "color": "red"
+                }
+              }
+            }
+          ],
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Agreement/Missed Counters (Rate)",
+      "description": "Rate of cumulative validation agreements and misses per minute. These monotonic counters complement the rolling window percentages above.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 18,
+        "x": 6,
+        "y": 18
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_validation_agreements_total{exported_instance=~\"$node\"}[5m]) * 60",
+          "legendFormat": "Agreements/min [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_validation_missed_total{exported_instance=~\"$node\"}[5m]) * 60",
+          "legendFormat": "Missed/min [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Per Minute",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10,
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "Missed.*"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "mode": "fixed",
+                  "fixedColor": "red"
+                }
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "title": "Server State & Consensus",
+      "type": "row",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 26
+      },
+      "collapsed": false,
+      "panels": []
+    },
+    {
+      "title": "Validation Quorum",
+      "description": "Minimum number of trusted validations needed to declare a ledger validated. Tracks the current quorum requirement from the validator list.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 0,
+        "y": 27
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validator_health{metric=\"validation_quorum\",exported_instance=~\"$node\"}",
+          "legendFormat": "Quorum [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "State Value Timeline",
+      "description": "Numeric encoding of the server operating state over time. Useful for correlating state changes with other metrics.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 18,
+        "x": 6,
+        "y": 27
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_state_tracking{metric=\"state_value\",exported_instance=~\"$node\"}",
+          "legendFormat": "State [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "State",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10,
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Time in Current State",
+      "description": "How long the server has been in its current operating state, in seconds. Short durations with frequent changes indicate instability.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 35
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_state_tracking{metric=\"time_in_current_state_seconds\",exported_instance=~\"$node\"}",
+          "legendFormat": "Time in State [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "State Changes Rate",
+      "description": "Rate of server state changes per hour. A healthy node should have near-zero state changes. Frequent transitions indicate network or configuration issues.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 35
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_state_changes_total{exported_instance=~\"$node\"}[1h])",
+          "legendFormat": "Changes/hr [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 5
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Ledgers Closed Rate",
+      "description": "Rate of ledgers closed per minute (5m average). On mainnet, expect roughly 10-12/min. Deviations indicate consensus timing issues.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 35
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "rate(xrpld_ledgers_closed_total{exported_instance=~\"$node\"}[5m]) * 60",
+          "legendFormat": "Closed/min [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 5
+              },
+              {
+                "color": "green",
+                "value": 8
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Agreement % (7d)",
+      "description": "Validation agreement percentage over the trailing 7 days \u2014 the long-term reliability window used by external validator dashboards. Complements the 1h/24h stats.",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 43
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validation_agreement{metric=\"agreement_pct_7d\",exported_instance=~\"$node\"}",
+          "legendFormat": "Agreement 7d [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 80
+              },
+              {
+                "color": "green",
+                "value": 95
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Agreements vs Missed (7d)",
+      "description": "Agreed vs missed validation counts over the trailing 7 days. A rising missed trend signals sustained validator unreliability.",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 43
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validation_agreement{metric=\"agreements_7d\",exported_instance=~\"$node\"}",
+          "legendFormat": "Agreements 7d [{{exported_instance}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "expr": "xrpld_validation_agreement{metric=\"missed_7d\",exported_instance=~\"$node\"}",
+          "legendFormat": "Missed 7d [{{exported_instance}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Count",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["xrpld", "validator", "health"],
+  "templating": {
+    "list": [
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(xrpld_validation_agreement, exported_instance)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Validator Health",
+  "uid": "xrpld-validator-health",
+  "refresh": "5s"
+}

docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: xrpld-telemetry
+    orgId: 1
+    folder: xrpld
+    type: file
+    disableDeletion: false
+    editable: true
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: false

docker/telemetry/grafana/provisioning/datasources/loki.yaml

@@ -0,0 +1,39 @@
+# Grafana Loki data source provisioning for rippled log-trace correlation.
+#
+# Phase 8: Log-Trace Correlation and Centralized Log Ingestion
+#
+# Loki ingests rippled logs via OTel Collector's filelog receiver.
+# The derivedFields config links trace_id values in log lines back to
+# Tempo traces, enabling one-click log-to-trace navigation in Grafana.
+#
+# See: OpenTelemetryPlan/Phase8_taskList.md (Tasks 8.2, 8.4)
+
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    uid: loki
+    jsonData:
+      derivedFields:
+        # Trace ID is an OTel-issued 32-hex value emitted by Logs::format()
+        # as `trace_id=...`. Grafana treats the captured value as a Tempo
+        # trace ID, opening the trace directly via Tempo's trace-by-id API.
+        - datasourceUid: tempo
+          matcherRegex: "trace_id=(\\w+)"
+          name: TraceID
+          url: "$${__value.raw}"
+        # 64-char uppercase hex tokens in log bodies are XRPL ledger
+        # hashes (or tx hashes). They are NOT OTel trace IDs and cannot
+        # be resolved via the trace-by-id endpoint. Build a Grafana
+        # Explore deep-link that runs TraceQL on the span attribute
+        # `xrpl.consensus.ledger_id` — set on `consensus.round` spans to
+        # the full prev_ledger hash. This finds the round span whose
+        # first 16 bytes were folded into the OTel trace_id by
+        # SpanGuard::hashSpan().
+        - matcherRegex: "\\b([A-F0-9]{64})\\b"
+          name: ConsensusLedgerHash
+          urlDisplayLabel: "Search Tempo"
+          url: '/explore?schemaVersion=1&orgId=1&panes={"tempo":{"datasource":"tempo","queries":[{"refId":"A","queryType":"traceql","query":"{.xrpl.consensus.ledger_id=\"$${__value.raw}\"}"}],"range":{"from":"now-1h","to":"now"}}}'

docker/telemetry/grafana/provisioning/datasources/prometheus.yaml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true

docker/telemetry/grafana/provisioning/datasources/tempo.yaml

@@ -3,12 +3,10 @@
 # Access Grafana at http://localhost:3000, then use Explore -> Tempo
 # to browse xrpld traces using TraceQL.
 #
-# Search filters provide pre-configured dropdowns in the Explore UI.
-# Each phase adds filters for the span attributes it introduces.
-# Phase 1b (infra): Base filters — node identity, service, span name, status.
-# Phase 2 (RPC):    RPC command, status, role filters.
-# Phase 3 (TX):     Transaction hash, local/peer origin, status.
-# Phase 4 (Cons):   Consensus mode, round, ledger sequence, close time.
+# Search filters provide quick-start dropdowns in the Explore UI for the most
+# common investigation entry points. This is not an exhaustive attribute list —
+# use TraceQL autocomplete or see OpenTelemetryPlan/09-data-collection-reference.md §4
+# for the full attribute inventory and example queries.
 
 apiVersion: 1
 
@@ -26,132 +24,59 @@ datasources:
       # Prometheus service is added to docker-compose.yml.
       serviceMap:
         datasourceUid: prometheus
+      # Phase 8: Trace-to-log correlation — enables one-click navigation
+      # from a Tempo trace to the corresponding Loki log lines. Filters
+      # by trace_id so only logs from the same trace are shown.
+      tracesToLogs:
+        datasourceUid: loki
+        filterByTraceID: true
+        filterBySpanID: false
+        tags: ["partition", "severity"]
       tracesToMetrics:
         datasourceUid: prometheus
         spanStartTimeShift: "-1h"
         spanEndTimeShift: "1h"
       search:
         filters:
-          # --- Node identification filters ---
-          # service.name: logical service name (default: "xrpld").
-          #   Useful when running multiple service types in the same collector.
-          - id: service-name
-            tag: service.name
-            operator: "="
-            scope: resource
-            type: static
-          # service.instance.id: unique node identifier — defaults to the
-          #   node's public key (e.g., nHB1X37...). Distinguishes individual
-          #   nodes in a multi-node cluster or network.
+          # service.instance.id: unique node identifier (public key or configured name).
           - id: node-id
             tag: service.instance.id
             operator: "="
             scope: resource
             type: static
-          # service.version: xrpld build version (e.g., "2.4.0-b1").
-          #   Filter traces from specific software releases.
-          - id: node-version
-            tag: service.version
-            operator: "="
-            scope: resource
-            type: dynamic
-          # xrpl.network.id: numeric network identifier
-          #   (0 = mainnet, 1 = testnet, 2 = devnet, etc.).
-          #   Derived from the [network_id] config section.
-          - id: network-id
-            tag: xrpl.network.id
-            operator: "="
-            scope: resource
-            type: dynamic
-          # xrpl.network.type: human-readable network name derived from
-          #   network ID ("mainnet", "testnet", "devnet", "unknown").
-          - id: network-type
-            tag: xrpl.network.type
-            operator: "="
-            scope: resource
-            type: static
-          # --- Span intrinsic filters ---
-          # name: the span operation name (e.g., "rpc.command.server_info").
-          #   Use to find traces for a specific RPC command or subsystem.
+          # name: span operation name (e.g., "rpc.command.server_info").
           - id: span-name
             tag: name
             operator: "="
             scope: intrinsic
             type: static
           # status: span completion status ("ok", "error", "unset").
-          #   Filter for failed operations to diagnose errors.
           - id: span-status
             tag: status
             operator: "="
             scope: intrinsic
             type: static
-          # duration: span wall-clock duration. Use with ">" operator
-          #   to find slow operations (e.g., duration > 500ms).
-          - id: span-duration
-            tag: duration
-            operator: ">"
-            scope: intrinsic
-            type: static
-          # Phase 2: RPC tracing filters
+          # command: RPC command name (e.g., "server_info", "submit").
           - id: rpc-command
             tag: command
             operator: "="
             scope: span
             type: static
-          - id: rpc-status
-            tag: rpc_status
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: rpc-role
-            tag: rpc_role
-            operator: "="
-            scope: span
-            type: dynamic
-          # Phase 3: Transaction tracing filters
+          # tx_hash: transaction hash — direct lookup for a known transaction.
           - id: tx-hash
             tag: tx_hash
             operator: "="
             scope: span
             type: static
-          - id: tx-origin
-            tag: local
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: tx-status
-            tag: tx_status
-            operator: "="
-            scope: span
-            type: dynamic
-          # Phase 4: Consensus tracing filters
-          - id: consensus-mode
-            tag: xrpl.consensus.mode
+          # tx_type: transaction type (e.g., "Payment", "OfferCreate").
+          - id: tx-type
+            tag: tx_type
             operator: "="
             scope: span
             type: static
-          - id: consensus-round
-            tag: xrpl.consensus.round
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: consensus-ledger-seq
-            tag: xrpl.ledger.seq
+          # ledger_hash: ledger hash — scope all spans to a specific closed ledger.
+          - id: ledger-hash
+            tag: ledger_hash
             operator: "="
             scope: span
             type: static
-          - id: consensus-close-time-correct
-            tag: close_time_correct
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: consensus-state
-            tag: consensus_state
-            operator: "="
-            scope: span
-            type: dynamic
-          - id: consensus-close-resolution
-            tag: close_resolution_ms
-            operator: "="
-            scope: span
-            type: dynamic

docker/telemetry/integration-test.sh

@@ -0,0 +1,750 @@
+#!/usr/bin/env bash
+# Integration test for rippled OpenTelemetry instrumentation.
+#
+# Launches a 6-node xrpld consensus network with telemetry enabled,
+# exercises RPC / transaction / consensus code paths, then verifies
+# that the expected spans and metrics appear in Tempo and Prometheus.
+#
+# Usage:
+#   bash docker/telemetry/integration-test.sh
+#
+# Prerequisites:
+#   - .build/xrpld built with telemetry=ON
+#   - docker compose (v2)
+#   - curl, jq
+#
+# The script leaves the observability stack and xrpld nodes running
+# so you can manually inspect Tempo (localhost:3200) and Grafana
+# (localhost:3000). Run with --cleanup to tear down instead.
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+XRPLD="$REPO_ROOT/.build/xrpld"
+COMPOSE_FILE="$SCRIPT_DIR/docker-compose.yml"
+STANDALONE_CFG="$SCRIPT_DIR/xrpld-telemetry.cfg"
+WORKDIR="${WORKDIR:-/tmp/xrpld-integration}"
+NUM_NODES=6
+PEER_PORT_BASE=51235
+RPC_PORT_BASE=5005
+CONSENSUS_TIMEOUT=120
+GENESIS_ACCOUNT="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"
+GENESIS_SEED="snoPBrXtMeMyMHUVTgbuqAfg1SUTb"
+DEST_ACCOUNT="" # Generated dynamically via wallet_propose
+TEMPO="http://localhost:3200"
+PROM="http://localhost:9090"
+
+# Counters for pass/fail
+PASS=0
+FAIL=0
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+log() { printf "\033[1;34m[INFO]\033[0m  %s\n" "$*"; }
+ok() {
+    printf "\033[1;32m[PASS]\033[0m  %s\n" "$*"
+    PASS=$((PASS + 1))
+}
+fail() {
+    printf "\033[1;31m[FAIL]\033[0m  %s\n" "$*"
+    FAIL=$((FAIL + 1))
+}
+die() {
+    printf "\033[1;31m[ERROR]\033[0m %s\n" "$*" >&2
+    exit 1
+}
+
+check_span() {
+    local op="$1"
+    local count
+    count=$(curl -sf "$TEMPO/api/search" \
+        --data-urlencode "q={resource.service.name=\"rippled\" && name=\"$op\"}" \
+        --data-urlencode "limit=5" |
+        jq '.traces | length' 2>/dev/null || echo 0)
+    if [ "$count" -gt 0 ]; then
+        ok "$op  ($count traces)"
+    else
+        fail "$op  (0 traces)"
+    fi
+}
+
+# Phase 8: Verify trace_id injection in xrpld log output.
+# Greps all node debug.log files for the "trace_id=<hex> span_id=<hex>"
+# pattern that Logs::format() injects when an active OTel span exists.
+# Also cross-checks that a trace_id found in logs matches a trace in Tempo.
+check_log_correlation() {
+    log "Checking log-trace correlation..."
+
+    local total_matches=0
+    local files_scanned=0
+    local sample_trace_id=""
+
+    for i in $(seq 1 "$NUM_NODES"); do
+        local logfile="$WORKDIR/node$i/debug.log"
+        if [ ! -f "$logfile" ]; then
+            continue
+        fi
+        files_scanned=$((files_scanned + 1))
+        local matches
+        matches=$(grep -c 'trace_id=[a-f0-9]\{32\} span_id=[a-f0-9]\{16\}' "$logfile") || matches=0
+        total_matches=$((total_matches + matches))
+        # Capture the first trace_id we find for cross-referencing with Tempo
+        if [ -z "$sample_trace_id" ] && [ "$matches" -gt 0 ]; then
+            sample_trace_id=$(grep -o 'trace_id=[a-f0-9]\{32\}' "$logfile" | head -1 | cut -d= -f2)
+        fi
+    done
+
+    if [ "$files_scanned" -eq 0 ]; then
+        fail "Log correlation: no debug.log files found in $WORKDIR/node*/"
+        return
+    fi
+
+    if [ "$total_matches" -gt 0 ]; then
+        ok "Log correlation: found $total_matches log lines with trace_id ($files_scanned nodes scanned)"
+    else
+        fail "Log correlation: no trace_id found in any node debug.log ($files_scanned nodes scanned)"
+    fi
+
+    # Cross-check: verify the sample trace_id exists in Tempo
+    if [ -n "$sample_trace_id" ]; then
+        local trace_found
+        # Tempo /api/traces/{id} returns OTLP shape: {"batches":[...]}
+        trace_found=$(curl -sf "$TEMPO/api/traces/$sample_trace_id" |
+            jq '.batches | length' 2>/dev/null) || trace_found=0
+        if [ "$trace_found" -gt 0 ]; then
+            ok "Log-Tempo cross-check: trace_id=$sample_trace_id found in Tempo"
+        else
+            fail "Log-Tempo cross-check: trace_id=$sample_trace_id NOT found in Tempo"
+        fi
+    fi
+}
+
+cleanup() {
+    log "Cleaning up..."
+    # Kill xrpld nodes
+    for i in $(seq 1 "$NUM_NODES"); do
+        local pidfile="$WORKDIR/node$i/xrpld.pid"
+        if [ -f "$pidfile" ]; then
+            kill "$(cat "$pidfile")" 2>/dev/null || true
+            rm -f "$pidfile"
+        fi
+    done
+    # Also kill any straggling xrpld processes from our workdir
+    pkill -f "$WORKDIR" 2>/dev/null || true
+    # Stop docker stack
+    docker compose -f "$COMPOSE_FILE" down 2>/dev/null || true
+    # Remove workdir
+    rm -rf "$WORKDIR"
+    log "Cleanup complete."
+}
+
+# Handle --cleanup flag
+if [ "${1:-}" = "--cleanup" ]; then
+    cleanup
+    exit 0
+fi
+
+# ---------------------------------------------------------------------------
+# Step 0: Prerequisites
+# ---------------------------------------------------------------------------
+log "Checking prerequisites..."
+
+command -v docker >/dev/null 2>&1 || die "docker not found"
+docker compose version >/dev/null 2>&1 || die "docker compose (v2) not found"
+command -v curl >/dev/null 2>&1 || die "curl not found"
+command -v jq >/dev/null 2>&1 || die "jq not found"
+[ -x "$XRPLD" ] || die "xrpld binary not found at $XRPLD (build with telemetry=ON)"
+[ -f "$COMPOSE_FILE" ] || die "docker-compose.yml not found at $COMPOSE_FILE"
+[ -f "$STANDALONE_CFG" ] || die "xrpld-telemetry.cfg not found at $STANDALONE_CFG"
+
+log "All prerequisites met."
+
+# ---------------------------------------------------------------------------
+# Step 1: Clean previous run
+# ---------------------------------------------------------------------------
+log "Cleaning previous run data..."
+for i in $(seq 1 "$NUM_NODES"); do
+    pidfile="$WORKDIR/node$i/xrpld.pid"
+    if [ -f "$pidfile" ]; then
+        kill "$(cat "$pidfile")" 2>/dev/null || true
+    fi
+done
+pkill -f "$WORKDIR" 2>/dev/null || true
+# Kill any xrpld using the standalone config (from key generation)
+pkill -f "xrpld-telemetry.cfg" 2>/dev/null || true
+sleep 2
+rm -rf "$WORKDIR"
+mkdir -p "$WORKDIR"
+
+# ---------------------------------------------------------------------------
+# Step 2: Start observability stack
+# ---------------------------------------------------------------------------
+log "Starting observability stack..."
+docker compose -f "$COMPOSE_FILE" up -d
+
+log "Waiting for otel-collector to be ready..."
+for attempt in $(seq 1 30); do
+    # The OTLP HTTP endpoint returns 405 for GET (expects POST), which
+    # means it is listening.  curl -sf would fail on 405, so we check
+    # the HTTP status code explicitly.
+    status=$(curl -so /dev/null -w '%{http_code}' http://localhost:4318/ 2>/dev/null || echo 000)
+    if [ "$status" != "000" ]; then
+        log "otel-collector ready (attempt $attempt, HTTP $status)."
+        break
+    fi
+    if [ "$attempt" -eq 30 ]; then
+        die "otel-collector not ready after 30s"
+    fi
+    sleep 1
+done
+
+log "Waiting for Tempo to be ready..."
+for attempt in $(seq 1 30); do
+    if curl -sf "$TEMPO/ready" >/dev/null 2>&1; then
+        log "Tempo ready (attempt $attempt)."
+        break
+    fi
+    if [ "$attempt" -eq 30 ]; then
+        die "Tempo not ready after 30s"
+    fi
+    sleep 1
+done
+
+# ---------------------------------------------------------------------------
+# Step 3: Generate validator keys
+# ---------------------------------------------------------------------------
+log "Generating $NUM_NODES validator key pairs..."
+
+# Start a temporary standalone xrpld for key generation
+TEMP_DATA="$WORKDIR/temp-keygen"
+mkdir -p "$TEMP_DATA"
+
+# Create a minimal temp config for key generation
+TEMP_CFG="$TEMP_DATA/xrpld.cfg"
+cat >"$TEMP_CFG" <<EOCFG
+[server]
+port_rpc_temp
+
+[port_rpc_temp]
+port = 5099
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[node_db]
+type=NuDB
+path=$TEMP_DATA/nudb
+online_delete=256
+
+[database_path]
+$TEMP_DATA/db
+
+[debug_logfile]
+$TEMP_DATA/debug.log
+
+[ssl_verify]
+0
+EOCFG
+
+"$XRPLD" --conf "$TEMP_CFG" -a --start >"$TEMP_DATA/stdout.log" 2>&1 &
+TEMP_PID=$!
+log "Temporary xrpld started (PID $TEMP_PID), waiting for RPC..."
+
+for attempt in $(seq 1 30); do
+    if curl -sf http://localhost:5099 -d '{"method":"server_info"}' >/dev/null 2>&1; then
+        log "Temporary xrpld RPC ready (attempt $attempt)."
+        break
+    fi
+    if [ "$attempt" -eq 30 ]; then
+        kill "$TEMP_PID" 2>/dev/null || true
+        die "Temporary xrpld RPC not ready after 30s"
+    fi
+    sleep 1
+done
+
+declare -a SEEDS
+declare -a PUBKEYS
+
+for i in $(seq 1 "$NUM_NODES"); do
+    result=$(curl -sf http://localhost:5099 -d '{"method":"validation_create"}')
+    seed=$(echo "$result" | jq -r '.result.validation_seed')
+    pubkey=$(echo "$result" | jq -r '.result.validation_public_key')
+    if [ -z "$seed" ] || [ "$seed" = "null" ]; then
+        kill "$TEMP_PID" 2>/dev/null || true
+        die "Failed to generate key pair $i"
+    fi
+    SEEDS+=("$seed")
+    PUBKEYS+=("$pubkey")
+    log "  Node $i: $pubkey"
+done
+
+kill "$TEMP_PID" 2>/dev/null || true
+wait "$TEMP_PID" 2>/dev/null || true
+rm -rf "$TEMP_DATA"
+log "Key generation complete."
+
+# ---------------------------------------------------------------------------
+# Step 4: Generate node configs and validators.txt
+# ---------------------------------------------------------------------------
+log "Generating node configs..."
+
+# Create shared validators.txt
+VALIDATORS_FILE="$WORKDIR/validators.txt"
+{
+    echo "[validators]"
+    for i in $(seq 0 $((NUM_NODES - 1))); do
+        echo "${PUBKEYS[$i]}"
+    done
+} >"$VALIDATORS_FILE"
+
+# Create per-node configs
+for i in $(seq 1 "$NUM_NODES"); do
+    NODE_DIR="$WORKDIR/node$i"
+    mkdir -p "$NODE_DIR/nudb" "$NODE_DIR/db"
+
+    RPC_PORT=$((RPC_PORT_BASE + i - 1))
+    PEER_PORT=$((PEER_PORT_BASE + i - 1))
+    SEED="${SEEDS[$((i - 1))]}"
+
+    # Build ips_fixed list (all peers except self)
+    IPS_FIXED=""
+    for j in $(seq 1 "$NUM_NODES"); do
+        if [ "$j" -ne "$i" ]; then
+            IPS_FIXED="${IPS_FIXED}127.0.0.1 $((PEER_PORT_BASE + j - 1))
+"
+        fi
+    done
+
+    cat >"$NODE_DIR/xrpld.cfg" <<EOCFG
+[server]
+port_rpc
+port_peer
+
+[port_rpc]
+port = $RPC_PORT
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[port_peer]
+port = $PEER_PORT
+ip = 0.0.0.0
+protocol = peer
+
+[node_db]
+type=NuDB
+path=$NODE_DIR/nudb
+online_delete=256
+
+[database_path]
+$NODE_DIR/db
+
+[debug_logfile]
+$NODE_DIR/debug.log
+
+[validation_seed]
+$SEED
+
+[validators_file]
+$VALIDATORS_FILE
+
+[ips_fixed]
+${IPS_FIXED}
+[peer_private]
+1
+
+[telemetry]
+enabled=1
+service_instance_id=Node-${i}
+endpoint=http://localhost:4318/v1/traces
+exporter=otlp_http
+sampling_ratio=1.0
+batch_size=512
+batch_delay_ms=2000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1
+metrics_endpoint=http://localhost:4318/v1/metrics
+
+[insight]
+server=otel
+endpoint=http://localhost:4318/v1/metrics
+prefix=rippled
+service_instance_id=Node-${i}
+
+[insight]
+server=statsd
+address=127.0.0.1:8125
+prefix=rippled
+
+[rpc_startup]
+{ "command": "log_level", "severity": "warning" }
+
+[ssl_verify]
+0
+EOCFG
+
+    log "  Node $i config: RPC=$RPC_PORT, Peer=$PEER_PORT"
+done
+
+# ---------------------------------------------------------------------------
+# Step 5: Start all 6 nodes
+# ---------------------------------------------------------------------------
+log "Starting $NUM_NODES xrpld nodes..."
+
+for i in $(seq 1 "$NUM_NODES"); do
+    NODE_DIR="$WORKDIR/node$i"
+    "$XRPLD" --conf "$NODE_DIR/xrpld.cfg" --start >"$NODE_DIR/stdout.log" 2>&1 &
+    echo $! >"$NODE_DIR/xrpld.pid"
+    log "  Node $i started (PID $(cat "$NODE_DIR/xrpld.pid"))"
+done
+
+# Give nodes a moment to initialize
+sleep 5
+
+# ---------------------------------------------------------------------------
+# Step 6: Wait for consensus
+# ---------------------------------------------------------------------------
+log "Waiting for nodes to reach 'proposing' state (timeout: ${CONSENSUS_TIMEOUT}s)..."
+
+start_time=$(date +%s)
+nodes_ready=0
+
+while [ "$nodes_ready" -lt "$NUM_NODES" ]; do
+    elapsed=$(($(date +%s) - start_time))
+    if [ "$elapsed" -ge "$CONSENSUS_TIMEOUT" ]; then
+        fail "Consensus timeout after ${CONSENSUS_TIMEOUT}s ($nodes_ready/$NUM_NODES nodes ready)"
+        log "Continuing with partial consensus..."
+        break
+    fi
+
+    nodes_ready=0
+    for i in $(seq 1 "$NUM_NODES"); do
+        RPC_PORT=$((RPC_PORT_BASE + i - 1))
+        state=$(curl -sf "http://localhost:$RPC_PORT" \
+            -d '{"method":"server_info"}' 2>/dev/null |
+            jq -r '.result.info.server_state' 2>/dev/null || echo "unreachable")
+        if [ "$state" = "proposing" ]; then
+            nodes_ready=$((nodes_ready + 1))
+        fi
+    done
+    printf "\r  %d/%d nodes proposing (%ds elapsed)..." "$nodes_ready" "$NUM_NODES" "$elapsed"
+    if [ "$nodes_ready" -lt "$NUM_NODES" ]; then
+        sleep 3
+    fi
+done
+echo ""
+
+if [ "$nodes_ready" -eq "$NUM_NODES" ]; then
+    ok "All $NUM_NODES nodes reached 'proposing' state"
+else
+    fail "Only $nodes_ready/$NUM_NODES nodes reached 'proposing' state"
+fi
+
+# ---------------------------------------------------------------------------
+# Step 6b: Wait for validated ledger
+# ---------------------------------------------------------------------------
+log "Waiting for first validated ledger..."
+for attempt in $(seq 1 60); do
+    val_seq=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
+        -d '{"method":"server_info"}' 2>/dev/null |
+        jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
+    if [ "$val_seq" -gt 2 ] 2>/dev/null; then
+        ok "First validated ledger: seq $val_seq"
+        break
+    fi
+    if [ "$attempt" -eq 60 ]; then
+        fail "No validated ledger after 60s"
+    fi
+    sleep 1
+done
+
+# ---------------------------------------------------------------------------
+# Step 7: Exercise RPC spans (Phase 2)
+# ---------------------------------------------------------------------------
+log "Exercising RPC spans..."
+
+curl -sf "http://localhost:$RPC_PORT_BASE" \
+    -d '{"method":"server_info"}' >/dev/null
+curl -sf "http://localhost:$RPC_PORT_BASE" \
+    -d '{"method":"server_state"}' >/dev/null
+curl -sf "http://localhost:$RPC_PORT_BASE" \
+    -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' >/dev/null
+
+log "RPC commands sent. Waiting 5s for batch export..."
+sleep 5
+
+# ---------------------------------------------------------------------------
+# Step 8: Submit transaction (Phase 3)
+# ---------------------------------------------------------------------------
+log "Submitting Payment transaction..."
+
+# Generate a destination wallet
+log "  Generating destination wallet..."
+wallet_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
+    -d '{"method":"wallet_propose"}')
+DEST_ACCOUNT=$(echo "$wallet_result" | jq -r '.result.account_id' 2>/dev/null)
+if [ -z "$DEST_ACCOUNT" ] || [ "$DEST_ACCOUNT" = "null" ]; then
+    fail "Could not generate destination wallet"
+    DEST_ACCOUNT="rrrrrrrrrrrrrrrrrrrrrhoLvTp" # ACCOUNT_ZERO fallback
+fi
+log "  Destination: $DEST_ACCOUNT"
+
+# Get genesis account info
+acct_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
+    -d "{\"method\":\"account_info\",\"params\":[{\"account\":\"$GENESIS_ACCOUNT\"}]}")
+seq_num=$(echo "$acct_result" | jq -r '.result.account_data.Sequence' 2>/dev/null || echo "unknown")
+log "  Genesis account sequence: $seq_num"
+
+# Submit payment
+submit_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
+    -d "{\"method\":\"submit\",\"params\":[{\"secret\":\"$GENESIS_SEED\",\"tx_json\":{\"TransactionType\":\"Payment\",\"Account\":\"$GENESIS_ACCOUNT\",\"Destination\":\"$DEST_ACCOUNT\",\"Amount\":\"10000000\"}}]}")
+
+engine_result=$(echo "$submit_result" | jq -r '.result.engine_result' 2>/dev/null || echo "unknown")
+tx_hash=$(echo "$submit_result" | jq -r '.result.tx_json.hash' 2>/dev/null || echo "unknown")
+
+if [ "$engine_result" = "tesSUCCESS" ] || [ "$engine_result" = "terQUEUED" ]; then
+    ok "Transaction submitted: $engine_result (hash: ${tx_hash:0:16}...)"
+else
+    fail "Transaction submission: $engine_result"
+    log "  Full response: $(echo "$submit_result" | jq -c .result 2>/dev/null)"
+fi
+
+log "Waiting 15s for consensus round + batch export..."
+sleep 15
+
+# ---------------------------------------------------------------------------
+# Step 9: Verify Tempo traces
+# ---------------------------------------------------------------------------
+log "Verifying spans in Tempo..."
+
+# Check service registration
+services=$(curl -sf "$TEMPO/api/v2/search/tag/resource.service.name/values" |
+    jq -r '.tagValues[].value' 2>/dev/null || echo "")
+if echo "$services" | grep -q "rippled"; then
+    ok "Service 'rippled' registered in Tempo"
+else
+    fail "Service 'rippled' NOT found in Tempo (found: $services)"
+fi
+
+log ""
+log "--- Phase 2: RPC Spans ---"
+check_span "rpc.request"
+check_span "rpc.process"
+check_span "rpc.command.server_info"
+check_span "rpc.command.server_state"
+check_span "rpc.command.ledger"
+
+log ""
+log "--- Phase 3: Transaction Spans ---"
+check_span "tx.process"
+check_span "tx.receive"
+check_span "tx.apply"
+
+log ""
+log "--- Phase 4: Consensus Spans ---"
+check_span "consensus.proposal.send"
+check_span "consensus.ledger_close"
+check_span "consensus.accept"
+check_span "consensus.validation.send"
+
+log ""
+log "--- Phase 5: Ledger Spans ---"
+check_span "ledger.build"
+check_span "ledger.validate"
+check_span "ledger.store"
+
+log ""
+log "--- Phase 5: Peer Spans (trace_peer=1) ---"
+check_span "peer.proposal.receive"
+check_span "peer.validation.receive"
+
+# ---------------------------------------------------------------------------
+# Step 9b: Verify log-trace correlation (Phase 8)
+# ---------------------------------------------------------------------------
+log ""
+log "--- Phase 8: Log-Trace Correlation ---"
+check_log_correlation
+
+# ---------------------------------------------------------------------------
+# Step 10: Verify Prometheus spanmetrics
+# ---------------------------------------------------------------------------
+log ""
+log "--- Phase 5: Spanmetrics ---"
+log "Waiting 20s for Prometheus scrape cycle..."
+sleep 20
+
+calls_count=$(curl -sf "$PROM/api/v1/query?query=traces_span_metrics_calls_total" |
+    jq '.data.result | length' 2>/dev/null || echo 0)
+if [ "$calls_count" -gt 0 ]; then
+    ok "Prometheus: traces_span_metrics_calls_total ($calls_count series)"
+else
+    fail "Prometheus: traces_span_metrics_calls_total (0 series)"
+fi
+
+duration_count=$(curl -sf "$PROM/api/v1/query?query=traces_span_metrics_duration_milliseconds_count" |
+    jq '.data.result | length' 2>/dev/null || echo 0)
+if [ "$duration_count" -gt 0 ]; then
+    ok "Prometheus: duration histogram ($duration_count series)"
+else
+    fail "Prometheus: duration histogram (0 series)"
+fi
+
+# Check Grafana
+if curl -sf http://localhost:3000/api/health >/dev/null 2>&1; then
+    ok "Grafana: healthy at localhost:3000"
+else
+    fail "Grafana: not reachable at localhost:3000"
+fi
+
+# ---------------------------------------------------------------------------
+# Step 10b: Verify native OTel metrics in Prometheus (beast::insight)
+# ---------------------------------------------------------------------------
+log ""
+log "--- Phase 7: Native OTel Metrics (beast::insight via OTLP) ---"
+log "Waiting 20s for OTLP metric export + Prometheus scrape..."
+sleep 20
+
+check_otel_metric() {
+    local metric_name="$1"
+    local result
+    result=$(curl -sf "$PROM/api/v1/query?query=$metric_name" |
+        jq '.data.result | length' 2>/dev/null || echo 0)
+    if [ "$result" -gt 0 ]; then
+        ok "OTel: $metric_name ($result series)"
+    else
+        fail "OTel: $metric_name (0 series)"
+    fi
+}
+
+# Node health gauges (ObservableGauge — no _total suffix)
+check_otel_metric "rippled_LedgerMaster_Validated_Ledger_Age"
+check_otel_metric "rippled_LedgerMaster_Published_Ledger_Age"
+check_otel_metric "rippled_job_count"
+
+# State accounting
+check_otel_metric "rippled_State_Accounting_Full_duration"
+
+# Peer finder
+check_otel_metric "rippled_Peer_Finder_Active_Inbound_Peers"
+check_otel_metric "rippled_Peer_Finder_Active_Outbound_Peers"
+
+# RPC counters (Counter — Prometheus adds _total suffix automatically)
+check_otel_metric "rippled_rpc_requests_total"
+
+# Overlay traffic
+check_otel_metric "rippled_total_Bytes_In"
+
+# Verify StatsD receiver is NOT required (no statsd receiver in pipeline)
+log ""
+log "--- Verify StatsD receiver is not required ---"
+statsd_port_check=$(curl -sf "http://localhost:8125" 2>&1 || echo "refused")
+if echo "$statsd_port_check" | grep -qi "refused\|error\|connection"; then
+    ok "StatsD port 8125 is not listening (not required)"
+else
+    fail "StatsD port 8125 appears to be listening (should not be needed)"
+fi
+
+# ---------------------------------------------------------------------------
+# Step 10c: Verify Phase 9 OTel SDK Metrics
+# ---------------------------------------------------------------------------
+log ""
+log "--- Phase 9: OTel SDK Metrics (MetricsRegistry) ---"
+log "Waiting 15s for OTel metric export + Prometheus scrape..."
+sleep 15
+
+check_otel_metric() {
+    local metric_name="$1"
+    local result
+    result=$(curl -sf "$PROM/api/v1/query?query=$metric_name" |
+        jq '.data.result | length' 2>/dev/null || echo 0)
+    if [ "$result" -gt 0 ]; then
+        ok "OTel: $metric_name ($result series)"
+    else
+        fail "OTel: $metric_name (0 series)"
+    fi
+}
+
+# Task 9.1: NodeStore I/O
+check_otel_metric 'xrpld_nodestore_state{metric="node_reads_total"}'
+check_otel_metric 'xrpld_nodestore_state{metric="write_load"}'
+
+# Task 9.2: Cache hit rates
+check_otel_metric 'xrpld_cache_metrics{metric="SLE_hit_rate"}'
+check_otel_metric 'xrpld_cache_metrics{metric="treenode_cache_size"}'
+
+# Task 9.3: TxQ metrics
+check_otel_metric 'xrpld_txq_metrics{metric="txq_count"}'
+check_otel_metric 'xrpld_txq_metrics{metric="txq_reference_fee_level"}'
+
+# Task 9.4: Per-RPC metrics
+check_otel_metric "xrpld_rpc_method_started_total"
+check_otel_metric "xrpld_rpc_method_finished_total"
+
+# Task 9.5: Per-job metrics
+check_otel_metric "xrpld_job_queued_total"
+check_otel_metric "xrpld_job_finished_total"
+
+# Task 9.6: Counted object instances
+check_otel_metric "xrpld_object_count"
+
+# Task 9.7: Load factor breakdown
+check_otel_metric 'xrpld_load_factor_metrics{metric="load_factor"}'
+check_otel_metric 'xrpld_load_factor_metrics{metric="load_factor_server"}'
+
+# Task 7.15 / Phase 9: ValidationTracker rolling-window agreement gauge.
+# MetricsRegistry::registerValidationAgreementGauge() publishes
+# xrpld_validation_agreement with a `metric` label for each window
+# (1h / 24h / 7d) plus the matching agreement/miss counts. The 7-day
+# window matches the external xrpl-validator-dashboard parity target.
+check_otel_metric 'xrpld_validation_agreement{metric="agreement_pct_1h"}'
+check_otel_metric 'xrpld_validation_agreement{metric="agreement_pct_24h"}'
+check_otel_metric 'xrpld_validation_agreement{metric="agreement_pct_7d"}'
+check_otel_metric 'xrpld_validation_agreement{metric="agreements_1h"}'
+check_otel_metric 'xrpld_validation_agreement{metric="missed_1h"}'
+check_otel_metric 'xrpld_validation_agreement{metric="agreements_24h"}'
+check_otel_metric 'xrpld_validation_agreement{metric="missed_24h"}'
+check_otel_metric 'xrpld_validation_agreement{metric="agreements_7d"}'
+check_otel_metric 'xrpld_validation_agreement{metric="missed_7d"}'
+
+# ---------------------------------------------------------------------------
+# Step 11: Summary
+# ---------------------------------------------------------------------------
+echo ""
+echo "==========================================================="
+echo "  INTEGRATION TEST RESULTS"
+echo "==========================================================="
+printf "  \033[1;32mPASSED: %d\033[0m\n" "$PASS"
+printf "  \033[1;31mFAILED: %d\033[0m\n" "$FAIL"
+echo "==========================================================="
+echo ""
+echo "  Observability stack is running:"
+echo ""
+echo "    Tempo:         http://localhost:3200"
+echo "    Grafana:       http://localhost:3000"
+echo "    Prometheus:    http://localhost:9090"
+echo "    Loki:          http://localhost:3100"
+echo ""
+echo "  xrpld nodes (6) are running:"
+for i in $(seq 1 "$NUM_NODES"); do
+    RPC_PORT=$((RPC_PORT_BASE + i - 1))
+    PEER_PORT=$((PEER_PORT_BASE + i - 1))
+    echo "    Node $i: RPC=localhost:$RPC_PORT  Peer=:$PEER_PORT  PID=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null || echo 'unknown')"
+done
+echo ""
+echo "  To tear down:"
+echo "    bash docker/telemetry/integration-test.sh --cleanup"
+echo ""
+echo "==========================================================="
+
+if [ "$FAIL" -gt 0 ]; then
+    exit 1
+fi

docker/telemetry/otel-collector-config.yaml

@@ -1,9 +1,28 @@
 # OpenTelemetry Collector configuration for xrpld development.
 #
-# Pipeline: OTLP receiver -> batch processor -> debug + Tempo.
+# Pipelines:
+#   traces: OTLP receiver -> batch processor -> debug + Tempo + spanmetrics
+#   metrics: OTLP receiver + spanmetrics connector -> Prometheus exporter
+#   logs:   filelog receiver -> batch processor -> otlphttp/Loki (Phase 8)
+#
 # xrpld sends traces via OTLP/HTTP to port 4318. The collector batches
-# them and forwards to Tempo via OTLP/gRPC on the Docker network. Tempo
-# is queryable via Grafana Explore using TraceQL.
+# them, forwards to Tempo, and derives RED metrics via the spanmetrics
+# connector, which Prometheus scrapes on port 8889.
+#
+# xrpld sends beast::insight metrics natively via OTLP/HTTP to port 4318
+# (same endpoint as traces). The OTLP receiver feeds both the traces and
+# metrics pipelines. Metrics are exported to Prometheus alongside
+# span-derived metrics.
+#
+# Phase 8: The filelog receiver tails xrpld's debug.log files under
+# /var/log/rippled/ (mounted from the host). A regex_parser operator
+# extracts timestamp, partition, severity, and optional trace_id/span_id
+# fields injected by Logs::format(). Parsed logs are exported to Grafana
+# Loki for log-trace correlation.
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
 
 receivers:
   otlp:
@@ -12,11 +31,87 @@ receivers:
         endpoint: 0.0.0.0:4317
       http:
         endpoint: 0.0.0.0:4318
+  # Phase 8: Filelog receiver tails xrpld debug.log files for log-trace
+  # correlation. Extracts structured fields (timestamp, partition, severity,
+  # trace_id, span_id, message) via regex. The trace_id and span_id are
+  # optional — only present when the log was emitted within an active span.
+  filelog:
+    # Tails both user-run xrpld logs (/home/pratik/xrpld-logs/...)
+    # and integration-test logs (/tmp/xrpld-integration/...).
+    include:
+      - /var/log/rippled/*/debug.log
+      - /var/log/rippled-integration/*/debug.log
+    operators:
+      # Log format emitted by Logs::format() is:
+      #   YYYY-Mmm-DD HH:MM:SS.ffffff UTC <partition>:<severity> [trace_id=... span_id=...] <message>
+      # The `partition:` prefix is omitted when partition is empty, so the
+      # capture group is non-capturing optional. Fractional seconds up to 6
+      # digits are parsed via the `%f` strptime directive.
+      - type: regex_parser
+        regex: '^(?P<timestamp>\S+\s+\S+)\s+\S+\s+(?:(?P<partition>\S+):)?(?P<severity>\S+)\s+(?:trace_id=(?P<trace_id>[a-f0-9]+)\s+span_id=(?P<span_id>[a-f0-9]+)\s+)?(?P<message>.*)$'
+        timestamp:
+          parse_from: attributes.timestamp
+          layout: "%Y-%b-%d %H:%M:%S.%f"
+          location: UTC
 
 processors:
   batch:
     timeout: 1s
     send_batch_size: 100
+  resource/logs:
+    attributes:
+      - key: service.name
+        value: xrpld
+        action: upsert
+      # Loki 3.x OTLP ingestion converts `service.name` to the label
+      # `service_name`. The runbook and integration-test queries use the
+      # canonical Loki label `job` so operators can paste `{job="xrpld"}`
+      # without guessing the otel-to-loki naming convention. Upsert the
+      # `job` resource attribute here so it round-trips through OTLP
+      # into Loki as the `job` label.
+      - key: job
+        value: xrpld
+        action: upsert
+
+connectors:
+  spanmetrics:
+    # Expose service.instance.id (node public key) as a Prometheus label so
+    # Grafana dashboards can filter metrics by individual node.
+    resource_metrics_key_attributes:
+      - service.instance.id
+    histogram:
+      explicit:
+        buckets: [1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 5s]
+    dimensions:
+      - name: command
+      - name: rpc_status
+      - name: consensus_mode
+      - name: close_time_correct
+      - name: consensus_state
+      - name: local
+      - name: suppressed
+      - name: proposal_trusted
+      - name: validation_trusted
+      - name: tx_type
+      - name: ter_result
+      # Apply-pipeline stage (preflight|preclaim|apply) — splits the
+      # tx.preflight/tx.preclaim/tx.transactor span RED metrics per stage.
+      - name: stage
+      - name: txq_status
+      - name: load_type
+      - name: is_batch
+      # Consensus lifecycle dimensions (low cardinality, bounded value sets).
+      - name: mode_new
+      - name: consensus_stalled
+      - name: consensus_phase
+      - name: consensus_result
+      # gRPC surface dimensions (bounded: method names, role, status).
+      - name: method
+      - name: grpc_role
+      - name: grpc_status
+      # ledger.acquire dimensions (bounded: outcome, acquire reason).
+      - name: outcome
+      - name: acquire_reason
 
 exporters:
   debug:
@@ -25,10 +120,13 @@ exporters:
     endpoint: tempo:4317
     tls:
       insecure: true
-
-extensions:
-  health_check:
-    endpoint: 0.0.0.0:13133
+  # Phase 8: Export logs to Grafana Loki via OTLP/HTTP. Loki 3.x supports
+  # native OTLP ingestion on its /otlp endpoint, replacing the removed
+  # loki exporter (dropped in otel-collector-contrib v0.147.0).
+  otlphttp/loki:
+    endpoint: http://loki:3100/otlp
+  prometheus:
+    endpoint: 0.0.0.0:8889
 
 service:
   extensions: [health_check]
@@ -36,4 +134,14 @@ service:
     traces:
       receivers: [otlp]
       processors: [batch]
-      exporters: [debug, otlp/tempo]
+      exporters: [debug, otlp/tempo, spanmetrics]
+    metrics:
+      receivers: [otlp, spanmetrics]
+      processors: [batch]
+      exporters: [prometheus]
+    # Phase 8: Log pipeline ingests xrpld debug.log via filelog receiver,
+    # batches entries, and exports to Loki for log-trace correlation.
+    logs:
+      receivers: [filelog]
+      processors: [resource/logs, batch]
+      exporters: [otlphttp/loki]

docker/telemetry/prometheus.yml

@@ -0,0 +1,9 @@
+# Prometheus configuration for scraping spanmetrics from OTel Collector.
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: otel-collector
+    static_configs:
+      - targets: ["otel-collector:8889"]

docker/telemetry/tempo.yaml

@@ -17,6 +17,14 @@ stream_over_http_enabled: true
 server:
   http_listen_port: 3200
 
+# Raise the TraceQL metrics query range limit. The default
+# query_frontend.metrics.max_duration is 3h, so a dashboard set to a longer
+# window (e.g. 6h/12h) fails with "range exceeds 3h0m0s". 168h matches the
+# search max_duration and gives dashboards generous headroom.
+query_frontend:
+  metrics:
+    max_duration: 168h
+
 distributor:
   receivers:
     otlp:
@@ -31,9 +39,11 @@ compactor:
   compaction:
     block_retention: 1h
 
-# Enable metrics generator for service graph and span metrics.
-# Produces RED metrics (rate, errors, duration) per service/span,
-# feeding Grafana's service map visualization.
+# Enable metrics generator for service graph, span metrics, and the
+# local-blocks processor. Produces RED metrics (rate, errors, duration) per
+# service/span for the service map, and keeps recent trace blocks queryable so
+# TraceQL metrics queries (quantile_over_time, count_over_time, etc. via
+# /api/metrics/query_range) work.
 metrics_generator:
   registry:
     external_labels:
@@ -44,6 +54,18 @@ metrics_generator:
     # to enable remote_write for service graph metrics:
     # remote_write:
     #   - url: http://prometheus:9090/api/v1/write
+  # Separate WAL the local-blocks processor flushes traces to for metrics
+  # queries. Required when flush_to_storage is true.
+  traces_storage:
+    path: /var/tempo/generator/traces
+  processor:
+    local_blocks:
+      # xrpld consensus/transaction spans are SPAN_KIND_INTERNAL. By default
+      # local-blocks keeps only server spans for TraceQL metrics, so attribute
+      # aggregations over internal spans return nothing. Keep all spans.
+      filter_server_spans: false
+      # Flush recent blocks to traces_storage so query_range can read them.
+      flush_to_storage: true
 
 overrides:
   defaults:
@@ -51,6 +73,7 @@ overrides:
       processors:
         - service-graphs
         - span-metrics
+        - local-blocks
 
 storage:
   trace:

docker/telemetry/validators-devnet.txt

@@ -0,0 +1,11 @@
+# Devnet validator list configuration.
+#
+# Uses the Devnet validator list publisher operated by Ripple.
+# This fetches the current set of trusted validators for the
+# XRP Ledger Devnet automatically.
+
+[validator_list_sites]
+https://vl.devnet.rippletest.net
+
+[validator_list_keys]
+EDBB54B0D9AEE071BB37784AF5A9E7CC49AC7A0EFCE868C54532BCB966B9CFC13B

docker/telemetry/validators-mainnet.txt

@@ -0,0 +1,15 @@
+# Mainnet validator list configuration.
+#
+# Uses the production XRPL validator list publishers operated by
+# Ripple and the XRPL Foundation. These fetch the current set of
+# trusted validators for XRPL Mainnet automatically.
+
+[validator_list_sites]
+https://vl.ripple.com
+https://unl.xrplf.org
+
+[validator_list_keys]
+# vl.ripple.com
+ED2677ABFFD1B33AC6FBC3062B71F1E8397C1505E1C42C64D11AD1B28FF73F4734
+# unl.xrplf.org
+ED42AEC58B701EEBB77356FFFEC26F83C1F0407263530F068C7C73D392C7E06FD1

docker/telemetry/workload/README.md

@@ -0,0 +1,371 @@
+# Telemetry Workload Tools
+
+Synthetic workload generation and validation tools for xrpld's OpenTelemetry telemetry stack. These tools validate that all spans, metrics, dashboards, and log-trace correlation work end-to-end under controlled load.
+
+## Quick Start
+
+```bash
+# Build xrpld with telemetry enabled
+conan install . --build=missing -o telemetry=True
+cmake --preset default -Dtelemetry=ON
+cmake --build --preset default
+
+# Run full validation (starts everything, runs load, validates)
+docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld
+
+# Cleanup when done
+docker/telemetry/workload/run-full-validation.sh --cleanup
+```
+
+## Architecture
+
+The validation suite runs a multi-node xrpld cluster as local processes alongside
+a Docker Compose telemetry stack. The cluster exercises consensus, peer-to-peer
+spans (proposals, validations), and all metric pipelines.
+
+```
+run-full-validation.sh (shell orchestrator)
+  |
+  |-- docker-compose.workload.yaml
+  |     |-- otel-collector (traces via OTLP + StatsD receiver)
+  |     |-- tempo (trace backend + TraceQL search API)
+  |     |-- prometheus (metrics scraping)
+  |     |-- grafana (dashboards, provisioned automatically)
+  |
+  |-- generate-validator-keys.sh
+  |     -> validator-keys.json, validators.txt
+  |
+  |-- Nx xrpld nodes (local processes, full telemetry)
+  |     - Each node: [telemetry] enabled=1, trace_rpc/consensus/transactions
+  |     - [signing_support] true (server-side signing for tx_submitter)
+  |     - Peer discovery via [ips] (not [ips_fixed]) for active peer counts
+  |
+  |-- workload_orchestrator.py (phased load execution)
+  |     |-- rpc_load_generator.py (WebSocket RPC traffic)
+  |     |-- tx_submitter.py (transaction diversity)
+  |     -> workload-report.json + per-phase reports
+  |
+  |-- validate_telemetry.py (pass/fail checks)
+  |     -> validation-report.json
+  |
+  |-- benchmark.sh (baseline vs telemetry comparison)
+        -> benchmark-report-*.md
+```
+
+## Workload Profiles
+
+The workload orchestrator (`workload_orchestrator.py`) reads named profiles
+from `workload-profiles.json` and executes sequential load phases. Within
+each phase, the RPC generator and TX submitter run concurrently.
+
+### Available Profiles
+
+| Profile           | Phases | Duration                     | Purpose                                                     |
+| ----------------- | ------ | ---------------------------- | ----------------------------------------------------------- |
+| `full-validation` | 6      | ~5 min + 1 min propagation   | Full 18-dashboard coverage with burst/idle/plateau patterns |
+| `quick-smoke`     | 1      | ~30s + 30s propagation       | Fast CI smoke test                                          |
+| `stress`          | 3      | ~3.5 min + 1 min propagation | Heavy sustained load for benchmarking                       |
+
+### full-validation Phases
+
+| Phase        | RPC Rate | TX TPS | Duration | Dashboard Coverage                              |
+| ------------ | -------- | ------ | -------- | ----------------------------------------------- |
+| warmup       | 5 RPS    | —      | 30s      | Node Health, Validator Health (baseline gauges) |
+| steady-state | 30 RPS   | 3 TPS  | 60s      | All dashboards (plateau data)                   |
+| rpc-burst    | 100 RPS  | —      | 30s      | Job Queue, RPC Performance (latency spikes)     |
+| tx-flood     | 5 RPS    | 20 TPS | 30s      | Fee Market & TxQ, Transaction Overview          |
+| mixed-peak   | 50 RPS   | 10 TPS | 60s      | Consensus Health, Ledger Operations             |
+| cooldown     | 5 RPS    | —      | 30s      | Recovery patterns, state transitions            |
+
+### Custom Profiles
+
+Add profiles to `workload-profiles.json`:
+
+```json
+{
+  "profiles": {
+    "my-custom": {
+      "description": "Custom profile for specific testing",
+      "phases": [
+        {
+          "name": "phase-name",
+          "description": "What this phase exercises",
+          "duration_sec": 60,
+          "rpc": { "rate": 50, "weights": { "server_info": 80, "fee": 20 } },
+          "tx": { "tps": 5, "weights": { "Payment": 100 } }
+        }
+      ],
+      "propagation_wait_sec": 30
+    }
+  }
+}
+```
+
+Set `"rpc"` or `"tx"` to `null` to skip that generator for a phase.
+Custom `"weights"` override the default command/transaction distribution.
+
+## Tools Reference
+
+### run-full-validation.sh
+
+Orchestrates the complete validation pipeline. Starts the telemetry stack, starts a multi-node xrpld cluster, generates load, and validates the results.
+
+```bash
+# Full validation with defaults (uses full-validation profile)
+./run-full-validation.sh --xrpld /path/to/xrpld
+
+# Quick smoke test
+./run-full-validation.sh --xrpld /path/to/xrpld --profile quick-smoke
+
+# Stress test with benchmarks
+./run-full-validation.sh --xrpld /path/to/xrpld --profile stress --with-benchmark
+
+# Skip Loki checks (if Phase 8 not deployed)
+./run-full-validation.sh --xrpld /path/to/xrpld --skip-loki
+```
+
+### workload_orchestrator.py
+
+Reads a named profile from `workload-profiles.json` and executes sequential
+load phases. Within each phase, `rpc_load_generator.py` and `tx_submitter.py`
+run as concurrent subprocesses. Produces per-phase reports and a combined
+summary.
+
+```bash
+# Run with a specific profile
+python3 workload_orchestrator.py --profile full-validation
+
+# Multiple endpoints
+python3 workload_orchestrator.py --profile full-validation \
+    --endpoints ws://localhost:6006 ws://localhost:6007
+
+# Save combined report
+python3 workload_orchestrator.py --profile stress --report /tmp/report.json
+```
+
+### rpc_load_generator.py
+
+Generates RPC traffic matching realistic production distribution. Uses
+xrpld's **native WebSocket command format** (`{"command": ...}`) with flat
+parameters — the same format as `tx_submitter.py`.
+
+- 40% health checks (server_info, fee)
+- 30% wallet queries (account_info, account_lines, account_objects)
+- 15% explorer queries (ledger, ledger_data)
+- 10% transaction lookups (tx, account_tx)
+- 5% DEX queries (book_offers, amm_info)
+
+```bash
+# Basic usage
+python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120
+
+# Multiple endpoints (round-robin)
+python3 rpc_load_generator.py \
+    --endpoints ws://localhost:6006 ws://localhost:6007 \
+    --rate 100 --duration 300
+
+# Custom weights
+python3 rpc_load_generator.py --endpoints ws://localhost:6006 \
+    --weights '{"server_info": 80, "account_info": 20}'
+```
+
+### tx_submitter.py
+
+Submits diverse transaction types to exercise the full span and metric surface.
+Uses xrpld's **native WebSocket command format** (`{"command": ...}`) rather
+than JSON-RPC format. The response payload is inside the `"result"` key, with
+`"status"` at the top level.
+
+Supported transaction types:
+
+- Payment (XRP transfers) — exercises `tx.process`, `tx.receive`, `tx.apply`
+- OfferCreate / OfferCancel (DEX activity)
+- TrustSet (trust line creation)
+- NFTokenMint / NFTokenCreateOffer (NFT activity)
+- EscrowCreate / EscrowFinish (escrow lifecycle)
+- AMMCreate / AMMDeposit (AMM pool operations)
+
+Requires `[signing_support] true` in the node config for server-side signing.
+
+```bash
+# Basic usage
+python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120
+
+# Custom mix
+python3 tx_submitter.py --endpoint ws://localhost:6006 \
+    --weights '{"Payment": 60, "OfferCreate": 20, "TrustSet": 20}'
+```
+
+### validate_telemetry.py
+
+Automated validation that all expected telemetry data exists. Every metric and span is required — if it doesn't fire, the validation fails.
+
+- **Span validation**: All span types from `expected_spans.json` with required attributes and parent-child hierarchies
+- **Metric validation**: All metrics from `expected_metrics.json` — SpanMetrics, StatsD gauges/counters/histograms, Phase 9 OTLP metrics. Every listed metric must have > 0 series. Uses the Prometheus `/api/v1/series` endpoint (not instant queries) to avoid false negatives from stale gauges.
+- **Log-trace correlation**: trace_id/span_id in Loki logs (requires Loki)
+- **Dashboard validation**: All 10 Grafana dashboards load with panels
+
+```bash
+# Run all validations
+python3 validate_telemetry.py --report /tmp/report.json
+
+# Skip Loki checks
+python3 validate_telemetry.py --skip-loki --report /tmp/report.json
+```
+
+### OTel Timings Regression Gate
+
+`capture_timings.py` + `compare_to_baseline.py` implement a regression gate
+that compares OTel-derived per-span/per-RPC/per-job timings against a
+committed baseline. Unlike `benchmark.sh` (which measures the overhead of
+enabling telemetry on the current binary), this gate catches **xrpld
+performance regressions over time** by diffing against a stored baseline
+from a prior run.
+
+How it runs inside the validation pipeline:
+
+1. `run-full-validation.sh` executes the normal workload and validation suite.
+2. After validation, `capture_timings.py` queries Prometheus for every
+   metric in `regression-metrics.json` and writes `reports/timings.json`.
+3. `compare_to_baseline.py` reads `timings.json`,
+   `baselines/baseline-timings.json`, and `regression-thresholds.json`,
+   then either:
+   - Prints the paste-me JSON block (when the baseline is a placeholder
+     or empty) and exits 0.
+   - Prints a delta table, writes `reports/regression-report.json`, and
+     exits non-zero if any metric breached both the percentage AND
+     absolute bound.
+
+Bootstrapping a baseline:
+
+1. Push the branch. The `Telemetry Validation` CI run prints the full
+   timings JSON under "Paste into `baselines/baseline-timings.json`" in
+   the workflow Step Summary.
+2. Open a PR copying that JSON block verbatim into
+   `baselines/baseline-timings.json`. Reviewer approval is the audit gate.
+3. Subsequent runs compare against it; the gate fails on regression.
+
+Per-run tuning:
+
+- `--skip-regression` disables the gate (local exploration only).
+- `REGRESSION_WINDOW` env var overrides the default Prometheus `rate()`
+  window (`3m`). Keep close to the workload duration.
+- Metric surface lives in `regression-metrics.json`; thresholds in
+  `regression-thresholds.json`; both are reviewed changes.
+
+See [`baselines/README.md`](./baselines/README.md) for the baseline
+lifecycle and refresh process.
+
+### benchmark.sh
+
+Compares baseline (no telemetry) vs telemetry-enabled performance:
+
+```bash
+./benchmark.sh --xrpld /path/to/xrpld --duration 300
+```
+
+Thresholds (configurable via environment):
+
+| Metric            | Threshold | Env Variable                |
+| ----------------- | --------- | --------------------------- |
+| CPU overhead      | < 3%      | BENCH_CPU_OVERHEAD_PCT      |
+| Memory overhead   | < 5MB     | BENCH_MEM_OVERHEAD_MB       |
+| RPC p99 latency   | < 2ms     | BENCH_RPC_LATENCY_IMPACT_MS |
+| Throughput impact | < 5%      | BENCH_TPS_IMPACT_PCT        |
+| Consensus impact  | < 1%      | BENCH_CONSENSUS_IMPACT_PCT  |
+
+## Reading Validation Reports
+
+The validation report (`validation-report.json`) is structured as:
+
+```json
+{
+  "summary": {
+    "total": 45,
+    "passed": 42,
+    "failed": 3,
+    "all_passed": false
+  },
+  "checks": [
+    {
+      "name": "span.rpc.ws_message",
+      "category": "span",
+      "passed": true,
+      "message": "rpc.ws_message: 15 traces found",
+      "details": { "trace_count": 15 }
+    }
+  ]
+}
+```
+
+Categories:
+
+- **span**: Span type existence and attribute validation
+- **metric**: Prometheus metric existence
+- **log**: Log-trace correlation checks
+- **dashboard**: Grafana dashboard accessibility
+
+## CI Integration
+
+The validation runs as a GitHub Actions workflow (`.github/workflows/telemetry-validation.yml`):
+
+- Triggered manually or on pushes to telemetry branches
+- Builds xrpld, starts the full stack, runs load, validates
+- Uploads reports as artifacts
+- Posts summary to PR
+
+## Configuration Files
+
+| File                              | Purpose                                                       |
+| --------------------------------- | ------------------------------------------------------------- |
+| `workload-profiles.json`          | Named load profiles with phase definitions                    |
+| `expected_spans.json`             | Span inventory (names, attributes, hierarchies, config flags) |
+| `expected_metrics.json`           | Metric inventory — every listed metric must be present        |
+| `test_accounts.json`              | Test account roles (keys generated at runtime)                |
+| `regression-metrics.json`         | Metric surface for the OTel regression gate                   |
+| `regression-thresholds.json`      | Per-metric regression bounds (pct AND abs)                    |
+| `baselines/baseline-timings.json` | Committed baseline — populated from first CI run              |
+| `requirements.txt`                | Python dependencies                                           |
+
+### expected_metrics.json Format
+
+```json
+{
+  "category_name": {
+    "description": "Human-readable description.",
+    "metrics": ["metric_1", "metric_2"]
+  }
+}
+```
+
+Every metric listed must produce > 0 Prometheus series during the validation run. If a metric doesn't fire, the workload generators need to produce enough load to trigger it.
+
+### expected_spans.json Format
+
+Each span entry defines its name, category, parent (for hierarchy validation),
+required attributes, and the `config_flag` that must be enabled:
+
+```json
+{
+  "name": "rpc.command.*",
+  "category": "rpc",
+  "parent": "rpc.process",
+  "required_attributes": ["command", "version", "rpc_role", "rpc_status"],
+  "config_flag": "trace_rpc"
+}
+```
+
+## Node Configuration Notes
+
+The orchestrator (`run-full-validation.sh`) generates node configs with:
+
+- `[telemetry] enabled=1` with all trace categories (`trace_rpc`, `trace_consensus`, `trace_transactions`)
+- `[signing_support] true` — required for `tx_submitter.py` to submit signed transactions via WebSocket
+- `[ips]` (not `[ips_fixed]`) — ensures peer connections are counted in `Peer_Finder_Active_Inbound/Outbound_Peers` metrics (fixed peers are excluded from these counters by design)
+
+## StatsD Gauge Behaviour
+
+Beast::insight StatsD gauges only emit when their value _changes_ from the previous sample. This can cause two problems in the validation environment:
+
+1. **Initial-zero gauges** — if a gauge value is 0 from startup and never changes, the gauge would never emit. To address this, `StatsDGaugeImpl` initializes `m_dirty = true`, ensuring the first flush always emits the initial value.
+2. **Stale gauges** — once a gauge stabilizes (e.g., peer count stays at 1), it stops emitting new data points. Prometheus marks it stale after ~5 minutes. The validation script uses the Prometheus `/api/v1/series` endpoint instead of instant queries to catch such gauges.

docker/telemetry/workload/baselines/README.md

@@ -0,0 +1,72 @@
+# Performance Baselines
+
+This directory holds the committed baseline file used by the OTel-driven regression gate.
+
+## How the gate works
+
+After the validation suite runs, `capture_timings.py` queries Prometheus for the timings
+declared in [`../regression-metrics.json`](../regression-metrics.json) and writes a
+`timings.json`. Then `compare_to_baseline.py` reads [`baseline-timings.json`](./baseline-timings.json),
+[`../regression-thresholds.json`](../regression-thresholds.json), and the captured
+`timings.json`. The comparator picks one of two modes automatically:
+
+- **Placeholder baseline** (`"placeholder": true` or empty `metrics`): the comparator
+  prints the captured timings JSON in exactly the format expected for this file, then
+  exits 0 without gating. This is how we bootstrap the baseline.
+- **Populated baseline**: the comparator diffs per-metric, enforces the thresholds
+  (regression = current exceeds baseline on BOTH the percentage AND absolute bound),
+  and exits non-zero on any regression.
+
+The regression gate runs against whatever workload profile `run-full-validation.sh`
+was invoked with. Capture and comparison are profile-agnostic — they only read
+Prometheus — so all existing profiles (`full-validation`, `quick-smoke`, `stress`)
+continue to work unchanged.
+
+## Bootstrapping the baseline
+
+1. Merge a CI run with a `"placeholder": true` baseline. The telemetry-validation
+   workflow runs, fails no gate, and prints the captured timings block to the workflow
+   Step Summary under the heading `### Paste into baselines/baseline-timings.json`.
+2. Open a new PR. Copy the full JSON block from the Step Summary (or download the
+   `timings.json` artifact) into this file, replacing the placeholder contents. The
+   JSON is emitted in the exact byte-for-byte format this file expects — sorted keys,
+   2-space indent, trailing newline.
+3. The committed baseline PR needs reviewer approval just like any other code change.
+   This is the primary audit point for "who moved the performance bar."
+
+## Refreshing the baseline
+
+Refresh when a legitimate performance change lands on `develop` (for example, a
+deliberate rewrite that changes a span's structure). The process is identical to
+bootstrapping: run CI with the current baseline, inspect the delta, and if the
+new numbers should become the norm, open a PR pasting the fresh timings into
+`baseline-timings.json`. The reviewer decides whether the new baseline is acceptable.
+
+Do **not** edit `baseline-timings.json` by hand outside of this process — every entry
+should trace back to a real CI run so variance characteristics are preserved.
+
+## Schema
+
+```json
+{
+  "schema_version": 1,
+  "captured_at": "2026-04-24T17:30:00Z",
+  "window": "3m",
+  "git_sha": "<SHA of the commit that produced these numbers>",
+  "profile": "<workload profile used>",
+  "metrics": {
+    "span.tx.process.p99": { "value": 12.4, "unit": "ms" },
+    "rpc.server_info.p95": { "value": 850.0, "unit": "us" },
+    "job.transaction.queued.p95": { "value": 1500.0, "unit": "us" }
+  }
+}
+```
+
+Placeholder baselines additionally include `"placeholder": true`. The comparator
+detects this field (or an empty `metrics` object) to switch into "populate" mode
+instead of enforcing thresholds. Remove the `placeholder` key when pasting real
+captured timings.
+
+Missing metrics (value `null`) in a captured run do not count as regressions — they
+are reported separately in `regression-report.json` under `missing_in_current`.
+This keeps the gate robust when a profile doesn't exercise every span on every run.

docker/telemetry/workload/baselines/baseline-timings.json

@@ -0,0 +1,133 @@
+{
+  "captured_at": "2026-06-05T18:41:52Z",
+  "git_sha": "fd1c8c6060f7a15cc9e65b16f99629d9ab7ac7dc",
+  "metrics": {
+    "job.acceptLedger.queued.p95": {
+      "unit": "us",
+      "value": 96.78571428571428
+    },
+    "job.acceptLedger.running.p95": {
+      "unit": "us",
+      "value": 10562.499999999945
+    },
+    "job.transaction.queued.p95": {
+      "unit": "us",
+      "value": 478.96551724137925
+    },
+    "job.transaction.running.p95": {
+      "unit": "us",
+      "value": 494.1361256544502
+    },
+    "span.consensus.accept.p50": {
+      "unit": "ms",
+      "value": 1.059405940594059
+    },
+    "span.consensus.accept.p95": {
+      "unit": "ms",
+      "value": 9.749999999999996
+    },
+    "span.consensus.accept.p99": {
+      "unit": "ms",
+      "value": 23.704545454545432
+    },
+    "span.consensus.ledger_close.p50": {
+      "unit": "ms",
+      "value": 0.5284697508896797
+    },
+    "span.consensus.ledger_close.p95": {
+      "unit": "ms",
+      "value": 1.511111111111103
+    },
+    "span.consensus.ledger_close.p99": {
+      "unit": "ms",
+      "value": 7.878571428571429
+    },
+    "span.ledger.build.p50": {
+      "unit": "ms",
+      "value": 0.7412060301507538
+    },
+    "span.ledger.build.p95": {
+      "unit": "ms",
+      "value": 4.611111111111112
+    },
+    "span.ledger.build.p99": {
+      "unit": "ms",
+      "value": 7.541666666666674
+    },
+    "span.ledger.store.p50": {
+      "unit": "ms",
+      "value": 0.5
+    },
+    "span.ledger.store.p95": {
+      "unit": "ms",
+      "value": 0.95
+    },
+    "span.ledger.store.p99": {
+      "unit": "ms",
+      "value": 0.9900000000000001
+    },
+    "span.ledger.validate.p50": {
+      "unit": "ms",
+      "value": 0.5283687943262412
+    },
+    "span.ledger.validate.p95": {
+      "unit": "ms",
+      "value": 1.3666666666666627
+    },
+    "span.ledger.validate.p99": {
+      "unit": "ms",
+      "value": 6.699999999999978
+    },
+    "span.rpc.process.p50": {
+      "unit": "ms",
+      "value": null
+    },
+    "span.rpc.process.p95": {
+      "unit": "ms",
+      "value": null
+    },
+    "span.rpc.process.p99": {
+      "unit": "ms",
+      "value": null
+    },
+    "span.rpc.ws_message.p50": {
+      "unit": "ms",
+      "value": 0.5026522773001647
+    },
+    "span.rpc.ws_message.p95": {
+      "unit": "ms",
+      "value": 0.9550393268703128
+    },
+    "span.rpc.ws_message.p99": {
+      "unit": "ms",
+      "value": 0.9952515090543261
+    },
+    "span.tx.apply.p50": {
+      "unit": "ms",
+      "value": 0.6330472103004292
+    },
+    "span.tx.apply.p95": {
+      "unit": "ms",
+      "value": 4.203389830508474
+    },
+    "span.tx.apply.p99": {
+      "unit": "ms",
+      "value": 5.083333333333319
+    },
+    "span.tx.process.p50": {
+      "unit": "ms",
+      "value": 0.5042801992591597
+    },
+    "span.tx.process.p95": {
+      "unit": "ms",
+      "value": 0.9581323781882418
+    },
+    "span.tx.process.p99": {
+      "unit": "ms",
+      "value": 0.998474791584883
+    }
+  },
+  "profile": "full-validation",
+  "schema_version": 1,
+  "window": "3m"
+}

docker/telemetry/workload/benchmark.sh

@@ -0,0 +1,397 @@
+#!/usr/bin/env bash
+# benchmark.sh — Performance benchmark for rippled telemetry overhead.
+#
+# Runs two identical workloads against a rippled cluster:
+#   1. Baseline: telemetry disabled ([telemetry] enabled=0)
+#   2. Telemetry: full telemetry enabled (traces + StatsD + all categories)
+#
+# Compares CPU, memory, RPC latency, TPS, and consensus round time.
+# Outputs a Markdown table with pass/fail against configured thresholds.
+#
+# Usage:
+#   ./benchmark.sh --xrpld /path/to/xrpld --duration 300
+#
+# Thresholds (configurable via environment variables):
+#   BENCH_CPU_OVERHEAD_PCT=3       CPU overhead < 3%
+#   BENCH_MEM_OVERHEAD_MB=5        Memory overhead < 5MB
+#   BENCH_RPC_LATENCY_IMPACT_MS=2  RPC p99 latency impact < 2ms
+#   BENCH_TPS_IMPACT_PCT=5         Throughput impact < 5%
+#   BENCH_CONSENSUS_IMPACT_PCT=1   Consensus round time impact < 1%
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Colored output helpers
+# ---------------------------------------------------------------------------
+log() { printf "\033[1;34m[BENCH]\033[0m  %s\n" "$*"; }
+ok() { printf "\033[1;32m[BENCH]\033[0m  %s\n" "$*"; }
+warn() { printf "\033[1;33m[BENCH]\033[0m  %s\n" "$*"; }
+fail() { printf "\033[1;31m[BENCH]\033[0m  %s\n" "$*"; }
+die() {
+    printf "\033[1;31m[BENCH]\033[0m  %s\n" "$*" >&2
+    exit 1
+}
+
+# ---------------------------------------------------------------------------
+# Defaults and thresholds
+# ---------------------------------------------------------------------------
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+
+# Configurable thresholds via environment variables.
+CPU_THRESHOLD="${BENCH_CPU_OVERHEAD_PCT:-3}"
+MEM_THRESHOLD="${BENCH_MEM_OVERHEAD_MB:-5}"
+RPC_THRESHOLD="${BENCH_RPC_LATENCY_IMPACT_MS:-2}"
+TPS_THRESHOLD="${BENCH_TPS_IMPACT_PCT:-5}"
+CONSENSUS_THRESHOLD="${BENCH_CONSENSUS_IMPACT_PCT:-1}"
+
+XRPLD="${BENCH_XRPLD:-$REPO_ROOT/.build/xrpld}"
+DURATION=300
+NUM_NODES=3
+WORKDIR="/tmp/xrpld-benchmark"
+RESULTS_DIR="$SCRIPT_DIR/benchmark-results"
+RPC_PORT_BASE=5020
+PEER_PORT_BASE=51250
+
+# ---------------------------------------------------------------------------
+# Argument parsing
+# ---------------------------------------------------------------------------
+usage() {
+    echo "Usage: $0 [OPTIONS]"
+    echo ""
+    echo "Options:"
+    echo "  --xrpld PATH      Path to xrpld binary (default: \$REPO_ROOT/.build/xrpld)"
+    echo "  --duration SECS   Benchmark duration per run (default: 300)"
+    echo "  --nodes NUM       Number of validator nodes (default: 3)"
+    echo "  --output DIR      Results output directory"
+    echo "  -h, --help        Show this help"
+    exit 0
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --xrpld)
+            XRPLD="$2"
+            shift 2
+            ;;
+        --duration)
+            DURATION="$2"
+            shift 2
+            ;;
+        --nodes)
+            NUM_NODES="$2"
+            shift 2
+            ;;
+        --output)
+            RESULTS_DIR="$2"
+            shift 2
+            ;;
+        -h | --help) usage ;;
+        *) die "Unknown option: $1" ;;
+    esac
+done
+
+# Validate prerequisites.
+[ -x "$XRPLD" ] || die "xrpld not found at $XRPLD"
+command -v jq >/dev/null 2>&1 || die "jq not found"
+command -v bc >/dev/null 2>&1 || die "bc not found"
+command -v curl >/dev/null 2>&1 || die "curl not found"
+
+mkdir -p "$RESULTS_DIR"
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+
+# ---------------------------------------------------------------------------
+# Node cluster management
+# ---------------------------------------------------------------------------
+start_cluster() {
+    local telemetry_enabled="$1"
+    local label="$2"
+
+    log "Starting $NUM_NODES-node cluster ($label, telemetry=$telemetry_enabled)..."
+
+    rm -rf "$WORKDIR"
+    mkdir -p "$WORKDIR"
+
+    # Generate keys using first node.
+    bash "$SCRIPT_DIR/generate-validator-keys.sh" "$XRPLD" "$NUM_NODES" "$WORKDIR"
+
+    # Build per-node configs.
+    for i in $(seq 1 "$NUM_NODES"); do
+        local node_dir="$WORKDIR/node$i"
+        mkdir -p "$node_dir/nudb" "$node_dir/db"
+
+        local rpc_port
+        rpc_port=$((RPC_PORT_BASE + i - 1))
+        local peer_port
+        peer_port=$((PEER_PORT_BASE + i - 1))
+        local seed
+        seed=$(jq -r ".[$((i - 1))].seed" "$WORKDIR/validator-keys.json")
+
+        # Build ips_fixed list.
+        local ips_fixed=""
+        for j in $(seq 1 "$NUM_NODES"); do
+            if [ "$j" -ne "$i" ]; then
+                ips_fixed="${ips_fixed}127.0.0.1 $((PEER_PORT_BASE + j - 1))
+"
+            fi
+        done
+
+        # Build telemetry section.
+        local telemetry_section=""
+        if [ "$telemetry_enabled" = "1" ]; then
+            telemetry_section="
+[telemetry]
+enabled=1
+service_instance_id=bench-node-${i}
+endpoint=http://localhost:4318/v1/traces
+exporter=otlp_http
+sampling_ratio=1.0
+batch_size=512
+batch_delay_ms=2000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1
+
+[insight]
+server=otel
+endpoint=http://localhost:4318/v1/metrics
+prefix=xrpld"
+        else
+            telemetry_section="
+[telemetry]
+enabled=0"
+        fi
+
+        cat >"$node_dir/xrpld.cfg" <<EOCFG
+[server]
+port_rpc
+port_peer
+
+[port_rpc]
+port = $rpc_port
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[port_peer]
+port = $peer_port
+ip = 0.0.0.0
+protocol = peer
+
+[node_db]
+type=NuDB
+path=$node_dir/nudb
+online_delete=256
+
+[database_path]
+$node_dir/db
+
+[debug_logfile]
+$node_dir/debug.log
+
+[validation_seed]
+$seed
+
+[validators_file]
+$WORKDIR/validators.txt
+
+[ips_fixed]
+${ips_fixed}
+[peer_private]
+1
+${telemetry_section}
+
+[rpc_startup]
+{ "command": "log_level", "severity": "warning" }
+
+[ssl_verify]
+0
+EOCFG
+
+        "$XRPLD" --conf "$node_dir/xrpld.cfg" --start >"$node_dir/stdout.log" 2>&1 &
+        echo $! >"$node_dir/xrpld.pid"
+    done
+
+    # Wait for consensus.
+    log "Waiting for consensus..."
+    for attempt in $(seq 1 120); do
+        local ready=0
+        for i in $(seq 1 "$NUM_NODES"); do
+            local port
+            port=$((RPC_PORT_BASE + i - 1))
+            local state
+            state=$(curl -sf "http://localhost:$port" \
+                -d '{"method":"server_info"}' 2>/dev/null |
+                jq -r '.result.info.server_state' 2>/dev/null || echo "")
+            if [ "$state" = "proposing" ]; then
+                ready=$((ready + 1))
+            fi
+        done
+        if [ "$ready" -ge "$NUM_NODES" ]; then
+            ok "All $NUM_NODES nodes proposing (attempt $attempt)"
+            break
+        fi
+        if [ "$attempt" -eq 120 ]; then
+            warn "Consensus timeout — $ready/$NUM_NODES nodes ready"
+        fi
+        sleep 1
+    done
+
+    # Let the cluster stabilize.
+    sleep 5
+}
+
+stop_cluster() {
+    log "Stopping cluster..."
+    for i in $(seq 1 "$NUM_NODES"); do
+        local pidfile="$WORKDIR/node$i/xrpld.pid"
+        if [ -f "$pidfile" ]; then
+            kill "$(cat "$pidfile")" 2>/dev/null || true
+        fi
+    done
+    pkill -f "$WORKDIR" 2>/dev/null || true
+    sleep 3
+}
+
+# Build RPC ports CSV string.
+rpc_ports_csv() {
+    local ports=""
+    for i in $(seq 1 "$NUM_NODES"); do
+        [ -n "$ports" ] && ports="$ports,"
+        ports="$ports$((RPC_PORT_BASE + i - 1))"
+    done
+    echo "$ports"
+}
+
+# ---------------------------------------------------------------------------
+# Run benchmark
+# ---------------------------------------------------------------------------
+log "="
+log "  rippled Telemetry Performance Benchmark"
+log "  Nodes: $NUM_NODES | Duration: ${DURATION}s | Binary: $XRPLD"
+log "="
+
+# --- Baseline run ---
+BASELINE_FILE="$RESULTS_DIR/baseline-${TIMESTAMP}.json"
+start_cluster "0" "baseline"
+bash "$SCRIPT_DIR/collect_system_metrics.sh" "$(rpc_ports_csv)" "$DURATION" "$BASELINE_FILE"
+stop_cluster
+
+# --- Telemetry run ---
+TELEMETRY_FILE="$RESULTS_DIR/telemetry-${TIMESTAMP}.json"
+start_cluster "1" "telemetry"
+bash "$SCRIPT_DIR/collect_system_metrics.sh" "$(rpc_ports_csv)" "$DURATION" "$TELEMETRY_FILE"
+stop_cluster
+
+# ---------------------------------------------------------------------------
+# Compare results
+# ---------------------------------------------------------------------------
+log "Comparing results..."
+
+read_metric() {
+    local file="$1"
+    local key="$2"
+    jq -r ".$key // 0" "$file"
+}
+
+BASE_CPU=$(read_metric "$BASELINE_FILE" "cpu_pct_avg")
+TELE_CPU=$(read_metric "$TELEMETRY_FILE" "cpu_pct_avg")
+CPU_DELTA=$(echo "scale=2; $TELE_CPU - $BASE_CPU" | bc 2>/dev/null || echo "0")
+
+BASE_MEM=$(read_metric "$BASELINE_FILE" "memory_rss_mb_peak")
+TELE_MEM=$(read_metric "$TELEMETRY_FILE" "memory_rss_mb_peak")
+MEM_DELTA=$(echo "scale=2; $TELE_MEM - $BASE_MEM" | bc 2>/dev/null || echo "0")
+
+BASE_RPC=$(read_metric "$BASELINE_FILE" "rpc_p99_ms")
+TELE_RPC=$(read_metric "$TELEMETRY_FILE" "rpc_p99_ms")
+RPC_DELTA=$(echo "scale=2; $TELE_RPC - $BASE_RPC" | bc 2>/dev/null || echo "0")
+
+BASE_TPS=$(read_metric "$BASELINE_FILE" "tps")
+TELE_TPS=$(read_metric "$TELEMETRY_FILE" "tps")
+if [[ "$(echo "$BASE_TPS > 0" | bc 2>/dev/null)" = "1" ]]; then
+    TPS_IMPACT=$(echo "scale=2; ($BASE_TPS - $TELE_TPS) / $BASE_TPS * 100" | bc 2>/dev/null || echo "0")
+else
+    TPS_IMPACT="0"
+fi
+
+BASE_CONS=$(read_metric "$BASELINE_FILE" "consensus_round_p95_ms")
+TELE_CONS=$(read_metric "$TELEMETRY_FILE" "consensus_round_p95_ms")
+if [[ "$(echo "$BASE_CONS > 0" | bc 2>/dev/null)" = "1" ]]; then
+    CONS_IMPACT=$(echo "scale=2; ($TELE_CONS - $BASE_CONS) / $BASE_CONS * 100" | bc 2>/dev/null || echo "0")
+else
+    CONS_IMPACT="0"
+fi
+
+# ---------------------------------------------------------------------------
+# Pass/fail checks
+# ---------------------------------------------------------------------------
+PASS_COUNT=0
+FAIL_COUNT=0
+
+check_threshold() {
+    local name="$1"
+    local actual="$2"
+    local threshold="$3"
+    local unit="$4"
+
+    # Compare: actual <= threshold
+    if [[ "$(echo "$actual <= $threshold" | bc 2>/dev/null)" = "1" ]]; then
+        ok "$name: ${actual}${unit} <= ${threshold}${unit} PASS"
+        PASS_COUNT=$((PASS_COUNT + 1))
+        echo "PASS"
+    else
+        fail "$name: ${actual}${unit} > ${threshold}${unit} FAIL"
+        FAIL_COUNT=$((FAIL_COUNT + 1))
+        echo "FAIL"
+    fi
+}
+
+CPU_RESULT=$(check_threshold "CPU overhead" "$CPU_DELTA" "$CPU_THRESHOLD" "%")
+MEM_RESULT=$(check_threshold "Memory overhead" "$MEM_DELTA" "$MEM_THRESHOLD" "MB")
+RPC_RESULT=$(check_threshold "RPC p99 impact" "$RPC_DELTA" "$RPC_THRESHOLD" "ms")
+TPS_RESULT=$(check_threshold "TPS impact" "$TPS_IMPACT" "$TPS_THRESHOLD" "%")
+CONS_RESULT=$(check_threshold "Consensus impact" "$CONS_IMPACT" "$CONSENSUS_THRESHOLD" "%")
+
+# ---------------------------------------------------------------------------
+# Output Markdown table
+# ---------------------------------------------------------------------------
+REPORT_FILE="$RESULTS_DIR/benchmark-report-${TIMESTAMP}.md"
+
+cat >"$REPORT_FILE" <<EOMD
+# Telemetry Performance Benchmark Report
+
+**Date**: $(date -u +"%Y-%m-%d %H:%M:%S UTC")
+**Nodes**: $NUM_NODES | **Duration**: ${DURATION}s per run
+**Binary**: $XRPLD
+
+## Results
+
+| Metric | Baseline | Telemetry | Delta | Threshold | Result |
+|--------|----------|-----------|-------|-----------|--------|
+| CPU (avg %) | ${BASE_CPU}% | ${TELE_CPU}% | ${CPU_DELTA}% | < ${CPU_THRESHOLD}% | ${CPU_RESULT} |
+| Memory RSS (peak MB) | ${BASE_MEM} MB | ${TELE_MEM} MB | ${MEM_DELTA} MB | < ${MEM_THRESHOLD} MB | ${MEM_RESULT} |
+| RPC p99 Latency (ms) | ${BASE_RPC} ms | ${TELE_RPC} ms | ${RPC_DELTA} ms | < ${RPC_THRESHOLD} ms | ${RPC_RESULT} |
+| Throughput (TPS) | ${BASE_TPS} | ${TELE_TPS} | ${TPS_IMPACT}% | < ${TPS_THRESHOLD}% | ${TPS_RESULT} |
+| Consensus Round p95 (ms) | ${BASE_CONS} ms | ${TELE_CONS} ms | ${CONS_IMPACT}% | < ${CONSENSUS_THRESHOLD}% | ${CONS_RESULT} |
+
+## Summary
+
+- **Passed**: $PASS_COUNT / $((PASS_COUNT + FAIL_COUNT))
+- **Failed**: $FAIL_COUNT / $((PASS_COUNT + FAIL_COUNT))
+
+## Raw Data
+
+- Baseline: \`$(basename "$BASELINE_FILE")\`
+- Telemetry: \`$(basename "$TELEMETRY_FILE")\`
+EOMD
+
+ok "Benchmark report written to $REPORT_FILE"
+cat "$REPORT_FILE"
+
+# Exit with failure if any check failed.
+if [ "$FAIL_COUNT" -gt 0 ]; then
+    exit 1
+fi

docker/telemetry/workload/capture_timings.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+"""Capture OTel-derived timings from Prometheus for the regression gate.
+
+Queries Prometheus for every metric declared in ``regression-metrics.json``
+and writes the results to a JSON file in the exact schema
+``baseline-timings.json`` expects. When a user wants to refresh the
+baseline, they copy a CI run's ``timings.json`` artifact (or the block
+printed to the workflow step summary) into
+``baselines/baseline-timings.json`` in a reviewable PR.
+
+Output schema (stable — ``compare_to_baseline.py`` reads it verbatim)::
+
+    {
+        "schema_version": 1,
+        "captured_at": "2026-04-24T17:30:00Z",
+        "window": "3m",
+        "git_sha": "<from $GITHUB_SHA or `git rev-parse HEAD`>",
+        "profile": "regression",
+        "metrics": {
+            "span.tx.process.p99": {"value": 12.4, "unit": "ms"},
+            "rpc.server_info.p95": {"value": 850.0, "unit": "us"},
+            ...
+        }
+    }
+
+Usage::
+
+    python3 capture_timings.py \\
+        --prometheus http://localhost:9090 \\
+        --metrics regression-metrics.json \\
+        --output /tmp/timings.json \\
+        --window 3m \\
+        --profile regression
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import os
+import subprocess
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import aiohttp
+
+from prom_queries import build_query_plan, run_query_plan
+
+logger = logging.getLogger("capture_timings")
+
+SCHEMA_VERSION = 1
+
+
+async def capture(
+    prom_url: str,
+    metrics_path: Path,
+    window: str,
+    profile: str,
+) -> dict:
+    """Build and execute the query plan, return the full report dict."""
+    plan = build_query_plan(metrics_path, window=window)
+    logger.info("Capturing %d metrics from %s (window=%s)", len(plan), prom_url, window)
+
+    async with aiohttp.ClientSession() as session:
+        metrics = await run_query_plan(session, prom_url, plan)
+
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "captured_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "window": window,
+        "git_sha": _detect_git_sha(),
+        "profile": profile,
+        "metrics": dict(sorted(metrics.items())),
+    }
+
+
+def _detect_git_sha() -> str:
+    """Return the current commit SHA from env or git, else ``"unknown"``.
+
+    Prefers ``GITHUB_SHA`` (set in Actions), falls back to ``git rev-parse``.
+    Silent fallback is fine here — a missing SHA only affects the captured
+    metadata, not the comparison logic.
+    """
+    env_sha = os.environ.get("GITHUB_SHA")
+    if env_sha:
+        return env_sha
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+            check=False,
+        )
+        if result.returncode == 0:
+            return result.stdout.strip()
+    except (OSError, subprocess.SubprocessError):
+        pass
+    return "unknown"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--prometheus",
+        default="http://localhost:9090",
+        help="Prometheus base URL (default: http://localhost:9090)",
+    )
+    parser.add_argument(
+        "--metrics",
+        type=Path,
+        default=Path(__file__).parent / "regression-metrics.json",
+        help="Path to regression-metrics.json",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        required=True,
+        help="Where to write the captured timings JSON",
+    )
+    parser.add_argument(
+        "--window",
+        default="3m",
+        help="Prometheus rate() window (default: 3m)",
+    )
+    parser.add_argument(
+        "--profile",
+        default="regression",
+        help="Workload profile used during capture (metadata only)",
+    )
+    parser.add_argument(
+        "--min-capture-ratio",
+        type=float,
+        default=0.5,
+        help="Fail if fewer than this fraction of metrics are captured (default: 0.5)",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable debug logging",
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(levelname)s %(name)s: %(message)s",
+    )
+
+    report = asyncio.run(
+        capture(
+            prom_url=args.prometheus,
+            metrics_path=args.metrics,
+            window=args.window,
+            profile=args.profile,
+        )
+    )
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with open(args.output, "w") as f:
+        json.dump(report, f, indent=2, sort_keys=True)
+        f.write("\n")
+
+    captured = sum(1 for v in report["metrics"].values() if v["value"] is not None)
+    total = len(report["metrics"])
+    logger.info("Wrote %s (%d/%d metrics captured)", args.output, captured, total)
+
+    if total > 0 and (captured / total) < args.min_capture_ratio:
+        logger.error(
+            "Only %d/%d (%.0f%%) metrics captured — below the %.0f%% minimum. "
+            "Is Prometheus reachable at %s?",
+            captured,
+            total,
+            captured / total * 100,
+            args.min_capture_ratio * 100,
+            args.prometheus,
+        )
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docker/telemetry/workload/collect_system_metrics.sh

@@ -0,0 +1,236 @@
+#!/usr/bin/env bash
+# collect_system_metrics.sh — Collect CPU, memory, and RPC latency metrics
+# from running xrpld nodes for benchmark comparison.
+#
+# Samples system metrics at regular intervals and writes a JSON summary.
+# Used by benchmark.sh for baseline vs telemetry comparison.
+#
+# Usage:
+#   ./collect_system_metrics.sh <rpc_ports_csv> <duration_seconds> <output_file>
+#
+# Example:
+#   ./collect_system_metrics.sh "5005,5006,5007" 300 /tmp/metrics-baseline.json
+#
+# Output JSON format:
+#   {
+#     "cpu_pct_avg": 12.5,
+#     "memory_rss_mb_peak": 450.2,
+#     "rpc_p99_ms": 15.3,
+#     "tps": 4.8,
+#     "consensus_round_p95_ms": 3200,
+#     "samples": 60
+#   }
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Colored output helpers
+# ---------------------------------------------------------------------------
+log() { printf "\033[1;34m[METRICS]\033[0m %s\n" "$*"; }
+ok() { printf "\033[1;32m[METRICS]\033[0m %s\n" "$*"; }
+die() {
+    printf "\033[1;31m[METRICS]\033[0m %s\n" "$*" >&2
+    exit 1
+}
+
+# ---------------------------------------------------------------------------
+# Argument parsing
+# ---------------------------------------------------------------------------
+usage() {
+    echo "Usage: $0 <rpc_ports_csv> <duration_seconds> <output_file>"
+    echo ""
+    echo "Arguments:"
+    echo "  rpc_ports_csv     Comma-separated RPC ports (e.g., 5005,5006,5007)"
+    echo "  duration_seconds  How long to collect metrics"
+    echo "  output_file       Path to write JSON results"
+    exit 1
+}
+
+if [ $# -lt 3 ]; then
+    usage
+fi
+
+RPC_PORTS_CSV="$1"
+DURATION="$2"
+OUTPUT_FILE="$3"
+
+IFS=',' read -ra RPC_PORTS <<<"$RPC_PORTS_CSV"
+SAMPLE_INTERVAL=5
+SAMPLES=$((DURATION / SAMPLE_INTERVAL))
+
+log "Collecting metrics for ${DURATION}s (${SAMPLES} samples, ${#RPC_PORTS[@]} nodes)..."
+
+# ---------------------------------------------------------------------------
+# Temporary files for aggregation
+# ---------------------------------------------------------------------------
+TMPDIR_METRICS="$(mktemp -d)"
+CPU_FILE="$TMPDIR_METRICS/cpu.txt"
+MEM_FILE="$TMPDIR_METRICS/mem.txt"
+RPC_FILE="$TMPDIR_METRICS/rpc.txt"
+LEDGER_FILE="$TMPDIR_METRICS/ledger.txt"
+
+touch "$CPU_FILE" "$MEM_FILE" "$RPC_FILE" "$LEDGER_FILE"
+
+cleanup() {
+    rm -rf "$TMPDIR_METRICS"
+}
+trap cleanup EXIT
+
+# ---------------------------------------------------------------------------
+# Get initial ledger sequence for TPS calculation
+# ---------------------------------------------------------------------------
+INITIAL_SEQ=0
+INITIAL_TIME=$(date +%s)
+for port in "${RPC_PORTS[@]}"; do
+    seq=$(curl -sf "http://localhost:$port" \
+        -d '{"method":"server_info"}' 2>/dev/null |
+        jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
+    if [ "$seq" -gt "$INITIAL_SEQ" ]; then
+        INITIAL_SEQ=$seq
+    fi
+done
+log "Initial validated ledger seq: $INITIAL_SEQ"
+
+# ---------------------------------------------------------------------------
+# Sampling loop
+# ---------------------------------------------------------------------------
+for sample in $(seq 1 "$SAMPLES"); do
+    # Collect CPU usage for xrpld processes.
+    # Uses ps to find all xrpld processes and average their CPU%.
+    cpu_sum=0
+    cpu_count=0
+    while IFS= read -r line; do
+        cpu_val=$(echo "$line" | awk '{print $1}')
+        if [ -n "$cpu_val" ] && [ "$cpu_val" != "0.0" ]; then
+            cpu_sum=$(echo "$cpu_sum + $cpu_val" | bc 2>/dev/null || echo "$cpu_sum")
+            cpu_count=$((cpu_count + 1))
+        fi
+    done < <(ps aux 2>/dev/null | grep '[x]rpld' | awk '{print $3}')
+
+    if [ "$cpu_count" -gt 0 ]; then
+        cpu_avg=$(echo "scale=2; $cpu_sum / $cpu_count" | bc 2>/dev/null || echo "0")
+        echo "$cpu_avg" >>"$CPU_FILE"
+    fi
+
+    # Collect memory RSS for xrpld processes.
+    while IFS= read -r line; do
+        rss_kb=$(echo "$line" | awk '{print $1}')
+        if [ -n "$rss_kb" ] && [ "$rss_kb" != "0" ]; then
+            rss_mb=$(echo "scale=2; $rss_kb / 1024" | bc 2>/dev/null || echo "0")
+            echo "$rss_mb" >>"$MEM_FILE"
+        fi
+    done < <(ps aux 2>/dev/null | grep '[x]rpld' | awk '{print $6}')
+
+    # Collect RPC latency from each node.
+    for port in "${RPC_PORTS[@]}"; do
+        start_ms=$(date +%s%N)
+        curl -sf "http://localhost:$port" \
+            -d '{"method":"server_info"}' >/dev/null 2>&1 || true
+        end_ms=$(date +%s%N)
+        latency_ms=$(((end_ms - start_ms) / 1000000))
+        echo "$latency_ms" >>"$RPC_FILE"
+    done
+
+    # Record current validated ledger seq.
+    for port in "${RPC_PORTS[@]}"; do
+        seq=$(curl -sf "http://localhost:$port" \
+            -d '{"method":"server_info"}' 2>/dev/null |
+            jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
+        echo "$seq" >>"$LEDGER_FILE"
+        break # Only need one node's seq per sample.
+    done
+
+    # Progress indicator.
+    if [ $((sample % 10)) -eq 0 ]; then
+        log "  Sample $sample/$SAMPLES..."
+    fi
+
+    sleep "$SAMPLE_INTERVAL"
+done
+
+# ---------------------------------------------------------------------------
+# Compute aggregated metrics
+# ---------------------------------------------------------------------------
+log "Computing aggregated metrics..."
+
+# CPU average.
+if [ -s "$CPU_FILE" ]; then
+    CPU_AVG=$(awk '{ sum += $1; n++ } END { if (n>0) printf "%.2f", sum/n; else print "0" }' "$CPU_FILE")
+else
+    CPU_AVG="0"
+fi
+
+# Memory peak RSS (MB).
+if [ -s "$MEM_FILE" ]; then
+    MEM_PEAK=$(sort -n "$MEM_FILE" | tail -1)
+else
+    MEM_PEAK="0"
+fi
+
+# RPC latency p99 (ms).
+if [ -s "$RPC_FILE" ]; then
+    RPC_COUNT=$(wc -l <"$RPC_FILE")
+    P99_INDEX=$(echo "scale=0; $RPC_COUNT * 99 / 100" | bc)
+    RPC_P99=$(sort -n "$RPC_FILE" | sed -n "${P99_INDEX}p")
+    [ -z "$RPC_P99" ] && RPC_P99="0"
+else
+    RPC_P99="0"
+fi
+
+# TPS calculation from ledger sequence advancement.
+FINAL_SEQ=0
+for port in "${RPC_PORTS[@]}"; do
+    seq=$(curl -sf "http://localhost:$port" \
+        -d '{"method":"server_info"}' 2>/dev/null |
+        jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
+    if [ "$seq" -gt "$FINAL_SEQ" ]; then
+        FINAL_SEQ=$seq
+    fi
+done
+FINAL_TIME=$(date +%s)
+ELAPSED=$((FINAL_TIME - INITIAL_TIME))
+LEDGER_ADVANCE=$((FINAL_SEQ - INITIAL_SEQ))
+if [ "$ELAPSED" -gt 0 ] && [ "$LEDGER_ADVANCE" -gt 0 ]; then
+    # Rough TPS: assume ~avg_txs_per_ledger * ledgers / elapsed.
+    # Without tx count, use ledger close rate as proxy.
+    TPS=$(echo "scale=2; $LEDGER_ADVANCE / $ELAPSED" | bc 2>/dev/null || echo "0")
+else
+    TPS="0"
+fi
+
+# Consensus round time p95 (from ledger close interval).
+# Approximate by looking at ledger sequence progression intervals.
+if [ -s "$LEDGER_FILE" ]; then
+    # Calculate intervals between consecutive ledger sequences.
+    LEDGER_COUNT=$(wc -l <"$LEDGER_FILE")
+    # Rough estimate: DURATION / number_of_distinct_ledgers * 1000 ms
+    UNIQUE_LEDGERS=$(sort -u "$LEDGER_FILE" | wc -l)
+    if [ "$UNIQUE_LEDGERS" -gt 1 ]; then
+        CONSENSUS_P95=$(echo "scale=0; $DURATION * 1000 / ($UNIQUE_LEDGERS - 1)" | bc 2>/dev/null || echo "0")
+    else
+        CONSENSUS_P95="0"
+    fi
+else
+    CONSENSUS_P95="0"
+fi
+
+# ---------------------------------------------------------------------------
+# Write output JSON
+# ---------------------------------------------------------------------------
+cat >"$OUTPUT_FILE" <<EOF_JSON
+{
+  "cpu_pct_avg": $CPU_AVG,
+  "memory_rss_mb_peak": $MEM_PEAK,
+  "rpc_p99_ms": $RPC_P99,
+  "tps": $TPS,
+  "consensus_round_p95_ms": $CONSENSUS_P95,
+  "samples": $SAMPLES,
+  "duration_seconds": $DURATION,
+  "node_count": ${#RPC_PORTS[@]},
+  "initial_ledger_seq": $INITIAL_SEQ,
+  "final_ledger_seq": $FINAL_SEQ
+}
+EOF_JSON
+
+ok "Metrics written to $OUTPUT_FILE"
+cat "$OUTPUT_FILE"

docker/telemetry/workload/compare_to_baseline.py

@@ -0,0 +1,401 @@
+#!/usr/bin/env python3
+"""Compare captured OTel timings against a committed baseline.
+
+Operating modes (chosen automatically based on the baseline file contents):
+
+1. **No baseline** — if ``baseline-timings.json`` has an empty
+   ``metrics`` object (or is marked with ``"placeholder": true``), this
+   script is in "populate" mode. It prints the captured timings JSON in
+   the exact format expected for pasting into
+   ``baselines/baseline-timings.json``, then exits 0. No regression check.
+
+2. **Populated baseline** — per-metric percentage AND absolute deltas are
+   computed against thresholds from ``regression-thresholds.json``. A
+   regression occurs when BOTH bounds are breached for the same quantile.
+   Prints a human-readable table and writes a full JSON report.
+   Exits 1 if any regression was detected, else 0.
+
+Inputs:
+    --timings     Captured timings JSON (from capture_timings.py)
+    --baseline    Committed baseline JSON
+    --thresholds  Threshold policy JSON
+    --report      Where to write regression-report.json (optional)
+
+Exit codes:
+    0 — No baseline (paste-me emitted), OR baseline populated and no regression
+    1 — Regression detected (at least one metric breached both bounds)
+    2 — Internal error (e.g. bad JSON, baseline/current key mismatch)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+from dataclasses import dataclass, asdict
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("compare_to_baseline")
+
+
+@dataclass
+class MetricDelta:
+    """Single metric's baseline-vs-current comparison outcome.
+
+    Attributes:
+        key:                Flat metric key (e.g. span.tx.process.p99).
+        baseline:           Baseline value (may be None if unpopulated).
+        current:            Current run value (may be None if not captured).
+        delta:              current - baseline (None if either side None).
+        pct_change:         100 * delta / baseline (None if baseline ≤ 0).
+        unit:               Unit from baseline (preserved as-is).
+        threshold_pct:      Resolved per-metric pct threshold.
+        threshold_abs:      Resolved per-metric absolute threshold.
+        regressed:          True iff both bounds breached.
+        note:               Human-readable classification when not regressed.
+    """
+
+    key: str
+    baseline: float | None
+    current: float | None
+    delta: float | None
+    pct_change: float | None
+    unit: str
+    threshold_pct: float | None
+    threshold_abs: float | None
+    regressed: bool
+    note: str
+
+
+def load_json(path: Path) -> dict:
+    with open(path) as f:
+        return json.load(f)
+
+
+def is_placeholder(baseline: dict) -> bool:
+    """A baseline is a placeholder if explicitly marked OR metrics are empty."""
+    if baseline.get("placeholder") is True:
+        return True
+    return not baseline.get("metrics")
+
+
+def print_paste_me(timings: dict) -> None:
+    """Print captured timings in the exact baseline-timings.json format.
+
+    The output between the two banner lines is the file contents to paste,
+    byte-for-byte — sorted keys, 2-space indent, trailing newline.
+    """
+    banner = "=" * 72
+    print(banner, file=sys.stderr)
+    print(
+        "  NO BASELINE FOUND — paste the JSON below into",
+        file=sys.stderr,
+    )
+    print(
+        "  docker/telemetry/workload/baselines/baseline-timings.json",
+        file=sys.stderr,
+    )
+    print(banner, file=sys.stderr)
+
+    print(json.dumps(timings, indent=2, sort_keys=True))
+
+    print(banner, file=sys.stderr)
+    print(
+        "  (End of paste-me JSON. Gate did NOT run — baseline is empty.)",
+        file=sys.stderr,
+    )
+    print(banner, file=sys.stderr)
+
+
+def resolve_thresholds(
+    key: str,
+    thresholds: dict,
+) -> tuple[float | None, float | None]:
+    """Return ``(pct_threshold, abs_threshold)`` for a metric key.
+
+    Per-metric overrides win over defaults. Returns ``(None, None)`` if no
+    threshold is defined for this category/quantile — such metrics are
+    captured but never gate the build.
+    """
+    parts = key.split(".")
+    if len(parts) < 3:
+        return (None, None)
+    category_key = parts[0]
+    quantile_key = parts[-1]
+
+    category_map = {
+        "span": "span",
+        "rpc": "rpc_method",
+        "job": "job_queue",
+    }
+    cat = category_map.get(category_key)
+    if cat is None:
+        return (None, None)
+
+    override_key = f"{category_key}.{'.'.join(parts[1:-1])}"
+    overrides = thresholds.get("overrides", {})
+    defaults = thresholds.get("defaults", {}).get(cat, {})
+
+    rule = overrides.get(override_key, {}).get(quantile_key)
+    if rule is None:
+        rule = defaults.get(quantile_key)
+    if rule is None:
+        return (None, None)
+
+    pct = rule.get("max_pct_increase")
+    abs_bound = rule.get("max_abs_increase_ms")
+    if abs_bound is None:
+        abs_bound = rule.get("max_abs_increase_us")
+    return (pct, abs_bound)
+
+
+def _skip_delta(
+    key: str,
+    baseline: float | None,
+    current: float | None,
+    unit: str,
+    thresholds: dict,
+    note: str,
+) -> MetricDelta:
+    """Build a MetricDelta for cases where comparison is not possible."""
+    pct_threshold, abs_threshold = resolve_thresholds(key, thresholds)
+    return MetricDelta(
+        key=key,
+        baseline=baseline,
+        current=current,
+        delta=None,
+        pct_change=None,
+        unit=unit,
+        threshold_pct=pct_threshold,
+        threshold_abs=abs_threshold,
+        regressed=False,
+        note=note,
+    )
+
+
+def compute_delta(
+    key: str,
+    baseline_entry: dict | None,
+    current_entry: dict | None,
+    thresholds: dict,
+) -> MetricDelta:
+    """Compute a MetricDelta for one metric key.
+
+    A regression requires BOTH bounds to be breached simultaneously. This
+    tolerates small-value noise: a 100% increase on a 0.5 ms metric
+    (to 1.0 ms) is not a regression under a 5 ms absolute bound.
+    """
+    baseline = baseline_entry.get("value") if baseline_entry else None
+    current = current_entry.get("value") if current_entry else None
+    unit = (baseline_entry or current_entry or {}).get("unit", "")
+
+    if baseline is None and current is None:
+        return _skip_delta(
+            key, None, None, unit, thresholds, "no data (neither baseline nor current)"
+        )
+
+    if baseline is None:
+        return _skip_delta(
+            key, None, current, unit, thresholds, "new metric (not in baseline)"
+        )
+
+    if current is None:
+        return _skip_delta(
+            key, baseline, None, unit, thresholds, "not captured in current run"
+        )
+
+    pct_threshold, abs_threshold = resolve_thresholds(key, thresholds)
+    delta = current - baseline
+    pct_change = (delta / baseline * 100.0) if baseline > 0 else None
+
+    if pct_threshold is None or abs_threshold is None:
+        return MetricDelta(
+            key=key,
+            baseline=baseline,
+            current=current,
+            delta=delta,
+            pct_change=pct_change,
+            unit=unit,
+            threshold_pct=pct_threshold,
+            threshold_abs=abs_threshold,
+            regressed=False,
+            note="no threshold configured",
+        )
+
+    pct_breach = pct_change is not None and pct_change > pct_threshold
+    abs_breach = delta > abs_threshold
+    regressed = pct_breach and abs_breach
+
+    if regressed:
+        note = "REGRESSION"
+    elif delta < 0:
+        note = "improved"
+    else:
+        note = "within bounds"
+
+    return MetricDelta(
+        key=key,
+        baseline=baseline,
+        current=current,
+        delta=delta,
+        pct_change=pct_change,
+        unit=unit,
+        threshold_pct=pct_threshold,
+        threshold_abs=abs_threshold,
+        regressed=regressed,
+        note=note,
+    )
+
+
+def print_summary(deltas: list[MetricDelta]) -> None:
+    """Print a sorted, human-readable table of per-metric results."""
+    regressions = [d for d in deltas if d.regressed]
+    improvements = [
+        d
+        for d in deltas
+        if d.delta is not None and d.delta < 0 and d.baseline not in (None, 0)
+    ]
+    improvements.sort(key=lambda d: d.pct_change or 0)
+    regressions.sort(key=lambda d: -(d.pct_change or 0))
+
+    print("=" * 72)
+    print(f"  Regression check: {len(regressions)} regression(s) detected")
+    print("=" * 72)
+
+    if regressions:
+        print("\nRegressions (breached BOTH pct AND absolute bounds):")
+        _print_table(regressions)
+
+    if improvements:
+        top = improvements[:5]
+        print("\nTop improvements:")
+        _print_table(top)
+
+    missing = [d for d in deltas if d.note == "not captured in current run"]
+    if missing:
+        print(f"\n{len(missing)} baseline metric(s) not captured in current run:")
+        for d in missing:
+            print(f"  {d.key}")
+
+
+def _print_table(rows: list[MetricDelta]) -> None:
+    """Print a fixed-width table for a list of deltas."""
+    header = f"  {'METRIC':<45} {'BASE':>10} {'CUR':>10} {'Δ':>10} {'%':>8} UNIT"
+    print(header)
+    print("  " + "-" * (len(header) - 2))
+    for d in rows:
+        base = f"{d.baseline:.2f}" if d.baseline is not None else "-"
+        cur = f"{d.current:.2f}" if d.current is not None else "-"
+        delta = f"{d.delta:+.2f}" if d.delta is not None else "-"
+        pct = f"{d.pct_change:+.1f}%" if d.pct_change is not None else "-"
+        print(f"  {d.key:<45} {base:>10} {cur:>10} {delta:>10} {pct:>8} {d.unit}")
+
+
+def write_report(
+    deltas: list[MetricDelta],
+    report_path: Path,
+    baseline: dict,
+    timings: dict,
+) -> None:
+    """Write regression-report.json — machine-readable artifact for CI."""
+    regressions = [d for d in deltas if d.regressed]
+    payload = {
+        "schema_version": 1,
+        "baseline_captured_at": baseline.get("captured_at"),
+        "baseline_git_sha": baseline.get("git_sha"),
+        "current_captured_at": timings.get("captured_at"),
+        "current_git_sha": timings.get("git_sha"),
+        "window": timings.get("window"),
+        "profile": timings.get("profile"),
+        "summary": {
+            "total": len(deltas),
+            "regressions": len(regressions),
+            "improvements": sum(
+                1
+                for d in deltas
+                if d.delta is not None and d.delta < 0 and d.baseline not in (None, 0)
+            ),
+            "missing_in_current": sum(
+                1 for d in deltas if d.note == "not captured in current run"
+            ),
+        },
+        "metrics": [asdict(d) for d in deltas],
+    }
+    report_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(report_path, "w") as f:
+        json.dump(payload, f, indent=2, sort_keys=True)
+        f.write("\n")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--timings",
+        type=Path,
+        required=True,
+        help="Captured timings JSON (from capture_timings.py)",
+    )
+    parser.add_argument(
+        "--baseline",
+        type=Path,
+        required=True,
+        help="Committed baseline-timings.json",
+    )
+    parser.add_argument(
+        "--thresholds",
+        type=Path,
+        default=Path(__file__).parent / "regression-thresholds.json",
+        help="Threshold policy JSON",
+    )
+    parser.add_argument(
+        "--report",
+        type=Path,
+        default=None,
+        help="Where to write regression-report.json (optional)",
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(levelname)s %(name)s: %(message)s",
+    )
+
+    try:
+        timings = load_json(args.timings)
+        baseline = load_json(args.baseline)
+        thresholds = load_json(args.thresholds)
+    except (OSError, json.JSONDecodeError) as exc:
+        logger.error("failed to load inputs: %s", exc)
+        return 2
+
+    if is_placeholder(baseline):
+        print_paste_me(timings)
+        return 0
+
+    baseline_metrics = baseline.get("metrics", {})
+    current_metrics = timings.get("metrics", {})
+
+    all_keys = sorted(set(baseline_metrics) | set(current_metrics))
+    deltas = [
+        compute_delta(
+            key,
+            baseline_metrics.get(key),
+            current_metrics.get(key),
+            thresholds,
+        )
+        for key in all_keys
+    ]
+
+    print_summary(deltas)
+
+    if args.report:
+        write_report(deltas, args.report, baseline, timings)
+        logger.info("wrote %s", args.report)
+
+    return 1 if any(d.regressed for d in deltas) else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docker/telemetry/workload/expected_metrics.json

@@ -0,0 +1,146 @@
+{
+  "description": "Expected metric inventory for xrpld telemetry validation. Metric names use the xrpld_ prefix (the [insight] prefix and OTel resource service name). Sourced from the live Grafana dashboards and MetricsRegistry.cpp.",
+  "spanmetrics": {
+    "description": "SpanMetrics-derived RED metrics from the OTel Collector spanmetrics connector.",
+    "metrics": [
+      "traces_span_metrics_calls_total",
+      "traces_span_metrics_duration_milliseconds_bucket",
+      "traces_span_metrics_duration_milliseconds_count",
+      "traces_span_metrics_duration_milliseconds_sum"
+    ],
+    "required_labels": [
+      "span_name",
+      "status_code",
+      "service_name",
+      "span_kind"
+    ],
+    "dimension_labels": [
+      "command",
+      "rpc_status",
+      "consensus_mode",
+      "local",
+      "proposal_trusted",
+      "validation_trusted",
+      "tx_type",
+      "ter_result",
+      "stage",
+      "txq_status",
+      "close_time_correct",
+      "consensus_state",
+      "suppressed"
+    ],
+    "_dimension_labels_note": "Bare label names as configured in otel-collector-config.yaml spanmetrics dimensions. Informational only (not asserted by the validator)."
+  },
+  "statsd_gauges": {
+    "description": "beast::insight gauges exported via OTLP/HTTP to the collector (server=otel).",
+    "metrics": [
+      "xrpld_LedgerMaster_Validated_Ledger_Age",
+      "xrpld_LedgerMaster_Published_Ledger_Age",
+      "xrpld_State_Accounting_Full_duration",
+      "xrpld_Peer_Finder_Active_Inbound_Peers",
+      "xrpld_Peer_Finder_Active_Outbound_Peers",
+      "xrpld_jobq_job_count"
+    ]
+  },
+  "statsd_counters": {
+    "description": "beast::insight counters exported via OTLP/HTTP. The OTel Prometheus exporter appends _total to monotonic counters.",
+    "metrics": ["xrpld_rpc_requests_total", "xrpld_ledger_fetches_total"]
+  },
+  "overlay_traffic": {
+    "description": "Overlay traffic metrics (subset — full list has 45+ categories).",
+    "metrics": [
+      "xrpld_total_Bytes_In",
+      "xrpld_total_Bytes_Out",
+      "xrpld_total_Messages_In",
+      "xrpld_total_Messages_Out"
+    ]
+  },
+  "phase9_nodestore": {
+    "description": "Phase 9 NodeStore I/O observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label distinguishing sub-metrics.",
+    "metrics": ["xrpld_nodestore_state"]
+  },
+  "phase9_cache": {
+    "description": "Phase 9 cache hit rate observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
+    "metrics": ["xrpld_cache_metrics"]
+  },
+  "phase9_txq": {
+    "description": "Phase 9 transaction queue observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
+    "metrics": ["xrpld_txq_metrics"]
+  },
+  "phase9_rpc_method": {
+    "description": "Phase 9 per-RPC-method counters (MetricsRegistry via OTLP).",
+    "metrics": ["xrpld_rpc_method_started_total"]
+  },
+  "phase9_objects": {
+    "description": "Phase 9 counted object instances observable gauge (MetricsRegistry via OTLP).",
+    "metrics": ["xrpld_object_count"]
+  },
+  "phase9_load": {
+    "description": "Phase 9 fee escalation and load factor observable gauge (MetricsRegistry via OTLP).",
+    "metrics": ["xrpld_load_factor_metrics"]
+  },
+  "parity_validation_agreement": {
+    "description": "External dashboard parity: validation agreement percentages (MetricsRegistry).",
+    "metrics": [
+      "xrpld_validation_agreement{metric=\"agreement_pct_1h\"}",
+      "xrpld_validation_agreement{metric=\"agreement_pct_24h\"}"
+    ]
+  },
+  "parity_validator_health": {
+    "description": "External dashboard parity: validator health indicators (MetricsRegistry).",
+    "metrics": [
+      "xrpld_validator_health{metric=\"amendment_blocked\"}",
+      "xrpld_validator_health{metric=\"unl_expiry_days\"}"
+    ]
+  },
+  "parity_peer_quality": {
+    "description": "External dashboard parity: peer quality metrics (MetricsRegistry).",
+    "metrics": [
+      "xrpld_peer_quality{metric=\"peer_latency_p90_ms\"}",
+      "xrpld_peer_quality{metric=\"peers_insane_count\"}"
+    ]
+  },
+  "parity_ledger_economy": {
+    "description": "External dashboard parity: ledger economy metrics (MetricsRegistry).",
+    "metrics": [
+      "xrpld_ledger_economy{metric=\"base_fee_xrp\"}",
+      "xrpld_ledger_economy{metric=\"transaction_rate\"}"
+    ]
+  },
+  "parity_state_tracking": {
+    "description": "External dashboard parity: server state tracking (MetricsRegistry).",
+    "metrics": ["xrpld_state_tracking{metric=\"state_value\"}"]
+  },
+  "parity_counters": {
+    "description": "External dashboard parity: monotonic counters (MetricsRegistry).",
+    "metrics": [
+      "xrpld_ledgers_closed_total",
+      "xrpld_validations_sent_total",
+      "xrpld_state_changes_total"
+    ]
+  },
+  "parity_storage": {
+    "description": "External dashboard parity: storage detail metrics (MetricsRegistry).",
+    "metrics": ["xrpld_storage_detail{metric=\"nudb_bytes\"}"]
+  },
+  "grafana_dashboards": {
+    "description": "All Grafana dashboards that must render data (UIDs as provisioned on disk under docker/telemetry/grafana/dashboards/).",
+    "uids": [
+      "xrpld-rpc-perf",
+      "xrpld-rpc-perf-otel",
+      "xrpld-transactions",
+      "xrpld-consensus",
+      "xrpld-ledger-ops",
+      "xrpld-peer-net",
+      "xrpld-peer-quality",
+      "xrpld-fee-market",
+      "xrpld-job-queue",
+      "xrpld-validator-health",
+      "xrpld-system-node-health",
+      "xrpld-system-network",
+      "xrpld-system-rpc",
+      "xrpld-system-overlay-detail",
+      "xrpld-system-ledger-sync"
+    ]
+  }
+}

docker/telemetry/workload/expected_spans.json

@@ -0,0 +1,419 @@
+{
+  "description": "Expected span inventory for xrpld telemetry validation. Attribute keys follow the 2026-05-13 span-attr naming redesign (bare/underscore form; dotted xrpl.* reserved for resource attributes). Sourced from the *SpanNames.h headers. Spans marked \"optional\": true are conditional — they only fire under traffic the harness may not produce (e.g. gRPC client, missing-ledger fetch, mode transitions) and are not failed when absent.",
+  "spans": [
+    {
+      "name": "rpc.ws_message",
+      "category": "rpc",
+      "parent": null,
+      "required_attributes": ["command"],
+      "config_flag": "trace_rpc",
+      "note": "WebSocket RPC root span. The load generator uses WS, so this is the RPC entry span (not rpc.http_request, which needs an HTTP/JSON-RPC client)."
+    },
+    {
+      "name": "rpc.process",
+      "category": "rpc",
+      "parent": "rpc.ws_message",
+      "required_attributes": [],
+      "config_flag": "trace_rpc"
+    },
+    {
+      "name": "rpc.command.*",
+      "category": "rpc",
+      "parent": "rpc.process",
+      "required_attributes": ["command", "version", "rpc_role", "rpc_status"],
+      "config_flag": "trace_rpc",
+      "note": "Wildcard — matches rpc.command.server_info, rpc.command.ledger, etc."
+    },
+    {
+      "name": "rpc.http_request",
+      "category": "rpc",
+      "parent": null,
+      "required_attributes": ["request_payload_size"],
+      "config_flag": "trace_rpc",
+      "optional": true,
+      "note": "HTTP/JSON-RPC root span. The harness load generator is WebSocket-only, so this does not fire."
+    },
+    {
+      "name": "tx.process",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["tx_hash", "local", "path"],
+      "config_flag": "trace_transactions"
+    },
+    {
+      "name": "tx.receive",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["tx_hash", "peer_id", "suppressed"],
+      "config_flag": "trace_transactions",
+      "note": "Cross-node span: parent context propagated from the sender's tx.process via protobuf. Also carries tx_type and peer_version. tx_status is only set when a tx is suppressed/known-bad, so it is not a required attribute on every tx.receive."
+    },
+    {
+      "name": "tx.apply",
+      "category": "transaction",
+      "parent": "ledger.build",
+      "required_attributes": ["tx_count", "tx_failed"],
+      "config_flag": "trace_transactions",
+      "note": "Apply-step span inside BuildLedger. Carries tx_count/tx_failed (ledger_seq lives on the parent ledger.build span)."
+    },
+    {
+      "name": "tx.preflight",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["stage", "tx_type", "ter_result"],
+      "config_flag": "trace_transactions",
+      "note": "Apply-pipeline stage span (stage=preflight). Shares a deterministic trace_id (txID[0:16]) with tx.preclaim/tx.transactor."
+    },
+    {
+      "name": "tx.preclaim",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["stage", "tx_type", "ter_result"],
+      "config_flag": "trace_transactions",
+      "note": "Apply-pipeline stage span (stage=preclaim)."
+    },
+    {
+      "name": "tx.transactor",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["stage", "tx_type"],
+      "config_flag": "trace_transactions",
+      "note": "Apply-pipeline stage span (stage=apply). Also carries applied."
+    },
+    {
+      "name": "txq.enqueue",
+      "category": "transaction",
+      "parent": "tx.process",
+      "required_attributes": ["tx_hash", "tx_type", "txq_status"],
+      "config_flag": "trace_transactions",
+      "optional": true,
+      "note": "Only fires when a tx is queued (fee below open-ledger level). Requires fee escalation — driven by the txq-burst workload phase. tx_hash/tx_type/txq_status are set on every code path; fee_level_paid/required_fee_level are conditional (TxQ.cpp ~895-898, after the rejected and applied_direct early exits), so they are NOT guaranteed on every txq.enqueue span and cannot be required."
+    },
+    {
+      "name": "txq.apply_direct",
+      "category": "transaction",
+      "parent": "txq.enqueue",
+      "required_attributes": [],
+      "config_flag": "trace_transactions",
+      "optional": true,
+      "note": "Child of txq.enqueue when the tx applies directly without queueing."
+    },
+    {
+      "name": "txq.batch_clear",
+      "category": "transaction",
+      "parent": "txq.enqueue",
+      "required_attributes": ["num_cleared"],
+      "config_flag": "trace_transactions",
+      "optional": true
+    },
+    {
+      "name": "txq.accept",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["queue_size", "ledger_changed"],
+      "config_flag": "trace_transactions",
+      "optional": true,
+      "note": "Ledger-close accept loop. Fires on the consensus thread; only meaningful when the queue is non-empty."
+    },
+    {
+      "name": "txq.accept.tx",
+      "category": "transaction",
+      "parent": "txq.accept",
+      "required_attributes": [
+        "tx_hash",
+        "ter_code",
+        "retries_remaining",
+        "txq_status"
+      ],
+      "config_flag": "trace_transactions",
+      "optional": true
+    },
+    {
+      "name": "txq.cleanup",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["ledger_seq", "expired_count"],
+      "config_flag": "trace_transactions",
+      "optional": true
+    },
+    {
+      "name": "consensus.round",
+      "category": "consensus",
+      "parent": null,
+      "required_attributes": [
+        "consensus_ledger_id",
+        "ledger_seq",
+        "consensus_mode",
+        "consensus_round_id",
+        "consensus_phase"
+      ],
+      "config_flag": "trace_consensus",
+      "note": "Root consensus span created per round. Also carries trace_strategy, previous_ledger_seq, previous_proposers, previous_round_time_ms."
+    },
+    {
+      "name": "consensus.phase.open",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": [],
+      "config_flag": "trace_consensus"
+    },
+    {
+      "name": "consensus.proposal.send",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": ["consensus_round"],
+      "config_flag": "trace_consensus",
+      "note": "Also carries is_bow_out."
+    },
+    {
+      "name": "consensus.ledger_close",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": ["ledger_seq", "consensus_mode"],
+      "config_flag": "trace_consensus",
+      "note": "Also carries tx_count_open, close_time_resolution_ms."
+    },
+    {
+      "name": "consensus.establish",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": [
+        "converge_percent",
+        "establish_count",
+        "proposers",
+        "disputes_count"
+      ],
+      "config_flag": "trace_consensus"
+    },
+    {
+      "name": "consensus.update_positions",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": [
+        "converge_percent",
+        "proposers",
+        "disputes_count"
+      ],
+      "config_flag": "trace_consensus"
+    },
+    {
+      "name": "consensus.check",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": [
+        "agree_count",
+        "disagree_count",
+        "threshold_percent",
+        "consensus_result"
+      ],
+      "config_flag": "trace_consensus"
+    },
+    {
+      "name": "consensus.accept",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": ["proposers", "round_time_ms", "quorum"],
+      "config_flag": "trace_consensus"
+    },
+    {
+      "name": "consensus.accept.apply",
+      "category": "consensus",
+      "parent": "consensus.accept",
+      "required_attributes": [
+        "ledger_seq",
+        "close_time",
+        "parent_close_time",
+        "close_time_self",
+        "close_time_vote_bins",
+        "resolution_direction"
+      ],
+      "config_flag": "trace_consensus",
+      "note": "Also carries close_time_correct, close_resolution_ms, consensus_state, proposing, round_time_ms, tx_count."
+    },
+    {
+      "name": "consensus.validation.send",
+      "category": "consensus",
+      "parent": null,
+      "required_attributes": [
+        "ledger_seq",
+        "proposing",
+        "ledger_hash",
+        "full_validation"
+      ],
+      "config_flag": "trace_consensus",
+      "note": "follows-from consensus.accept. ledger_hash is BARE here (consensus-owned attr), unlike peer.validation.receive which uses the shared dotted xrpl.ledger.hash. Also carries validation_sign_time."
+    },
+    {
+      "name": "consensus.proposal.receive",
+      "category": "consensus",
+      "parent": null,
+      "required_attributes": [],
+      "config_flag": "trace_consensus",
+      "note": "Context-propagated from the sending peer. No required local attributes."
+    },
+    {
+      "name": "consensus.validation.receive",
+      "category": "consensus",
+      "parent": null,
+      "required_attributes": [],
+      "config_flag": "trace_consensus",
+      "note": "Context-propagated from the sending peer. No required local attributes."
+    },
+    {
+      "name": "consensus.mode_change",
+      "category": "consensus",
+      "parent": null,
+      "required_attributes": ["mode_old", "mode_new"],
+      "config_flag": "trace_consensus",
+      "optional": true,
+      "note": "Only fires on an operating-mode transition; a steady cluster rarely changes mode after warmup."
+    },
+    {
+      "name": "ledger.build",
+      "category": "ledger",
+      "parent": null,
+      "required_attributes": [
+        "ledger_seq",
+        "close_time",
+        "close_time_correct",
+        "close_resolution_ms"
+      ],
+      "config_flag": "trace_ledger",
+      "note": "tx_count/tx_failed live on the child tx.apply span, not here."
+    },
+    {
+      "name": "ledger.validate",
+      "category": "ledger",
+      "parent": null,
+      "required_attributes": ["ledger_seq", "validations"],
+      "config_flag": "trace_ledger"
+    },
+    {
+      "name": "ledger.store",
+      "category": "ledger",
+      "parent": null,
+      "required_attributes": ["ledger_seq"],
+      "config_flag": "trace_ledger"
+    },
+    {
+      "name": "ledger.acquire",
+      "category": "ledger",
+      "parent": null,
+      "required_attributes": [
+        "ledger_seq",
+        "acquire_reason",
+        "timeouts",
+        "peer_count",
+        "outcome"
+      ],
+      "config_flag": "trace_ledger",
+      "optional": true,
+      "note": "Only fires when a node must fetch a missing ledger (InboundLedger). A healthy local cluster rarely back-fills history."
+    },
+    {
+      "name": "peer.proposal.receive",
+      "category": "peer",
+      "parent": null,
+      "required_attributes": ["peer_id", "proposal_trusted"],
+      "config_flag": "trace_peer"
+    },
+    {
+      "name": "peer.validation.receive",
+      "category": "peer",
+      "parent": null,
+      "required_attributes": [
+        "peer_id",
+        "validation_trusted",
+        "xrpl.ledger.hash",
+        "validation_full"
+      ],
+      "config_flag": "trace_peer",
+      "note": "Uses the shared dotted xrpl.ledger.hash constant (intentionally dotted, unlike consensus.validation.send)."
+    },
+    {
+      "name": "pathfind.request",
+      "category": "pathfind",
+      "parent": null,
+      "required_attributes": [
+        "pathfind_source_account",
+        "pathfind_dest_account"
+      ],
+      "config_flag": "trace_rpc",
+      "note": "Fires on ripple_path_find / path_find RPC. Driven by the ripple_path_find load in rpc_load_generator.py."
+    },
+    {
+      "name": "pathfind.compute",
+      "category": "pathfind",
+      "parent": "pathfind.request",
+      "required_attributes": ["pathfind_fast"],
+      "config_flag": "trace_rpc",
+      "optional": true,
+      "note": "Only fires when PathRequest::doUpdate runs a computation; the self-to-self XRP probe from the load generator returns early without computing paths in a fresh cluster with no liquidity."
+    },
+    {
+      "name": "pathfind.discover",
+      "category": "pathfind",
+      "parent": "pathfind.compute",
+      "required_attributes": ["pathfind_search_level", "pathfind_num_paths"],
+      "config_flag": "trace_rpc",
+      "optional": true,
+      "note": "Graph exploration; only fires under pathfind.compute, which needs real path liquidity not present in the fresh test cluster."
+    },
+    {
+      "name": "pathfind.update_all",
+      "category": "pathfind",
+      "parent": null,
+      "required_attributes": ["pathfind_ledger_index", "pathfind_num_requests"],
+      "config_flag": "trace_rpc",
+      "optional": true,
+      "note": "Async recomputation at ledger close; only fires when there are active path_find subscriptions (the one-shot ripple_path_find load does not register one)."
+    },
+    {
+      "name": "grpc.*",
+      "category": "grpc",
+      "parent": null,
+      "required_attributes": ["method", "grpc_role", "grpc_status"],
+      "config_flag": "trace_rpc",
+      "optional": true,
+      "note": "Wildcard — grpc.<MethodName>. The harness has no gRPC client, so these do not fire. Tracked for completeness."
+    }
+  ],
+  "parent_child_relationships": [
+    {
+      "parent": "rpc.ws_message",
+      "child": "rpc.process",
+      "description": "WebSocket message contains processing span",
+      "skip": true,
+      "skip_reason": "rpc.ws_message and rpc.process run on different threads (the WS handler posts a coroutine to JobQueue for processing). Span context is not propagated across the thread boundary. Requires a C++ fix to capture and forward the span context through the coroutine lambda."
+    },
+    {
+      "parent": "rpc.process",
+      "child": "rpc.command.*",
+      "description": "Processing span contains per-command span"
+    },
+    {
+      "parent": "ledger.build",
+      "child": "tx.apply",
+      "description": "Ledger build contains transaction application"
+    },
+    {
+      "parent": "consensus.round",
+      "child": "consensus.accept",
+      "description": "Consensus round contains the accept sub-span"
+    },
+    {
+      "parent": "consensus.accept",
+      "child": "consensus.accept.apply",
+      "description": "Accept contains the ledger-apply sub-span"
+    },
+    {
+      "parent": "pathfind.request",
+      "child": "pathfind.compute",
+      "description": "Pathfind request contains the compute sub-span",
+      "skip": true,
+      "skip_reason": "pathfind.compute only fires when a path computation actually runs; the self-to-self XRP probe in a fresh cluster with no liquidity returns before computing, so the child is not emitted under the harness workload."
+    }
+  ],
+  "total_span_types": 40,
+  "total_unique_attributes": 58
+}

docker/telemetry/workload/generate-validator-keys.sh

@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+# generate-validator-keys.sh — Generate validator key pairs for the workload harness.
+#
+# Uses a temporary standalone xrpld instance to call `validation_create` RPC
+# for each node. Outputs a JSON file mapping node index to seed + public key.
+#
+# Usage:
+#   ./generate-validator-keys.sh <xrpld_binary> <num_nodes> <output_dir>
+#
+# Output:
+#   <output_dir>/validator-keys.json — JSON array of {index, seed, public_key}
+#   <output_dir>/validators.txt     — [validators] section for xrpld.cfg
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Colored output helpers
+# ---------------------------------------------------------------------------
+log() { printf "\033[1;34m[KEYGEN]\033[0m %s\n" "$*"; }
+ok() { printf "\033[1;32m[KEYGEN]\033[0m %s\n" "$*"; }
+die() {
+    printf "\033[1;31m[KEYGEN]\033[0m %s\n" "$*" >&2
+    exit 1
+}
+
+# ---------------------------------------------------------------------------
+# Argument parsing
+# ---------------------------------------------------------------------------
+usage() {
+    echo "Usage: $0 <xrpld_binary> <num_nodes> <output_dir>"
+    echo ""
+    echo "Arguments:"
+    echo "  xrpld_binary  Path to xrpld binary (built with telemetry=ON)"
+    echo "  num_nodes     Number of validator key pairs to generate (1-20)"
+    echo "  output_dir    Directory to write validator-keys.json and validators.txt"
+    exit 1
+}
+
+if [ $# -lt 3 ]; then
+    usage
+fi
+
+XRPLD="$1"
+NUM_NODES="$2"
+OUTPUT_DIR="$3"
+
+# Validate arguments
+[ -x "$XRPLD" ] || die "xrpld binary not found or not executable: $XRPLD"
+[[ "$NUM_NODES" =~ ^[0-9]+$ ]] || die "num_nodes must be a positive integer"
+[ "$NUM_NODES" -ge 1 ] && [ "$NUM_NODES" -le 20 ] || die "num_nodes must be between 1 and 20"
+
+mkdir -p "$OUTPUT_DIR"
+
+# ---------------------------------------------------------------------------
+# Start a temporary standalone xrpld for key generation
+# ---------------------------------------------------------------------------
+TEMP_DIR="$(mktemp -d)"
+TEMP_PORT=5099
+TEMP_CFG="$TEMP_DIR/xrpld.cfg"
+
+log "Starting temporary xrpld for key generation (port $TEMP_PORT)..."
+
+cat >"$TEMP_CFG" <<EOCFG
+[server]
+port_rpc_keygen
+
+[port_rpc_keygen]
+port = $TEMP_PORT
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[node_db]
+type=NuDB
+path=$TEMP_DIR/nudb
+online_delete=256
+
+[database_path]
+$TEMP_DIR/db
+
+[debug_logfile]
+$TEMP_DIR/debug.log
+
+[ssl_verify]
+0
+EOCFG
+
+"$XRPLD" --conf "$TEMP_CFG" -a --start >"$TEMP_DIR/stdout.log" 2>&1 &
+TEMP_PID=$!
+
+# Ensure cleanup on exit
+cleanup_temp() {
+    kill "$TEMP_PID" 2>/dev/null || true
+    wait "$TEMP_PID" 2>/dev/null || true
+    rm -rf "$TEMP_DIR"
+}
+trap cleanup_temp EXIT
+
+# Wait for RPC to become available
+for attempt in $(seq 1 30); do
+    if curl -sf "http://localhost:$TEMP_PORT" \
+        -d '{"method":"server_info"}' >/dev/null 2>&1; then
+        log "Temporary xrpld RPC ready (attempt $attempt)."
+        break
+    fi
+    if [ "$attempt" -eq 30 ]; then
+        die "Temporary xrpld RPC not ready after 30s"
+    fi
+    sleep 1
+done
+
+# ---------------------------------------------------------------------------
+# Generate key pairs
+# ---------------------------------------------------------------------------
+log "Generating $NUM_NODES validator key pairs..."
+
+KEYS_JSON="["
+VALIDATORS_TXT="[validators]"
+
+for i in $(seq 1 "$NUM_NODES"); do
+    result=$(curl -sf "http://localhost:$TEMP_PORT" \
+        -d '{"method":"validation_create"}')
+    seed=$(echo "$result" | jq -r '.result.validation_seed')
+    pubkey=$(echo "$result" | jq -r '.result.validation_public_key')
+
+    if [ -z "$seed" ] || [ "$seed" = "null" ]; then
+        die "Failed to generate key pair for node $i"
+    fi
+
+    log "  Node $i: ${pubkey:0:20}..."
+
+    # Build JSON entry
+    entry="{\"index\": $i, \"seed\": \"$seed\", \"public_key\": \"$pubkey\"}"
+    if [ "$i" -gt 1 ]; then
+        KEYS_JSON="$KEYS_JSON,"
+    fi
+    KEYS_JSON="$KEYS_JSON$entry"
+
+    VALIDATORS_TXT="$VALIDATORS_TXT
+$pubkey"
+done
+
+KEYS_JSON="$KEYS_JSON]"
+
+# ---------------------------------------------------------------------------
+# Write output files
+# ---------------------------------------------------------------------------
+echo "$KEYS_JSON" | jq '.' >"$OUTPUT_DIR/validator-keys.json"
+echo "$VALIDATORS_TXT" >"$OUTPUT_DIR/validators.txt"
+
+ok "Generated $NUM_NODES key pairs:"
+ok "  Keys:       $OUTPUT_DIR/validator-keys.json"
+ok "  Validators: $OUTPUT_DIR/validators.txt"

docker/telemetry/workload/prom_queries.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""Shared Prometheus query helpers for the regression gate.
+
+Single source of truth for how regression metrics are computed. Both
+``capture_timings.py`` and any future tooling consume this module so metric
+name → PromQL expression stays consistent.
+
+Design:
+- Every captured metric has a key in the form ``{category}.{name}.p{quantile}``
+  (e.g. ``span.tx.process.p99``). Keys are flat strings so JSON diffing is
+  trivial.
+- Quantile queries go through ``histogram_quantile`` over the standard
+  ``_bucket`` series. The rate window is a parameter (defaults to the
+  capture window, not Prometheus's default 5m) so short CI runs are usable.
+- The catalogue of what to capture lives in ``regression-metrics.json`` —
+  this module only knows how to translate that JSON into HTTP queries.
+
+Usage::
+
+    import asyncio, aiohttp
+    from prom_queries import build_query_plan, run_query_plan
+
+    plan = build_query_plan("regression-metrics.json", window="3m")
+    async with aiohttp.ClientSession() as s:
+        timings = await run_query_plan(s, "http://localhost:9090", plan)
+    # timings = {"span.tx.process.p99": {"value": 12.4, "unit": "ms"}, ...}
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import aiohttp
+
+logger = logging.getLogger("prom_queries")
+
+
+@dataclass(frozen=True)
+class QueryEntry:
+    """One metric to capture from Prometheus.
+
+    Attributes:
+        key:      Flat output key, e.g. ``span.tx.process.p99``.
+        promql:   The PromQL expression to send to /api/v1/query.
+        unit:     Unit of the returned value, e.g. ``ms`` or ``us``.
+                  Baseline JSON preserves this so the comparator can
+                  sanity-check unit drift.
+    """
+
+    key: str
+    promql: str
+    unit: str
+
+
+def _build_simple_entries(
+    cfg: dict,
+    prefix: str,
+    window: str,
+) -> list[QueryEntry]:
+    """Build QueryEntry list for a single-template category (spans, rpc)."""
+    tmpl = cfg.get("_query_template", "")
+    unit = cfg.get("_unit", "ms")
+    entries: list[QueryEntry] = []
+    for name in cfg.get("names", []):
+        for q in cfg.get("_quantiles", []):
+            expr = (
+                tmpl.replace("{quantile}", _format_quantile(q))
+                .replace("{name}", name)
+                .replace("{window}", window)
+            )
+            entries.append(
+                QueryEntry(
+                    key=f"{prefix}.{name}.p{_quantile_label(q)}",
+                    promql=expr,
+                    unit=unit,
+                )
+            )
+    return entries
+
+
+def _build_job_entries(cfg: dict, window: str) -> list[QueryEntry]:
+    """Build QueryEntry list for the job_queue category (multi-phase)."""
+    unit = cfg.get("_unit", "us")
+    phases = cfg.get("_phases", ["queued", "running"])
+    tmpl_map = {
+        "queued": cfg.get("_queued_template", ""),
+        "running": cfg.get("_running_template", ""),
+    }
+    entries: list[QueryEntry] = []
+    for name in cfg.get("names", []):
+        for phase in phases:
+            tmpl = tmpl_map.get(phase, "")
+            if not tmpl:
+                continue
+            for q in cfg.get("_quantiles", []):
+                expr = (
+                    tmpl.replace("{quantile}", _format_quantile(q))
+                    .replace("{name}", name)
+                    .replace("{window}", window)
+                )
+                entries.append(
+                    QueryEntry(
+                        key=f"job.{name}.{phase}.p{_quantile_label(q)}",
+                        promql=expr,
+                        unit=unit,
+                    )
+                )
+    return entries
+
+
+def build_query_plan(metrics_path: str | Path, window: str = "3m") -> list[QueryEntry]:
+    """Translate regression-metrics.json into a list of PromQL queries.
+
+    Args:
+        metrics_path: Path to ``regression-metrics.json``.
+        window:       Rate window passed to ``rate()``. For short CI runs
+                      keep this close to the test duration so the bucket
+                      counts are meaningful. Default 3m matches the
+                      ``regression`` workload profile.
+
+    Returns:
+        A list of ``QueryEntry`` values, one per (metric × quantile).
+    """
+    with open(metrics_path) as f:
+        cfg = json.load(f)
+
+    plan: list[QueryEntry] = []
+    plan.extend(_build_simple_entries(cfg.get("spans", {}), "span", window))
+    plan.extend(_build_simple_entries(cfg.get("rpc_methods", {}), "rpc", window))
+    plan.extend(_build_job_entries(cfg.get("job_queue", {}), window))
+    return plan
+
+
+async def run_query_plan(
+    session: aiohttp.ClientSession,
+    prom_url: str,
+    plan: list[QueryEntry],
+) -> dict[str, dict[str, Any]]:
+    """Execute a query plan and return a flat ``key → {value, unit}`` map.
+
+    Queries that return no data (NaN, empty result) are still included in
+    the output with ``value: null`` — the comparator treats missing values
+    as "not yet observed" rather than as a regression. This keeps the
+    baseline schema stable across runs with different load levels.
+
+    Args:
+        session:  Shared aiohttp session.
+        prom_url: Base URL of Prometheus (e.g. ``http://localhost:9090``).
+        plan:     Output of :func:`build_query_plan`.
+
+    Returns:
+        Mapping from metric key to ``{"value": float|None, "unit": str}``.
+    """
+    results: dict[str, dict[str, Any]] = {}
+    for entry in plan:
+        value = await _instant_query(session, prom_url, entry.promql)
+        results[entry.key] = {"value": value, "unit": entry.unit}
+    return results
+
+
+async def _instant_query(
+    session: aiohttp.ClientSession,
+    prom_url: str,
+    promql: str,
+) -> float | None:
+    """POST an instant query to Prometheus; return the scalar value or None.
+
+    None is returned for NaN, empty results, or HTTP errors — every call
+    site treats None identically ("no data captured").
+    """
+    url = f"{prom_url.rstrip('/')}/api/v1/query"
+    try:
+        async with session.post(
+            url, data={"query": promql}, timeout=aiohttp.ClientTimeout(total=30)
+        ) as resp:
+            if resp.status != 200:
+                logger.warning("query HTTP %d: %s", resp.status, promql)
+                return None
+            body = await resp.json()
+    except (aiohttp.ClientError, TimeoutError) as exc:
+        logger.warning("query failed: %s — %s", promql, exc)
+        return None
+
+    if body.get("status") != "success":
+        logger.warning("query status=%s: %s", body.get("status"), promql)
+        return None
+
+    result = body.get("data", {}).get("result", [])
+    if not result:
+        return None
+
+    raw = result[0].get("value", [None, None])[1]
+    if raw is None or raw in ("NaN", "+Inf", "-Inf"):
+        return None
+    try:
+        return float(raw)
+    except (TypeError, ValueError):
+        return None
+
+
+def _format_quantile(q: float) -> str:
+    """Format a quantile for PromQL (``0.99`` → ``"0.99"``)."""
+    return f"{q:g}"
+
+
+def _quantile_label(q: float) -> str:
+    """Format a quantile for the output key (``0.95`` → ``"95"``)."""
+    return str(int(round(q * 100)))

docker/telemetry/workload/regression-metrics.json

@@ -0,0 +1,28 @@
+{
+  "_description": "Metric surface for the OTel-driven regression gate. Each entry names a metric, the quantiles to capture, and how to query Prometheus. The comparator compares current run against baseline-timings.json under these exact keys.",
+  "_key_format": "{category}.{name}.p{quantile}  (e.g. span.tx.process.p99, rpc.server_info.p95, job.transaction.queued.p95)",
+  "spans": {
+    "_query_template": "histogram_quantile({quantile}, sum by (le) (rate(traces_span_metrics_duration_milliseconds_bucket{span_name=\"{name}\"}[{window}])))",
+    "_unit": "ms",
+    "_quantiles": [0.5, 0.95, 0.99],
+    "names": [
+      "rpc.ws_message",
+      "rpc.process",
+      "tx.process",
+      "tx.apply",
+      "ledger.build",
+      "ledger.validate",
+      "ledger.store",
+      "consensus.ledger_close",
+      "consensus.accept"
+    ]
+  },
+  "job_queue": {
+    "_queued_template": "histogram_quantile({quantile}, sum by (le) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"{name}\"}[{window}])))",
+    "_running_template": "histogram_quantile({quantile}, sum by (le) (rate(xrpld_job_running_duration_us_bucket{job_type=\"{name}\"}[{window}])))",
+    "_unit": "us",
+    "_quantiles": [0.95],
+    "_phases": ["queued", "running"],
+    "names": ["transaction", "acceptLedger"]
+  }
+}

docker/telemetry/workload/regression-thresholds.json

@@ -0,0 +1,26 @@
+{
+  "_description": "Per-metric regression thresholds. A metric regresses when current - baseline exceeds BOTH the percentage and absolute bounds (AND, not OR — this tolerates small-value noise). Defaults apply unless a per-metric override exists.",
+  "_bucket_note": "SpanMetrics latency histograms use explicit buckets [1,5,10,25,50,100,250,500,1000,5000]ms. A quantile sitting near a low-end boundary can jump a full bucket (e.g. 1ms->5ms) between runs with no real change, so absolute span bounds are set to ~2 low-end bucket widths (10ms) to tolerate that quantization noise while still catching genuine multi-bucket regressions. The job_queue running bound is widened similarly — per-ledger apply work scales with TxQ burst load.",
+  "defaults": {
+    "span": {
+      "p50": { "max_pct_increase": 50.0, "max_abs_increase_ms": 10.0 },
+      "p95": { "max_pct_increase": 50.0, "max_abs_increase_ms": 10.0 },
+      "p99": { "max_pct_increase": 50.0, "max_abs_increase_ms": 15.0 }
+    },
+    "job_queue": {
+      "p95": { "max_pct_increase": 50.0, "max_abs_increase_us": 20000.0 }
+    }
+  },
+  "overrides": {
+    "span.consensus.ledger_close": {
+      "p50": { "max_pct_increase": 5.0, "max_abs_increase_ms": 200.0 },
+      "p95": { "max_pct_increase": 5.0, "max_abs_increase_ms": 500.0 },
+      "p99": { "max_pct_increase": 5.0, "max_abs_increase_ms": 1000.0 }
+    },
+    "span.consensus.accept": {
+      "p50": { "max_pct_increase": 5.0, "max_abs_increase_ms": 200.0 },
+      "p95": { "max_pct_increase": 5.0, "max_abs_increase_ms": 500.0 },
+      "p99": { "max_pct_increase": 5.0, "max_abs_increase_ms": 1000.0 }
+    }
+  }
+}

docker/telemetry/workload/requirements.txt

@@ -0,0 +1,6 @@
+# Python dependencies for Phase 10 workload tools.
+#
+# Install: pip install -r requirements.txt
+
+websockets>=12.0
+aiohttp>=3.9.0

docker/telemetry/workload/rpc_load_generator.py

@@ -0,0 +1,465 @@
+#!/usr/bin/env python3
+"""RPC Load Generator for rippled telemetry validation.
+
+Connects to one or more rippled WebSocket endpoints and fires all traced
+RPC commands at configurable rates with realistic production-like
+distribution.
+
+Command distribution (default weights):
+  40%  Health checks:   server_info, fee
+  30%  Wallet queries:  account_info, account_lines, account_objects
+  15%  Explorer:        ledger, ledger_data
+  10%  TX lookups:      tx, account_tx
+   5%  DEX queries:     book_offers, amm_info
+   3%  Pathfinding:     ripple_path_find
+
+Usage:
+    python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120
+
+    # Multiple endpoints (round-robin):
+    python3 rpc_load_generator.py \\
+        --endpoints ws://localhost:6006 ws://localhost:6007 \\
+        --rate 100 --duration 300
+
+    # Custom weights:
+    python3 rpc_load_generator.py --endpoints ws://localhost:6006 \\
+        --weights '{"server_info":60,"account_info":30,"ledger":10}'
+"""
+
+import argparse
+import asyncio
+import json
+import logging
+import random
+import sys
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any
+
+import websockets
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+# Default command distribution matching realistic production ratios.
+# Keys are RPC command names; values are relative weights.
+DEFAULT_WEIGHTS: dict[str, int] = {
+    # 40% health checks
+    "server_info": 25,
+    "fee": 15,
+    # 30% wallet queries
+    "account_info": 15,
+    "account_lines": 8,
+    "account_objects": 7,
+    # 15% explorer
+    "ledger": 10,
+    "ledger_data": 5,
+    # 10% tx lookups
+    "tx": 5,
+    "account_tx": 5,
+    # 5% DEX queries
+    "book_offers": 3,
+    "amm_info": 2,
+    # Pathfinding — exercises the pathfind.request/compute/discover spans.
+    # ripple_path_find is the synchronous (one-shot) variant that fits this
+    # fire-one-request WS client; path_find is a streaming subscription.
+    "ripple_path_find": 3,
+}
+
+# Well-known genesis account for queries that require an account parameter.
+GENESIS_ACCOUNT = "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"
+
+logger = logging.getLogger("rpc_load_generator")
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class LoadStats:
+    """Tracks request counts and latencies during a load run.
+
+    Attributes:
+        total_sent:     Total RPC requests dispatched.
+        total_success:  Requests that returned a valid result.
+        total_errors:   Requests that returned an error or timed out.
+        latencies:      Per-command list of round-trip times in seconds.
+        command_counts: Per-command request count.
+    """
+
+    total_sent: int = 0
+    total_success: int = 0
+    total_errors: int = 0
+    latencies: dict[str, list[float]] = field(default_factory=dict)
+    command_counts: dict[str, int] = field(default_factory=dict)
+
+    def record(self, command: str, latency: float, success: bool) -> None:
+        """Record the outcome of a single RPC call."""
+        self.total_sent += 1
+        if success:
+            self.total_success += 1
+        else:
+            self.total_errors += 1
+        self.latencies.setdefault(command, []).append(latency)
+        self.command_counts[command] = self.command_counts.get(command, 0) + 1
+
+    def summary(self) -> dict[str, Any]:
+        """Return a summary dict suitable for JSON serialization."""
+        per_command: dict[str, Any] = {}
+        for cmd, lats in self.latencies.items():
+            sorted_lats = sorted(lats)
+            n = len(sorted_lats)
+            per_command[cmd] = {
+                "count": self.command_counts.get(cmd, 0),
+                "p50_ms": round(sorted_lats[n // 2] * 1000, 2) if n else 0,
+                "p95_ms": (round(sorted_lats[int(n * 0.95)] * 1000, 2) if n else 0),
+                "p99_ms": (round(sorted_lats[int(n * 0.99)] * 1000, 2) if n else 0),
+            }
+        return {
+            "total_sent": self.total_sent,
+            "total_success": self.total_success,
+            "total_errors": self.total_errors,
+            "error_rate_pct": (
+                round(self.total_errors / self.total_sent * 100, 2)
+                if self.total_sent
+                else 0
+            ),
+            "per_command": per_command,
+        }
+
+
+# ---------------------------------------------------------------------------
+# RPC command builders
+# ---------------------------------------------------------------------------
+
+
+def build_rpc_request(command: str) -> dict[str, Any]:
+    """Build a native WebSocket command request for the given command.
+
+    Uses rippled's native WS format (``{"command": ...}``) with flat
+    parameters, NOT the JSON-RPC format (``{"method": ..., "params": [...]}``).
+
+    Args:
+        command: The rippled RPC command name.
+
+    Returns:
+        A dict representing the native WebSocket request body.
+    """
+    req: dict[str, Any] = {"command": command}
+
+    if command in ("server_info", "fee"):
+        pass  # No params needed.
+    elif command == "account_info":
+        req["account"] = GENESIS_ACCOUNT
+    elif command == "account_lines":
+        req["account"] = GENESIS_ACCOUNT
+    elif command == "account_objects":
+        req["account"] = GENESIS_ACCOUNT
+        req["limit"] = 10
+    elif command == "ledger":
+        req["ledger_index"] = "validated"
+    elif command == "ledger_data":
+        req["ledger_index"] = "validated"
+        req["limit"] = 5
+    elif command == "tx":
+        # Use a dummy hash — returns "txnNotFound" error but still exercises
+        # the full RPC span pipeline (rpc.ws_message -> rpc.process -> rpc.command.tx).
+        req["transaction"] = "0" * 64
+        req["binary"] = False
+    elif command == "account_tx":
+        req["account"] = GENESIS_ACCOUNT
+        req["ledger_index_min"] = -1
+        req["ledger_index_max"] = -1
+        req["limit"] = 5
+    elif command == "book_offers":
+        req["taker_pays"] = {"currency": "XRP"}
+        req["taker_gets"] = {
+            "currency": "USD",
+            "issuer": GENESIS_ACCOUNT,
+        }
+        req["limit"] = 5
+    elif command == "amm_info":
+        # AMM may not exist — the span is still created on the server side.
+        req["asset"] = {"currency": "XRP"}
+        req["asset2"] = {
+            "currency": "USD",
+            "issuer": GENESIS_ACCOUNT,
+        }
+    elif command == "ripple_path_find":
+        # Self-to-self XRP path search. It returns no usable paths, but the
+        # server still runs the full pathfinding pipeline (pathfind.request ->
+        # pathfind.compute -> pathfind.discover), which is what we trace.
+        req["source_account"] = GENESIS_ACCOUNT
+        req["destination_account"] = GENESIS_ACCOUNT
+        req["destination_amount"] = "1000000"  # 1 XRP in drops
+
+    return req
+
+
+def choose_command(weights: dict[str, int]) -> str:
+    """Select a random RPC command based on configured weights.
+
+    Args:
+        weights: Mapping of command name to relative weight.
+
+    Returns:
+        A command name string.
+    """
+    commands = list(weights.keys())
+    w = [weights[c] for c in commands]
+    return random.choices(commands, weights=w, k=1)[0]
+
+
+# ---------------------------------------------------------------------------
+# WebSocket RPC client
+# ---------------------------------------------------------------------------
+
+
+async def send_rpc(
+    ws: websockets.WebSocketClientProtocol,
+    command: str,
+    stats: LoadStats,
+    inject_traceparent: bool = True,
+) -> None:
+    """Send a single RPC request over WebSocket and record the result.
+
+    Args:
+        ws:                  Open WebSocket connection.
+        command:             RPC command name.
+        stats:               LoadStats instance to record results.
+        inject_traceparent:  If True, add a W3C traceparent header field
+                             to the request for context propagation testing.
+    """
+    request = build_rpc_request(command)
+
+    # Inject W3C traceparent for context propagation testing.
+    # The rippled WebSocket handler extracts this from the JSON body
+    # when present (Phase 2 context propagation).
+    if inject_traceparent:
+        trace_id = uuid.uuid4().hex
+        span_id = uuid.uuid4().hex[:16]
+        request["traceparent"] = f"00-{trace_id}-{span_id}-01"
+
+    t0 = time.monotonic()
+    try:
+        await ws.send(json.dumps(request))
+        raw = await asyncio.wait_for(ws.recv(), timeout=10.0)
+        latency = time.monotonic() - t0
+        response = json.loads(raw)
+        # Native WS responses have {"status": "success", "result": {...}}
+        # or {"status": "error", "error": "...", "error_message": "..."}.
+        success = response.get("status") == "success"
+        stats.record(command, latency, success)
+    except (asyncio.TimeoutError, websockets.exceptions.WebSocketException) as exc:
+        latency = time.monotonic() - t0
+        stats.record(command, latency, False)
+        logger.debug("RPC %s failed: %s", command, exc)
+
+
+async def run_load(
+    endpoints: list[str],
+    rate: float,
+    duration: float,
+    weights: dict[str, int],
+    inject_traceparent: bool,
+) -> LoadStats:
+    """Run the RPC load generator against the given endpoints.
+
+    Distributes requests round-robin across endpoints at the specified
+    rate (requests per second) for the given duration.
+
+    Args:
+        endpoints:          List of WebSocket URLs (ws://host:port).
+        rate:               Target requests per second.
+        duration:           Total run time in seconds.
+        weights:            Command distribution weights.
+        inject_traceparent: Whether to inject W3C traceparent headers.
+
+    Returns:
+        LoadStats with aggregated results.
+    """
+    stats = LoadStats()
+    interval = 1.0 / rate if rate > 0 else 0.1
+
+    # Open persistent connections to all endpoints.
+    connections: list[websockets.WebSocketClientProtocol] = []
+    for ep in endpoints:
+        try:
+            ws = await websockets.connect(ep, ping_interval=20, ping_timeout=10)
+            connections.append(ws)
+            logger.info("Connected to %s", ep)
+        except Exception as exc:
+            logger.error("Failed to connect to %s: %s", ep, exc)
+
+    if not connections:
+        logger.error("No connections established. Aborting.")
+        return stats
+
+    logger.info(
+        "Starting load: rate=%s RPS, duration=%ss, endpoints=%d",
+        rate,
+        duration,
+        len(connections),
+    )
+
+    start = time.monotonic()
+    conn_idx = 0
+
+    try:
+        while (time.monotonic() - start) < duration:
+            command = choose_command(weights)
+            ws = connections[conn_idx % len(connections)]
+            conn_idx += 1
+
+            # Fire-and-forget style with bounded concurrency via sleep.
+            asyncio.create_task(send_rpc(ws, command, stats, inject_traceparent))
+            await asyncio.sleep(interval)
+
+            # Periodic progress log.
+            elapsed = time.monotonic() - start
+            if stats.total_sent % 100 == 0 and stats.total_sent > 0:
+                actual_rps = stats.total_sent / elapsed if elapsed > 0 else 0
+                logger.info(
+                    "Progress: %d sent, %d errors, %.1f RPS (%.0fs elapsed)",
+                    stats.total_sent,
+                    stats.total_errors,
+                    actual_rps,
+                    elapsed,
+                )
+    except asyncio.CancelledError:
+        logger.info("Load generation cancelled.")
+    finally:
+        # Allow in-flight requests to complete.
+        await asyncio.sleep(2)
+        for ws in connections:
+            await ws.close()
+
+    elapsed = time.monotonic() - start
+    logger.info(
+        "Load complete: %d sent, %d success, %d errors in %.1fs (%.1f RPS)",
+        stats.total_sent,
+        stats.total_success,
+        stats.total_errors,
+        elapsed,
+        stats.total_sent / elapsed if elapsed > 0 else 0,
+    )
+
+    return stats
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="RPC Load Generator for rippled telemetry validation",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Basic usage (50 RPS for 2 minutes):
+  python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120
+
+  # Multiple endpoints with custom weights:
+  python3 rpc_load_generator.py \\
+      --endpoints ws://localhost:6006 ws://localhost:6007 \\
+      --rate 100 --duration 300 \\
+      --weights '{"server_info": 80, "account_info": 20}'
+        """,
+    )
+    parser.add_argument(
+        "--endpoints",
+        nargs="+",
+        default=["ws://localhost:6006"],
+        help="WebSocket endpoints (default: ws://localhost:6006)",
+    )
+    parser.add_argument(
+        "--rate",
+        type=float,
+        default=50.0,
+        help="Target requests per second (default: 50)",
+    )
+    parser.add_argument(
+        "--duration",
+        type=float,
+        default=120.0,
+        help="Run duration in seconds (default: 120)",
+    )
+    parser.add_argument(
+        "--weights",
+        type=str,
+        default=None,
+        help="JSON string of command weights (overrides defaults)",
+    )
+    parser.add_argument(
+        "--no-traceparent",
+        action="store_true",
+        help="Disable W3C traceparent injection",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default=None,
+        help="Write JSON summary to this file path",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable debug logging",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Main entry point for the RPC load generator."""
+    args = parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s [%(name)s] %(levelname)s %(message)s",
+    )
+
+    # Parse custom weights if provided.
+    weights = DEFAULT_WEIGHTS.copy()
+    if args.weights:
+        try:
+            custom = json.loads(args.weights)
+            weights = {k: int(v) for k, v in custom.items()}
+            logger.info("Using custom weights: %s", weights)
+        except (json.JSONDecodeError, ValueError) as exc:
+            logger.error("Invalid --weights JSON: %s", exc)
+            sys.exit(1)
+
+    # Run the load generator.
+    stats = asyncio.run(
+        run_load(
+            endpoints=args.endpoints,
+            rate=args.rate,
+            duration=args.duration,
+            weights=weights,
+            inject_traceparent=not args.no_traceparent,
+        )
+    )
+
+    summary = stats.summary()
+    print(json.dumps(summary, indent=2))
+
+    if args.output:
+        with open(args.output, "w") as f:
+            json.dump(summary, f, indent=2)
+        logger.info("Summary written to %s", args.output)
+
+    # Exit with error if error rate exceeds 50%.
+    if summary["error_rate_pct"] > 50:
+        logger.error("High error rate: %.1f%%", summary["error_rate_pct"])
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

docker/telemetry/workload/run-full-validation.sh

@@ -0,0 +1,499 @@
+#!/usr/bin/env bash
+# run-full-validation.sh — Orchestrates the full telemetry validation pipeline.
+#
+# Sequence:
+#   1. Start the observability stack (OTel Collector, Tempo, Prometheus, Loki, Grafana)
+#   2. Start a multi-node rippled cluster with full telemetry enabled
+#   3. Wait for consensus
+#   4. Run workload orchestrator (RPC load, TX submission, propagation wait)
+#   5. Run the telemetry validation suite
+#   6. Capture OTel timings and compare against committed baseline
+#   7. (Optional) Run the performance overhead benchmark
+#
+# Usage:
+#   ./run-full-validation.sh --xrpld /path/to/xrpld
+#   ./run-full-validation.sh --xrpld /path/to/xrpld --with-benchmark
+#   ./run-full-validation.sh --xrpld /path/to/xrpld --skip-regression
+#   ./run-full-validation.sh --cleanup
+#
+# Exit codes:
+#   0 — All validation checks and the regression gate passed
+#   1 — Validation checks failed OR the regression gate detected a regression
+#   2 — Infrastructure error (cluster/stack failed to start, timing capture failed)
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Colored output helpers
+# ---------------------------------------------------------------------------
+log() { printf "\033[1;34m[VALIDATE]\033[0m %s\n" "$*"; }
+ok() { printf "\033[1;32m[VALIDATE]\033[0m %s\n" "$*"; }
+warn() { printf "\033[1;33m[VALIDATE]\033[0m %s\n" "$*"; }
+fail() { printf "\033[1;31m[VALIDATE]\033[0m %s\n" "$*"; }
+die() {
+    printf "\033[1;31m[VALIDATE]\033[0m %s\n" "$*" >&2
+    exit 2
+}
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TELEMETRY_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+REPO_ROOT="$(cd "$TELEMETRY_DIR/../.." && pwd)"
+COMPOSE_FILE="$TELEMETRY_DIR/docker-compose.workload.yaml"
+WORKDIR="/tmp/xrpld-validation"
+
+XRPLD="${XRPLD:-$REPO_ROOT/.build/xrpld}"
+NUM_NODES=5
+RPC_PORT_BASE=5005
+WS_PORT_BASE=6006
+PEER_PORT_BASE=51235
+RPC_RATE=50
+RPC_DURATION=120
+TX_TPS=5
+TX_DURATION=120
+WITH_BENCHMARK=false
+SKIP_LOKI=false
+SKIP_REGRESSION=false
+WORKLOAD_PROFILE="full-validation"
+REPORT_DIR="$WORKDIR/reports"
+# Rate window handed to Prometheus `rate()` when capturing timings. Keep
+# this close to the active workload duration so histogram buckets cover
+# the measurement window; longer windows dilute short-lived regressions.
+REGRESSION_WINDOW="${REGRESSION_WINDOW:-3m}"
+BASELINE_FILE="${BASELINE_FILE:-$SCRIPT_DIR/baselines/baseline-timings.json}"
+THRESHOLDS_FILE="${THRESHOLDS_FILE:-$SCRIPT_DIR/regression-thresholds.json}"
+METRICS_FILE="${METRICS_FILE:-$SCRIPT_DIR/regression-metrics.json}"
+
+GENESIS_ACCOUNT="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"
+GENESIS_SEED="snoPBrXtMeMyMHUVTgbuqAfg1SUTb"
+
+# ---------------------------------------------------------------------------
+# Argument parsing
+# ---------------------------------------------------------------------------
+usage() {
+    echo "Usage: $0 [OPTIONS]"
+    echo ""
+    echo "Options:"
+    echo "  --xrpld PATH         Path to xrpld binary"
+    echo "  --nodes NUM          Number of validator nodes (default: 5)"
+    echo "  --rpc-rate RPS       RPC load rate (default: 50)"
+    echo "  --rpc-duration SECS  RPC load duration (default: 120)"
+    echo "  --tx-tps TPS         Transaction submit rate (default: 5)"
+    echo "  --tx-duration SECS   Transaction submit duration (default: 120)"
+    echo "  --profile NAME       Workload profile (default: full-validation)"
+    echo "  --with-benchmark     Also run performance overhead benchmark (telemetry off vs on)"
+    echo "  --skip-loki          Skip Loki log-trace correlation checks"
+    echo "  --skip-regression    Skip the OTel-baseline regression gate"
+    echo "  --cleanup            Tear down everything and exit"
+    echo "  -h, --help           Show this help"
+    exit 0
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --xrpld)
+            XRPLD="$2"
+            shift 2
+            ;;
+        --nodes)
+            NUM_NODES="$2"
+            shift 2
+            ;;
+        --rpc-rate)
+            RPC_RATE="$2"
+            shift 2
+            ;;
+        --rpc-duration)
+            RPC_DURATION="$2"
+            shift 2
+            ;;
+        --tx-tps)
+            TX_TPS="$2"
+            shift 2
+            ;;
+        --tx-duration)
+            TX_DURATION="$2"
+            shift 2
+            ;;
+        --profile)
+            WORKLOAD_PROFILE="$2"
+            shift 2
+            ;;
+        --with-benchmark)
+            WITH_BENCHMARK=true
+            shift
+            ;;
+        --skip-loki)
+            SKIP_LOKI=true
+            shift
+            ;;
+        --skip-regression)
+            SKIP_REGRESSION=true
+            shift
+            ;;
+        --cleanup) # Cleanup mode
+            log "Cleaning up..."
+            pkill -f "$WORKDIR" 2>/dev/null || true
+            docker compose -f "$COMPOSE_FILE" down 2>/dev/null || true
+            rm -rf "$WORKDIR"
+            ok "Cleanup complete."
+            exit 0
+            ;;
+        -h | --help) usage ;;
+        *) die "Unknown option: $1" ;;
+    esac
+done
+
+# ---------------------------------------------------------------------------
+# Prerequisites
+# ---------------------------------------------------------------------------
+log "Checking prerequisites..."
+[ -x "$XRPLD" ] || die "xrpld binary not found: $XRPLD"
+command -v docker >/dev/null 2>&1 || die "docker not found"
+docker compose version >/dev/null 2>&1 || die "docker compose (v2) not found"
+command -v python3 >/dev/null 2>&1 || die "python3 not found"
+command -v curl >/dev/null 2>&1 || die "curl not found"
+command -v jq >/dev/null 2>&1 || die "jq not found"
+[ -f "$COMPOSE_FILE" ] || die "docker-compose.workload.yaml not found"
+
+# Install Python dependencies.
+log "Installing Python dependencies..."
+pip3 install -q -r "$SCRIPT_DIR/requirements.txt" 2>/dev/null ||
+    pip install -q -r "$SCRIPT_DIR/requirements.txt" 2>/dev/null ||
+    warn "Could not install Python dependencies — they may already be present"
+
+ok "Prerequisites verified."
+
+# ---------------------------------------------------------------------------
+# Cleanup previous run
+# ---------------------------------------------------------------------------
+log "Cleaning up previous run..."
+pkill -f "$WORKDIR" 2>/dev/null || true
+sleep 2
+rm -rf "$WORKDIR"
+mkdir -p "$WORKDIR" "$REPORT_DIR"
+
+# ---------------------------------------------------------------------------
+# Step 1: Start observability stack
+# ---------------------------------------------------------------------------
+log "Step 1: Starting observability stack..."
+docker compose -f "$COMPOSE_FILE" up -d
+
+log "Waiting for OTel Collector..."
+for attempt in $(seq 1 30); do
+    status=$(curl -so /dev/null -w '%{http_code}' http://localhost:4318/ 2>/dev/null || echo 000)
+    if [ "$status" != "000" ]; then
+        ok "OTel Collector ready (attempt $attempt)"
+        break
+    fi
+    [ "$attempt" -eq 30 ] && die "OTel Collector not ready after 30s"
+    sleep 1
+done
+
+log "Waiting for Tempo..."
+for attempt in $(seq 1 30); do
+    if curl -sf "http://localhost:3200/ready" >/dev/null 2>&1; then
+        ok "Tempo ready (attempt $attempt)"
+        break
+    fi
+    [ "$attempt" -eq 30 ] && die "Tempo not ready after 30s"
+    sleep 1
+done
+
+log "Waiting for Prometheus..."
+for attempt in $(seq 1 30); do
+    if curl -sf "http://localhost:9090/-/healthy" >/dev/null 2>&1; then
+        ok "Prometheus ready (attempt $attempt)"
+        break
+    fi
+    [ "$attempt" -eq 30 ] && die "Prometheus not ready after 30s"
+    sleep 1
+done
+
+# ---------------------------------------------------------------------------
+# Step 2: Generate validator keys and start cluster
+# ---------------------------------------------------------------------------
+log "Step 2: Starting $NUM_NODES-node validator cluster..."
+
+bash "$SCRIPT_DIR/generate-validator-keys.sh" "$XRPLD" "$NUM_NODES" "$WORKDIR"
+
+for i in $(seq 1 "$NUM_NODES"); do
+    NODE_DIR="$WORKDIR/node$i"
+    mkdir -p "$NODE_DIR/nudb" "$NODE_DIR/db"
+
+    RPC_PORT=$((RPC_PORT_BASE + i - 1))
+    WS_PORT=$((WS_PORT_BASE + i - 1))
+    PEER_PORT=$((PEER_PORT_BASE + i - 1))
+    SEED=$(jq -r ".[$((i - 1))].seed" "$WORKDIR/validator-keys.json")
+
+    # Build ips_fixed.
+    IPS_FIXED=""
+    for j in $(seq 1 "$NUM_NODES"); do
+        if [ "$j" -ne "$i" ]; then
+            IPS_FIXED="${IPS_FIXED}127.0.0.1 $((PEER_PORT_BASE + j - 1))
+"
+        fi
+    done
+
+    cat >"$NODE_DIR/xrpld.cfg" <<EOCFG
+[server]
+port_rpc
+port_ws
+port_peer
+
+[port_rpc]
+port = $RPC_PORT
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[port_ws]
+port = $WS_PORT
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = ws
+
+[port_peer]
+port = $PEER_PORT
+ip = 0.0.0.0
+protocol = peer
+
+[node_db]
+type=NuDB
+path=$NODE_DIR/nudb
+online_delete=256
+
+[database_path]
+$NODE_DIR/db
+
+[debug_logfile]
+$NODE_DIR/debug.log
+
+[validation_seed]
+$SEED
+
+[validators_file]
+$WORKDIR/validators.txt
+
+[ips]
+${IPS_FIXED}
+
+[telemetry]
+enabled=1
+service_instance_id=validator-${i}
+endpoint=http://localhost:4318/v1/traces
+exporter=otlp_http
+sampling_ratio=1.0
+batch_size=512
+batch_delay_ms=2000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1
+
+[insight]
+# Native OTel metrics via OTLP/HTTP. The collector has no StatsD receiver
+# (metrics pipeline is [otlp, spanmetrics]), so beast::insight must export
+# over OTLP for system metrics to reach Prometheus. prefix=xrpld matches the
+# OTel resource service name and the xrpld_* names the dashboards query.
+server=otel
+endpoint=http://localhost:4318/v1/metrics
+prefix=xrpld
+
+[rpc_startup]
+{ "command": "log_level", "severity": "warning" }
+
+[signing_support]
+true
+
+[ssl_verify]
+0
+EOCFG
+
+    "$XRPLD" --conf "$NODE_DIR/xrpld.cfg" --start >"$NODE_DIR/stdout.log" 2>&1 &
+    echo $! >"$NODE_DIR/xrpld.pid"
+    log "  Node $i: RPC=$RPC_PORT WS=$WS_PORT Peer=$PEER_PORT PID=$!"
+done
+
+# ---------------------------------------------------------------------------
+# Step 3: Wait for consensus
+# ---------------------------------------------------------------------------
+log "Step 3: Waiting for consensus..."
+for attempt in $(seq 1 120); do
+    ready=0
+    for i in $(seq 1 "$NUM_NODES"); do
+        port=$((RPC_PORT_BASE + i - 1))
+        state=$(curl -sf "http://localhost:$port" \
+            -d '{"method":"server_info"}' 2>/dev/null |
+            jq -r '.result.info.server_state' 2>/dev/null || echo "")
+        if [ "$state" = "proposing" ]; then
+            ready=$((ready + 1))
+        fi
+    done
+    if [ "$ready" -ge "$NUM_NODES" ]; then
+        ok "All $NUM_NODES nodes proposing (attempt $attempt)"
+        break
+    fi
+    if [ "$attempt" -eq 120 ]; then
+        warn "Consensus timeout — $ready/$NUM_NODES nodes ready"
+    fi
+    printf "\r  %d/%d nodes proposing..." "$ready" "$NUM_NODES"
+    sleep 1
+done
+echo ""
+
+# Wait for first validated ledger.
+log "Waiting for validated ledger..."
+for attempt in $(seq 1 60); do
+    val_seq=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
+        -d '{"method":"server_info"}' 2>/dev/null |
+        jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
+    if [ "$val_seq" -gt 2 ] 2>/dev/null; then
+        ok "Validated ledger: seq $val_seq"
+        break
+    fi
+    [ "$attempt" -eq 60 ] && warn "No validated ledger after 60s"
+    sleep 1
+done
+
+# ---------------------------------------------------------------------------
+# Step 4: Run workload orchestrator
+# ---------------------------------------------------------------------------
+log "Step 4: Running workload orchestrator (profile: $WORKLOAD_PROFILE)..."
+
+WS_ENDPOINTS=""
+for i in $(seq 1 "$NUM_NODES"); do
+    WS_ENDPOINTS="$WS_ENDPOINTS ws://localhost:$((WS_PORT_BASE + i - 1))"
+done
+
+python3 "$SCRIPT_DIR/workload_orchestrator.py" \
+    --profile "$WORKLOAD_PROFILE" \
+    --endpoints $WS_ENDPOINTS \
+    --report "$REPORT_DIR/workload-report.json" \
+    --report-dir "$REPORT_DIR" ||
+    warn "Workload orchestrator returned non-zero exit"
+
+ok "Workload orchestration complete."
+
+# ---------------------------------------------------------------------------
+# Step 5: Run telemetry validation suite
+# ---------------------------------------------------------------------------
+log "Step 5: Running telemetry validation suite..."
+
+VALIDATION_ARGS="--report $REPORT_DIR/validation-report.json"
+if [ "$SKIP_LOKI" = true ]; then
+    VALIDATION_ARGS="$VALIDATION_ARGS --skip-loki"
+fi
+
+VALIDATION_EXIT=0
+python3 "$SCRIPT_DIR/validate_telemetry.py" $VALIDATION_ARGS || VALIDATION_EXIT=$?
+
+if [ "$VALIDATION_EXIT" -eq 0 ]; then
+    ok "All telemetry validation checks passed!"
+else
+    fail "Some telemetry validation checks failed (exit $VALIDATION_EXIT)"
+fi
+
+# ---------------------------------------------------------------------------
+# Step 6: Capture OTel timings and run the regression comparison
+# ---------------------------------------------------------------------------
+# This step ALWAYS captures timings (so CI always has an artifact from which
+# to bootstrap/refresh the committed baseline). The comparator then either:
+#   - prints the paste-me JSON when the baseline is a placeholder, or
+#   - enforces thresholds and fails the run on regression.
+# Use --skip-regression to opt out (e.g. for ad-hoc local exploration).
+TIMINGS_FILE="$REPORT_DIR/timings.json"
+REGRESSION_REPORT="$REPORT_DIR/regression-report.json"
+REGRESSION_EXIT=0
+
+if [ "$SKIP_REGRESSION" != true ]; then
+    log "Step 6: Capturing OTel timings from Prometheus..."
+    if python3 "$SCRIPT_DIR/capture_timings.py" \
+        --prometheus "http://localhost:9090" \
+        --metrics "$METRICS_FILE" \
+        --output "$TIMINGS_FILE" \
+        --window "$REGRESSION_WINDOW" \
+        --profile "$WORKLOAD_PROFILE"; then
+        ok "Timings captured: $TIMINGS_FILE"
+    else
+        fail "Failed to capture timings — skipping regression comparison."
+        REGRESSION_EXIT=2
+        SKIP_REGRESSION=true
+    fi
+fi
+
+if [ "$SKIP_REGRESSION" != true ]; then
+    log "Comparing against baseline $BASELINE_FILE..."
+    python3 "$SCRIPT_DIR/compare_to_baseline.py" \
+        --timings "$TIMINGS_FILE" \
+        --baseline "$BASELINE_FILE" \
+        --thresholds "$THRESHOLDS_FILE" \
+        --report "$REGRESSION_REPORT" || REGRESSION_EXIT=$?
+    if [ "$REGRESSION_EXIT" -eq 0 ]; then
+        ok "Regression gate passed (or baseline placeholder — paste JSON printed above)."
+    elif [ "$REGRESSION_EXIT" -eq 1 ]; then
+        fail "Regression detected — see $REGRESSION_REPORT"
+    else
+        fail "Regression comparator internal error (exit $REGRESSION_EXIT)"
+    fi
+else
+    warn "Regression gate skipped."
+fi
+
+# ---------------------------------------------------------------------------
+# Step 7: (Optional) Run overhead benchmark
+# ---------------------------------------------------------------------------
+if [ "$WITH_BENCHMARK" = true ]; then
+    log "Step 7: Running performance benchmark..."
+    bash "$SCRIPT_DIR/benchmark.sh" \
+        --xrpld "$XRPLD" \
+        --duration 120 \
+        --nodes 3 \
+        --output "$REPORT_DIR" ||
+        warn "Benchmark returned non-zero exit"
+fi
+
+# ---------------------------------------------------------------------------
+# Summary
+# ---------------------------------------------------------------------------
+echo ""
+echo "==========================================================="
+echo "  FULL VALIDATION RESULTS"
+echo "==========================================================="
+echo ""
+echo "  Reports directory: $REPORT_DIR"
+echo ""
+ls -la "$REPORT_DIR/" 2>/dev/null || true
+echo ""
+echo "  Observability stack is running:"
+echo "    Tempo:         http://localhost:3200"
+echo "    Grafana:       http://localhost:3000"
+echo "    Prometheus:    http://localhost:9090"
+echo ""
+echo "  xrpld nodes ($NUM_NODES) are running:"
+for i in $(seq 1 "$NUM_NODES"); do
+    rpc=$((RPC_PORT_BASE + i - 1))
+    ws=$((WS_PORT_BASE + i - 1))
+    pid=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null || echo 'unknown')
+    echo "    Node $i: RPC=$rpc WS=$ws PID=$pid"
+done
+echo ""
+echo "  To tear down:"
+echo "    $0 --cleanup"
+echo ""
+echo "==========================================================="
+
+# Fail the run if EITHER validation or the regression gate failed. The
+# `[ "$VAR" -gt N ]` comparison works here because exit codes are numeric.
+FINAL_EXIT=0
+if [ "$VALIDATION_EXIT" -ne 0 ]; then
+    FINAL_EXIT="$VALIDATION_EXIT"
+fi
+if [ "$REGRESSION_EXIT" -ne 0 ] && [ "$FINAL_EXIT" -eq 0 ]; then
+    FINAL_EXIT="$REGRESSION_EXIT"
+fi
+exit "$FINAL_EXIT"

docker/telemetry/workload/test_accounts.json

@@ -0,0 +1,42 @@
+{
+  "genesis": {
+    "account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh",
+    "seed": "snoPBrXtMeMyMHUVTgbuqAfg1SUTb",
+    "description": "Genesis account with all XRP. Used to fund test accounts."
+  },
+  "test_accounts": [
+    {
+      "name": "alice",
+      "description": "Primary sender for Payment and OfferCreate transactions."
+    },
+    {
+      "name": "bob",
+      "description": "Primary receiver for Payment transactions."
+    },
+    {
+      "name": "carol",
+      "description": "TrustSet and issued currency counterparty."
+    },
+    {
+      "name": "dave",
+      "description": "NFToken operations (mint, offer, accept)."
+    },
+    {
+      "name": "eve",
+      "description": "Escrow operations (create, finish)."
+    },
+    {
+      "name": "frank",
+      "description": "AMM pool operations (create, deposit, withdraw)."
+    },
+    {
+      "name": "grace",
+      "description": "Additional sender for parallel transaction submission."
+    },
+    {
+      "name": "heidi",
+      "description": "Additional receiver for payment diversity."
+    }
+  ],
+  "note": "Test account keypairs are generated dynamically at runtime via wallet_propose RPC. This file defines the logical roles. Actual keys are stored in the workdir during execution."
+}

docker/telemetry/workload/tx_submitter.py

@@ -0,0 +1,848 @@
+#!/usr/bin/env python3
+"""Transaction Submitter for rippled telemetry validation.
+
+Generates diverse transaction types against a rippled cluster to exercise
+the full span and metric surface: tx.process, tx.apply, ledger.build,
+consensus.*, and all associated attributes.
+
+Pre-funds test accounts from the genesis account, then submits a
+configurable mix of transaction types at a target TPS.
+
+Supported transaction types:
+  - Payment (XRP and issued currencies)
+  - OfferCreate / OfferCancel (DEX activity)
+  - TrustSet (trust line creation)
+  - NFTokenMint / NFTokenCreateOffer / NFTokenAcceptOffer
+  - EscrowCreate / EscrowFinish
+  - AMMCreate / AMMDeposit / AMMWithdraw (if amendment enabled)
+
+Usage:
+    python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120
+
+    # Custom transaction mix:
+    python3 tx_submitter.py --endpoint ws://localhost:6006 \\
+        --weights '{"Payment":50,"OfferCreate":20,"TrustSet":10,"NFTokenMint":10,"EscrowCreate":10}'
+"""
+
+import argparse
+import asyncio
+import json
+import logging
+import random
+import sys
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import websockets
+
+logger = logging.getLogger("tx_submitter")
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+GENESIS_ACCOUNT = "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"
+GENESIS_SEED = "snoPBrXtMeMyMHUVTgbuqAfg1SUTb"
+
+# Amount to fund each test account (100,000 XRP in drops).
+FUND_AMOUNT = "100000000000"
+
+# Default transaction mix weights (relative).
+DEFAULT_TX_WEIGHTS: dict[str, int] = {
+    "Payment": 40,
+    "OfferCreate": 15,
+    "OfferCancel": 5,
+    "TrustSet": 10,
+    "NFTokenMint": 10,
+    "NFTokenCreateOffer": 5,
+    "EscrowCreate": 5,
+    "EscrowFinish": 5,
+    "AMMCreate": 3,
+    "AMMDeposit": 2,
+}
+
+# Number of test accounts to create.
+NUM_TEST_ACCOUNTS = 8
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class Account:
+    """Represents a funded XRPL test account.
+
+    Attributes:
+        name:      Human-readable name (e.g., "alice").
+        account:   Classic address (rXXX...).
+        seed:      Secret seed for signing.
+        sequence:  Next available sequence number.
+    """
+
+    name: str
+    account: str
+    seed: str
+    sequence: int = 0
+
+
+@dataclass
+class TxStats:
+    """Tracks transaction submission results.
+
+    Attributes:
+        total_submitted: Total transactions sent to the network.
+        total_success:   Transactions that returned tesSUCCESS or terQUEUED.
+        total_errors:    Transactions that returned an error engine_result.
+        by_type:         Per-transaction-type count of submissions.
+        errors_by_type:  Per-transaction-type count of errors.
+    """
+
+    total_submitted: int = 0
+    total_success: int = 0
+    total_errors: int = 0
+    by_type: dict[str, int] = field(default_factory=dict)
+    errors_by_type: dict[str, int] = field(default_factory=dict)
+
+    def record(self, tx_type: str, success: bool) -> None:
+        """Record the result of a transaction submission."""
+        self.total_submitted += 1
+        self.by_type[tx_type] = self.by_type.get(tx_type, 0) + 1
+        if success:
+            self.total_success += 1
+        else:
+            self.total_errors += 1
+            self.errors_by_type[tx_type] = self.errors_by_type.get(tx_type, 0) + 1
+
+    def summary(self) -> dict[str, Any]:
+        """Return a summary dict suitable for JSON serialization."""
+        return {
+            "total_submitted": self.total_submitted,
+            "total_success": self.total_success,
+            "total_errors": self.total_errors,
+            "success_rate_pct": (
+                round(self.total_success / self.total_submitted * 100, 2)
+                if self.total_submitted
+                else 0
+            ),
+            "by_type": self.by_type,
+            "errors_by_type": self.errors_by_type,
+        }
+
+
+# ---------------------------------------------------------------------------
+# WebSocket RPC helpers
+# ---------------------------------------------------------------------------
+
+
+async def ws_request(
+    ws: websockets.WebSocketClientProtocol,
+    command: str,
+    params: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Send a native WebSocket command and return the result payload.
+
+    Uses rippled's native WebSocket format (``command`` key with flat
+    parameters).  The response has ``status`` at the top level and the
+    actual data payload inside ``result``.  This helper unwraps the
+    ``result`` dict so callers can read fields directly.
+
+    Args:
+        ws:      Open WebSocket connection.
+        command: RPC command name (e.g., ``account_info``, ``submit``).
+        params:  Optional flat parameter dict merged into the request.
+
+    Returns:
+        The inner ``result`` dict from the response.
+
+    Raises:
+        RuntimeError: If the request fails or times out.
+    """
+    request: dict[str, Any] = {"command": command}
+    if params:
+        request.update(params)
+    await ws.send(json.dumps(request))
+    raw = await asyncio.wait_for(ws.recv(), timeout=30.0)
+    resp = json.loads(raw)
+
+    # WS command format: {"status": "success", "result": {...}, "type": "response"}
+    # On error: {"status": "error", "error": "...", "error_message": "..."}
+    if resp.get("status") == "error":
+        logger.warning(
+            "%s error: %s — %s",
+            command,
+            resp.get("error", "unknown"),
+            resp.get("error_message", ""),
+        )
+    return resp.get("result", resp)
+
+
+async def create_account(ws: websockets.WebSocketClientProtocol, name: str) -> Account:
+    """Create a new account via wallet_propose RPC.
+
+    Args:
+        ws:   Open WebSocket connection.
+        name: Human-readable name for the account.
+
+    Returns:
+        An Account instance with the generated keypair.
+    """
+    result = await ws_request(ws, "wallet_propose")
+    if "account_id" not in result:
+        raise RuntimeError(
+            f"wallet_propose failed: {json.dumps(result, indent=None)[:300]}"
+        )
+    return Account(
+        name=name,
+        account=result["account_id"],
+        seed=result["master_seed"],
+    )
+
+
+async def fund_account(
+    ws: websockets.WebSocketClientProtocol,
+    dest: Account,
+    genesis_seq: int,
+) -> tuple[bool, int]:
+    """Fund a test account from genesis.
+
+    Args:
+        ws:          Open WebSocket connection.
+        dest:        Destination account to fund.
+        genesis_seq: Current genesis account sequence number.
+
+    Returns:
+        Tuple of (success: bool, next_sequence: int).
+    """
+    resp = await ws_request(
+        ws,
+        "submit",
+        {
+            "secret": GENESIS_SEED,
+            "tx_json": {
+                "TransactionType": "Payment",
+                "Account": GENESIS_ACCOUNT,
+                "Destination": dest.account,
+                "Amount": FUND_AMOUNT,
+                "Sequence": genesis_seq,
+            },
+        },
+    )
+    engine_result = resp.get("engine_result", "unknown")
+    success = engine_result in ("tesSUCCESS", "terQUEUED")
+    if not success:
+        # Log the full response to help diagnose submit failures in CI.
+        logger.warning(
+            "Fund %s failed: engine_result=%s, full response: %s",
+            dest.name,
+            engine_result,
+            json.dumps(resp, indent=None)[:500],
+        )
+    return success, genesis_seq + 1
+
+
+async def get_account_sequence(
+    ws: websockets.WebSocketClientProtocol, account: str
+) -> int:
+    """Get the current sequence number for an account.
+
+    Args:
+        ws:      Open WebSocket connection.
+        account: Classic address.
+
+    Returns:
+        Current sequence number.
+    """
+    resp = await ws_request(ws, "account_info", {"account": account})
+    if "account_data" not in resp:
+        # Log full response to diagnose WS API format issues.
+        logger.warning(
+            "account_info for %s: no account_data, full response: %s",
+            account[:12],
+            json.dumps(resp, indent=None)[:500],
+        )
+        return 0
+    return resp["account_data"].get("Sequence", 0)
+
+
+# ---------------------------------------------------------------------------
+# Transaction builders
+# ---------------------------------------------------------------------------
+
+
+def build_payment(sender: Account, receiver: Account) -> dict[str, Any]:
+    """Build an XRP Payment transaction.
+
+    Args:
+        sender:   Source account.
+        receiver: Destination account.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    amount = str(random.randint(1000, 1000000))  # 0.001 - 1 XRP
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "Payment",
+            "Account": sender.account,
+            "Destination": receiver.account,
+            "Amount": amount,
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_offer_create(sender: Account) -> dict[str, Any]:
+    """Build an OfferCreate transaction (XRP/USD pair).
+
+    Args:
+        sender: Account placing the offer.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "OfferCreate",
+            "Account": sender.account,
+            "TakerPays": str(random.randint(100000, 10000000)),
+            "TakerGets": {
+                "currency": "USD",
+                "issuer": GENESIS_ACCOUNT,
+                "value": str(round(random.uniform(0.1, 100.0), 2)),
+            },
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_offer_cancel(sender: Account) -> dict[str, Any]:
+    """Build an OfferCancel transaction.
+
+    Uses a non-existent offer sequence — will fail gracefully but still
+    exercises the tx.process span pipeline.
+
+    Args:
+        sender: Account cancelling the offer.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "OfferCancel",
+            "Account": sender.account,
+            "OfferSequence": max(1, sender.sequence - 1),
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_trust_set(sender: Account) -> dict[str, Any]:
+    """Build a TrustSet transaction for a USD trust line.
+
+    Args:
+        sender: Account setting the trust line.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "TrustSet",
+            "Account": sender.account,
+            "LimitAmount": {
+                "currency": "USD",
+                "issuer": GENESIS_ACCOUNT,
+                "value": "1000000",
+            },
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_nftoken_mint(sender: Account) -> dict[str, Any]:
+    """Build an NFTokenMint transaction.
+
+    Args:
+        sender: Account minting the NFT.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "NFTokenMint",
+            "Account": sender.account,
+            "NFTokenTaxon": random.randint(0, 100),
+            "Flags": 8,  # tfTransferable
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_nftoken_create_offer(sender: Account) -> dict[str, Any]:
+    """Build an NFTokenCreateOffer transaction.
+
+    Uses a dummy NFTokenID — will fail but exercises the span pipeline.
+
+    Args:
+        sender: Account creating the NFT offer.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "NFTokenCreateOffer",
+            "Account": sender.account,
+            "NFTokenID": "0" * 64,
+            "Amount": str(random.randint(100000, 1000000)),
+            "Flags": 1,  # tfSellNFToken
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_escrow_create(sender: Account, receiver: Account) -> dict[str, Any]:
+    """Build an EscrowCreate transaction.
+
+    Creates a time-based escrow that finishes 10 seconds from now.
+
+    Args:
+        sender:   Account creating the escrow.
+        receiver: Destination account for escrow funds.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    # Ripple epoch offset: 946684800 seconds from Unix epoch
+    ripple_time = int(time.time()) - 946684800
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "EscrowCreate",
+            "Account": sender.account,
+            "Destination": receiver.account,
+            "Amount": str(random.randint(100000, 1000000)),
+            "FinishAfter": ripple_time + 10,
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_escrow_finish(sender: Account, owner: Account) -> dict[str, Any]:
+    """Build an EscrowFinish transaction.
+
+    Uses a dummy offer sequence — will likely fail but exercises spans.
+
+    Args:
+        sender: Account finishing the escrow.
+        owner:  Account that created the escrow.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "EscrowFinish",
+            "Account": sender.account,
+            "Owner": owner.account,
+            "OfferSequence": max(1, owner.sequence - 2),
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_amm_create(sender: Account) -> dict[str, Any]:
+    """Build an AMMCreate transaction (XRP/USD pool).
+
+    Requires the AMM amendment to be enabled on the network.
+
+    Args:
+        sender: Account creating the AMM pool.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "AMMCreate",
+            "Account": sender.account,
+            "Amount": str(random.randint(10000000, 100000000)),
+            "Amount2": {
+                "currency": "USD",
+                "issuer": GENESIS_ACCOUNT,
+                "value": str(round(random.uniform(10.0, 1000.0), 2)),
+            },
+            "TradingFee": 500,  # 0.5%
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_amm_deposit(sender: Account) -> dict[str, Any]:
+    """Build an AMMDeposit transaction.
+
+    Args:
+        sender: Account depositing into the AMM pool.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "AMMDeposit",
+            "Account": sender.account,
+            "Asset": {"currency": "XRP"},
+            "Asset2": {
+                "currency": "USD",
+                "issuer": GENESIS_ACCOUNT,
+            },
+            "Amount": str(random.randint(1000000, 10000000)),
+            "Flags": 0x00080000,  # tfSingleAsset
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+# Transaction type -> builder function mapping.
+# Each builder takes (accounts: list[Account]) and returns submit params.
+TX_BUILDERS: dict[str, Any] = {
+    "Payment": lambda accts: build_payment(accts[0], accts[1]),
+    "OfferCreate": lambda accts: build_offer_create(accts[0]),
+    "OfferCancel": lambda accts: build_offer_cancel(accts[0]),
+    "TrustSet": lambda accts: build_trust_set(accts[2]),
+    "NFTokenMint": lambda accts: build_nftoken_mint(accts[3]),
+    "NFTokenCreateOffer": lambda accts: build_nftoken_create_offer(accts[3]),
+    "EscrowCreate": lambda accts: build_escrow_create(accts[4], accts[1]),
+    "EscrowFinish": lambda accts: build_escrow_finish(accts[4], accts[4]),
+    "AMMCreate": lambda accts: build_amm_create(accts[5]),
+    "AMMDeposit": lambda accts: build_amm_deposit(accts[5]),
+}
+
+
+# ---------------------------------------------------------------------------
+# Main submission loop
+# ---------------------------------------------------------------------------
+
+
+async def setup_accounts(
+    ws: websockets.WebSocketClientProtocol,
+) -> list[Account]:
+    """Create and fund test accounts from genesis.
+
+    Generates NUM_TEST_ACCOUNTS accounts via wallet_propose, then funds
+    each with FUND_AMOUNT XRP from genesis.
+
+    Args:
+        ws: Open WebSocket connection to a rippled node.
+
+    Returns:
+        List of funded Account instances.
+    """
+    account_names = ["alice", "bob", "carol", "dave", "eve", "frank", "grace", "heidi"]
+
+    logger.info("Creating %d test accounts...", NUM_TEST_ACCOUNTS)
+    accounts: list[Account] = []
+    for name in account_names[:NUM_TEST_ACCOUNTS]:
+        acct = await create_account(ws, name)
+        accounts.append(acct)
+        logger.info("  Created %s: %s", name, acct.account)
+
+    # Get genesis sequence.
+    genesis_seq = await get_account_sequence(ws, GENESIS_ACCOUNT)
+    logger.info("Genesis sequence: %d", genesis_seq)
+
+    # Fund all accounts.
+    logger.info("Funding test accounts...")
+    for acct in accounts:
+        success, genesis_seq = await fund_account(ws, acct, genesis_seq)
+        if success:
+            logger.info("  Funded %s", acct.name)
+        else:
+            logger.warning("  Failed to fund %s", acct.name)
+
+    # Wait for funding transactions to be validated.
+    logger.info("Waiting 10s for funding transactions to validate...")
+    await asyncio.sleep(10)
+
+    # Refresh sequence numbers for all accounts.
+    for acct in accounts:
+        try:
+            acct.sequence = await get_account_sequence(ws, acct.account)
+            logger.info("  %s sequence: %d", acct.name, acct.sequence)
+        except Exception as exc:
+            logger.warning("  Failed to get sequence for %s: %s", acct.name, exc)
+
+    return accounts
+
+
+async def submit_transaction(
+    ws: websockets.WebSocketClientProtocol,
+    tx_type: str,
+    accounts: list[Account],
+    stats: TxStats,
+) -> None:
+    """Submit a single transaction of the given type.
+
+    Selects the appropriate builder, constructs the transaction, submits
+    it via the submit RPC, and records the result.
+
+    Args:
+        ws:       Open WebSocket connection.
+        tx_type:  Transaction type name (e.g., "Payment").
+        accounts: List of funded test accounts.
+        stats:    TxStats instance to record results.
+    """
+    builder = TX_BUILDERS.get(tx_type)
+    if not builder:
+        logger.warning("Unknown transaction type: %s", tx_type)
+        return
+
+    try:
+        params = builder(accounts)
+        # Identify which account is the sender to bump its sequence.
+        sender_addr = params["tx_json"]["Account"]
+        sender = next((a for a in accounts if a.account == sender_addr), None)
+
+        resp = await ws_request(ws, "submit", params)
+        engine_result = resp.get("engine_result", "unknown")
+        success = engine_result in (
+            "tesSUCCESS",
+            "terQUEUED",
+            "tecUNFUNDED_OFFER",
+            "tecNO_DST_INSUF_XRP",
+        )
+        stats.record(tx_type, success)
+
+        if sender:
+            sender.sequence += 1
+
+        if not success:
+            logger.debug(
+                "%s result: %s (%s)",
+                tx_type,
+                engine_result,
+                resp.get("engine_result_message", ""),
+            )
+    except Exception as exc:
+        stats.record(tx_type, False)
+        logger.debug("%s error: %s", tx_type, exc)
+
+
+async def _refresh_sequences(
+    ws: websockets.WebSocketClientProtocol,
+    accounts: list[Account],
+) -> None:
+    """Re-sync account sequences from the validated ledger.
+
+    In a consensus network, other nodes' transactions advance sequences
+    beyond the submitter's local tracking.  Refreshing every ~10 s keeps
+    the local counter close to the ledger and prevents tefPAST_SEQ storms.
+    """
+    for acct in accounts:
+        try:
+            seq = await get_account_sequence(ws, acct.account)
+            if seq > acct.sequence:
+                acct.sequence = seq
+        except Exception:
+            pass
+
+
+async def run_submitter(
+    endpoint: str,
+    tps: float,
+    duration: float,
+    weights: dict[str, int],
+) -> TxStats:
+    """Run the transaction submitter against a single endpoint.
+
+    Args:
+        endpoint: WebSocket URL (ws://host:port).
+        tps:      Target transactions per second.
+        duration: Total run time in seconds.
+        weights:  Transaction type distribution weights.
+
+    Returns:
+        TxStats with aggregated results.
+    """
+    stats = TxStats()
+    interval = 1.0 / tps if tps > 0 else 0.5
+
+    ws = await websockets.connect(endpoint, ping_interval=20, ping_timeout=10)
+    logger.info("Connected to %s", endpoint)
+
+    try:
+        # Setup test accounts.
+        accounts = await setup_accounts(ws)
+        if len(accounts) < 6:
+            logger.error("Need at least 6 funded accounts, got %d", len(accounts))
+            return stats
+
+        # Build weighted command list.
+        tx_types = list(weights.keys())
+        tx_weights = [weights[t] for t in tx_types]
+
+        logger.info(
+            "Starting TX submission: tps=%s, duration=%ss, types=%d",
+            tps,
+            duration,
+            len(tx_types),
+        )
+
+        start = time.monotonic()
+        last_seq_refresh = start
+        seq_refresh_interval = 10.0
+        while (time.monotonic() - start) < duration:
+            # Periodically re-sync account sequences from the ledger so
+            # locally-tracked sequences don't drift behind consensus.
+            if (time.monotonic() - last_seq_refresh) >= seq_refresh_interval:
+                await _refresh_sequences(ws, accounts)
+                last_seq_refresh = time.monotonic()
+
+            tx_type = random.choices(tx_types, weights=tx_weights, k=1)[0]
+            await submit_transaction(ws, tx_type, accounts, stats)
+            await asyncio.sleep(interval)
+
+            # Progress logging every 50 transactions.
+            if stats.total_submitted % 50 == 0 and stats.total_submitted > 0:
+                elapsed = time.monotonic() - start
+                actual_tps = stats.total_submitted / elapsed if elapsed > 0 else 0
+                logger.info(
+                    "Progress: %d submitted, %d success, %d errors, "
+                    "%.1f TPS (%.0fs elapsed)",
+                    stats.total_submitted,
+                    stats.total_success,
+                    stats.total_errors,
+                    actual_tps,
+                    elapsed,
+                )
+
+    finally:
+        await ws.close()
+
+    elapsed = time.monotonic() - start
+    logger.info(
+        "Submission complete: %d submitted, %d success, %d errors "
+        "in %.1fs (%.1f TPS)",
+        stats.total_submitted,
+        stats.total_success,
+        stats.total_errors,
+        elapsed,
+        stats.total_submitted / elapsed if elapsed > 0 else 0,
+    )
+
+    return stats
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Transaction Submitter for rippled telemetry validation",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Basic usage (5 TPS for 2 minutes):
+  python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120
+
+  # Custom transaction mix:
+  python3 tx_submitter.py --endpoint ws://localhost:6006 \\
+      --weights '{"Payment": 60, "OfferCreate": 20, "TrustSet": 20}'
+        """,
+    )
+    parser.add_argument(
+        "--endpoint",
+        type=str,
+        default="ws://localhost:6006",
+        help="WebSocket endpoint (default: ws://localhost:6006)",
+    )
+    parser.add_argument(
+        "--tps",
+        type=float,
+        default=5.0,
+        help="Target transactions per second (default: 5)",
+    )
+    parser.add_argument(
+        "--duration",
+        type=float,
+        default=120.0,
+        help="Run duration in seconds (default: 120)",
+    )
+    parser.add_argument(
+        "--weights",
+        type=str,
+        default=None,
+        help="JSON string of transaction type weights (overrides defaults)",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default=None,
+        help="Write JSON summary to this file path",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable debug logging",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Main entry point for the transaction submitter."""
+    args = parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s [%(name)s] %(levelname)s %(message)s",
+    )
+
+    # Parse custom weights if provided.
+    weights = DEFAULT_TX_WEIGHTS.copy()
+    if args.weights:
+        try:
+            custom = json.loads(args.weights)
+            weights = {k: int(v) for k, v in custom.items()}
+            logger.info("Using custom weights: %s", weights)
+        except (json.JSONDecodeError, ValueError) as exc:
+            logger.error("Invalid --weights JSON: %s", exc)
+            sys.exit(1)
+
+    # Run the submitter.
+    stats = asyncio.run(
+        run_submitter(
+            endpoint=args.endpoint,
+            tps=args.tps,
+            duration=args.duration,
+            weights=weights,
+        )
+    )
+
+    summary = stats.summary()
+    print(json.dumps(summary, indent=2))
+
+    if args.output:
+        with open(args.output, "w") as f:
+            json.dump(summary, f, indent=2)
+        logger.info("Summary written to %s", args.output)
+
+
+if __name__ == "__main__":
+    main()

docker/telemetry/workload/workload-profiles.json

@@ -0,0 +1,108 @@
+{
+  "profiles": {
+    "full-validation": {
+      "description": "Full 18-dashboard coverage with burst/idle/plateau patterns",
+      "phases": [
+        {
+          "name": "warmup",
+          "description": "Low load to populate baseline gauges and node health metrics",
+          "duration_sec": 30,
+          "rpc": {
+            "rate": 5,
+            "weights": { "server_info": 50, "fee": 30, "ledger": 20 }
+          },
+          "tx": null
+        },
+        {
+          "name": "steady-state",
+          "description": "Medium sustained load — plateau data for all dashboards",
+          "duration_sec": 60,
+          "rpc": { "rate": 30 },
+          "tx": { "tps": 3 }
+        },
+        {
+          "name": "rpc-burst",
+          "description": "Heavy RPC to saturate job queue and spike latency",
+          "duration_sec": 30,
+          "rpc": { "rate": 100 },
+          "tx": null
+        },
+        {
+          "name": "tx-flood",
+          "description": "High TX rate for fee escalation and TxQ pressure",
+          "duration_sec": 30,
+          "rpc": { "rate": 5, "weights": { "server_info": 50, "fee": 50 } },
+          "tx": {
+            "tps": 20,
+            "weights": { "Payment": 70, "OfferCreate": 20, "TrustSet": 10 }
+          }
+        },
+        {
+          "name": "txq-burst",
+          "description": "Single-type Payment burst at high TPS to force open-ledger fee escalation and TxQ queueing, exercising the txq.* spans (txq.enqueue / txq.accept / txq.accept.tx / txq.cleanup)",
+          "duration_sec": 30,
+          "rpc": { "rate": 5, "weights": { "fee": 100 } },
+          "tx": {
+            "tps": 60,
+            "weights": { "Payment": 100 }
+          }
+        },
+        {
+          "name": "mixed-peak",
+          "description": "Realistic peak load — consensus and ledger ops under stress",
+          "duration_sec": 60,
+          "rpc": { "rate": 50 },
+          "tx": { "tps": 10 }
+        },
+        {
+          "name": "cooldown",
+          "description": "Low load for recovery metrics and state transition data",
+          "duration_sec": 30,
+          "rpc": { "rate": 5, "weights": { "server_info": 80, "fee": 20 } },
+          "tx": null
+        }
+      ],
+      "propagation_wait_sec": 60
+    },
+    "quick-smoke": {
+      "description": "Fast smoke test — minimal data for CI quick checks",
+      "phases": [
+        {
+          "name": "smoke",
+          "description": "Single phase covering all generator types",
+          "duration_sec": 30,
+          "rpc": { "rate": 20 },
+          "tx": { "tps": 3 }
+        }
+      ],
+      "propagation_wait_sec": 30
+    },
+    "stress": {
+      "description": "Heavy sustained load for performance benchmarking",
+      "phases": [
+        {
+          "name": "ramp-up",
+          "description": "Gradually increasing load",
+          "duration_sec": 30,
+          "rpc": { "rate": 20 },
+          "tx": { "tps": 5 }
+        },
+        {
+          "name": "peak",
+          "description": "Maximum sustained load",
+          "duration_sec": 120,
+          "rpc": { "rate": 150 },
+          "tx": { "tps": 25 }
+        },
+        {
+          "name": "sustain",
+          "description": "Continued high load for stability check",
+          "duration_sec": 60,
+          "rpc": { "rate": 100 },
+          "tx": { "tps": 15 }
+        }
+      ],
+      "propagation_wait_sec": 60
+    }
+  }
+}

docker/telemetry/workload/workload_orchestrator.py

@@ -0,0 +1,503 @@
+#!/usr/bin/env python3
+"""Workload Orchestrator for rippled telemetry validation.
+
+Reads a named profile from workload-profiles.json and executes sequential
+load phases, each with configurable RPC and TX parameters.  Produces a
+combined report with per-phase results.
+
+Phases run sequentially.  Within each phase, the RPC load generator and
+transaction submitter run concurrently (if both are configured).
+
+Orchestration Flow::
+
+    workload-profiles.json
+           |
+           v
+    workload_orchestrator.py
+           |
+      +----+----+----+----+----+----+
+      | Phase 1 | Phase 2 | ...... | Phase N |
+      +----+----+----+----+----+----+
+           |              |
+      +----+----+    +----+----+
+      | rpc_load |    | tx_sub  |   (concurrent within phase)
+      | _gen.py  |    | mitter  |
+      +----+----+    +----+----+
+           |              |
+           v              v
+      per-phase JSON reports
+           |
+           v
+      combined-report.json
+
+Usage:
+    python3 workload_orchestrator.py --profile full-validation
+    python3 workload_orchestrator.py --profile quick-smoke --endpoints ws://localhost:6006
+    python3 workload_orchestrator.py --profile stress --report /tmp/report.json
+
+Profiles are defined in workload-profiles.json in the same directory.
+"""
+
+import argparse
+import asyncio
+import json
+import logging
+import sys
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("workload_orchestrator")
+
+SCRIPT_DIR = Path(__file__).parent.resolve()
+PROFILES_FILE = SCRIPT_DIR / "workload-profiles.json"
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class PhaseResult:
+    """Result of a single workload phase.
+
+    Attributes:
+        name:         Phase name from the profile.
+        duration_sec: Configured duration.
+        actual_sec:   Actual elapsed time.
+        rpc_summary:  JSON summary from rpc_load_generator, or None.
+        tx_summary:   JSON summary from tx_submitter, or None.
+        errors:       List of error messages from subprocess failures.
+    """
+
+    name: str
+    duration_sec: int
+    actual_sec: float = 0.0
+    rpc_summary: dict[str, Any] | None = None
+    tx_summary: dict[str, Any] | None = None
+    errors: list[str] = field(default_factory=list)
+
+
+# ---------------------------------------------------------------------------
+# Profile loading
+# ---------------------------------------------------------------------------
+
+
+def load_profile(profile_name: str) -> dict[str, Any]:
+    """Load a named profile from workload-profiles.json.
+
+    Args:
+        profile_name: Key in the profiles dict (e.g., "full-validation").
+
+    Returns:
+        The profile dict with phases and propagation_wait_sec.
+
+    Raises:
+        SystemExit: If the profile file or name is not found.
+    """
+    if not PROFILES_FILE.exists():
+        logger.error("Profiles file not found: %s", PROFILES_FILE)
+        sys.exit(2)
+
+    with open(PROFILES_FILE) as f:
+        data = json.load(f)
+
+    profiles = data.get("profiles", {})
+    if profile_name not in profiles:
+        available = ", ".join(profiles.keys())
+        logger.error("Profile '%s' not found. Available: %s", profile_name, available)
+        sys.exit(2)
+
+    profile = profiles[profile_name]
+
+    # Validate profile schema — fail fast on bad config.
+    phases = profile.get("phases", [])
+    if not isinstance(phases, list) or not phases:
+        logger.error("Profile '%s' has no valid phases", profile_name)
+        sys.exit(2)
+    for i, phase in enumerate(phases):
+        if not isinstance(phase.get("name"), str):
+            logger.error("Phase %d missing valid 'name'", i)
+            sys.exit(2)
+        if (
+            not isinstance(phase.get("duration_sec"), (int, float))
+            or phase["duration_sec"] <= 0
+        ):
+            logger.error(
+                "Phase %d '%s' has invalid duration_sec",
+                i,
+                phase.get("name"),
+            )
+            sys.exit(2)
+
+    logger.info(
+        "Loaded profile '%s': %s (%d phases)",
+        profile_name,
+        profile.get("description", ""),
+        len(phases),
+    )
+    return profile
+
+
+# ---------------------------------------------------------------------------
+# Subprocess execution
+# ---------------------------------------------------------------------------
+
+
+async def run_subprocess(cmd: list[str], label: str) -> tuple[int, str, str]:
+    """Run a subprocess and capture its stdout and stderr.
+
+    Args:
+        cmd:   Command and arguments.
+        label: Human-readable label for logging.
+
+    Returns:
+        Tuple of (return_code, stdout_text, stderr_text).
+    """
+    logger.debug("Starting %s: %s", label, " ".join(cmd))
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate()
+    if proc.returncode != 0:
+        logger.warning(
+            "%s exited with code %d: %s",
+            label,
+            proc.returncode,
+            stderr.decode().strip()[-500:],
+        )
+    return proc.returncode, stdout.decode(), stderr.decode()
+
+
+# ---------------------------------------------------------------------------
+# Phase execution
+# ---------------------------------------------------------------------------
+
+
+def _collect_task_result(
+    label: str,
+    returncode: int,
+    stderr: str,
+    report_path: Path,
+    result: PhaseResult,
+) -> None:
+    """Process the result of a completed subprocess task.
+
+    Reads the JSON report file (if it exists) and records any errors.
+
+    Args:
+        label:       "rpc" or "tx".
+        returncode:  Subprocess exit code.
+        stderr:      Captured stderr text.
+        report_path: Path to the JSON report file.
+        result:      PhaseResult to update.
+    """
+    if report_path.exists():
+        try:
+            with open(report_path) as f:
+                summary = json.load(f)
+            if label == "rpc":
+                result.rpc_summary = summary
+            elif label == "tx":
+                result.tx_summary = summary
+        except (json.JSONDecodeError, OSError) as exc:
+            logger.warning("Failed to parse %s report %s: %s", label, report_path, exc)
+            result.errors.append(f"Failed to parse {label} report: {exc}")
+
+    if returncode != 0:
+        snippet = stderr.strip()[-200:] if stderr else ""
+        result.errors.append(
+            f"{label.upper()} generator exited with {returncode}: {snippet}"
+        )
+
+
+def _build_rpc_cmd(
+    endpoints: list[str], rpc_cfg: dict[str, Any], duration: int, output: Path
+) -> list[str]:
+    """Build the command list for the RPC load generator subprocess."""
+    cmd = [
+        sys.executable,
+        str(SCRIPT_DIR / "rpc_load_generator.py"),
+        "--endpoints",
+        *endpoints,
+        "--rate",
+        str(rpc_cfg.get("rate", 50)),
+        "--duration",
+        str(duration),
+        "--output",
+        str(output),
+    ]
+    weights = rpc_cfg.get("weights")
+    if weights:
+        cmd.extend(["--weights", json.dumps(weights)])
+    return cmd
+
+
+def _build_tx_cmd(
+    endpoint: str, tx_cfg: dict[str, Any], duration: int, output: Path
+) -> list[str]:
+    """Build the command list for the TX submitter subprocess."""
+    cmd = [
+        sys.executable,
+        str(SCRIPT_DIR / "tx_submitter.py"),
+        "--endpoint",
+        endpoint,
+        "--tps",
+        str(tx_cfg.get("tps", 5)),
+        "--duration",
+        str(duration),
+        "--output",
+        str(output),
+    ]
+    weights = tx_cfg.get("weights")
+    if weights:
+        cmd.extend(["--weights", json.dumps(weights)])
+    return cmd
+
+
+async def run_phase(
+    phase: dict[str, Any],
+    endpoints: list[str],
+    report_dir: Path,
+    phase_idx: int,
+) -> PhaseResult:
+    """Execute a single workload phase.
+
+    Launches rpc_load_generator.py and/or tx_submitter.py as subprocesses
+    based on the phase configuration.  Both run concurrently if configured.
+
+    Args:
+        phase:      Phase dict from the profile.
+        endpoints:  List of WebSocket endpoint URLs.
+        report_dir: Directory for per-phase JSON reports.
+        phase_idx:  Phase index (for file naming).
+
+    Returns:
+        PhaseResult with subprocess outputs.
+    """
+    name = phase["name"]
+    duration = phase["duration_sec"]
+    result = PhaseResult(name=name, duration_sec=duration)
+    prefix = f"phase{phase_idx + 1}-{name}"
+
+    logger.info(
+        "=== Phase %d: %s (%ds) — %s ===",
+        phase_idx + 1,
+        name,
+        duration,
+        phase.get("description", ""),
+    )
+
+    tasks: list[tuple[str, Path, asyncio.Task]] = []
+    t0 = time.monotonic()
+
+    rpc_cfg = phase.get("rpc")
+    if rpc_cfg:
+        rpc_out = report_dir / f"{prefix}-rpc.json"
+        cmd = _build_rpc_cmd(endpoints, rpc_cfg, duration, rpc_out)
+        tasks.append(
+            ("rpc", rpc_out, asyncio.create_task(run_subprocess(cmd, f"RPC [{name}]")))
+        )
+
+    tx_cfg = phase.get("tx")
+    if tx_cfg:
+        tx_out = report_dir / f"{prefix}-tx.json"
+        cmd = _build_tx_cmd(endpoints[0], tx_cfg, duration, tx_out)
+        tasks.append(
+            ("tx", tx_out, asyncio.create_task(run_subprocess(cmd, f"TX [{name}]")))
+        )
+
+    if not tasks:
+        logger.warning(
+            "Phase %d: %s — no workload configured, skipping", phase_idx + 1, name
+        )
+        return result
+
+    for label, report_path, task in tasks:
+        returncode, _stdout, stderr = await task
+        _collect_task_result(label, returncode, stderr, report_path, result)
+
+    result.actual_sec = time.monotonic() - t0
+    logger.info(
+        "Phase %d complete: %.1fs actual, %d errors",
+        phase_idx + 1,
+        result.actual_sec,
+        len(result.errors),
+    )
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Profile execution
+# ---------------------------------------------------------------------------
+
+
+async def run_profile(
+    profile: dict[str, Any],
+    endpoints: list[str],
+    report_dir: Path,
+) -> dict[str, Any]:
+    """Execute all phases in a profile sequentially.
+
+    Args:
+        profile:    Profile dict with phases and propagation_wait_sec.
+        endpoints:  WebSocket endpoints for the rippled cluster.
+        report_dir: Directory for phase reports.
+
+    Returns:
+        Combined report dict with per-phase results and totals.
+    """
+    phases = profile.get("phases", [])
+    propagation_wait = profile.get("propagation_wait_sec", 60)
+    results: list[PhaseResult] = []
+
+    total_start = time.monotonic()
+
+    for idx, phase in enumerate(phases):
+        result = await run_phase(phase, endpoints, report_dir, idx)
+        results.append(result)
+
+    # Wait for telemetry data to propagate through the collector pipeline.
+    logger.info("Waiting %ds for telemetry data to propagate...", propagation_wait)
+    await asyncio.sleep(propagation_wait)
+
+    total_elapsed = time.monotonic() - total_start
+
+    # Build combined report from all phase results.
+    total_rpc_sent = 0
+    total_rpc_errors = 0
+    total_tx_submitted = 0
+    total_tx_errors = 0
+    phase_reports = []
+
+    for r in results:
+        pr: dict[str, Any] = {
+            "name": r.name,
+            "duration_sec": r.duration_sec,
+            "actual_sec": round(r.actual_sec, 1),
+            "errors": r.errors,
+        }
+        if r.rpc_summary:
+            pr["rpc"] = r.rpc_summary
+            total_rpc_sent += r.rpc_summary.get("total_sent", 0)
+            total_rpc_errors += r.rpc_summary.get("total_errors", 0)
+        if r.tx_summary:
+            pr["tx"] = r.tx_summary
+            total_tx_submitted += r.tx_summary.get("total_submitted", 0)
+            total_tx_errors += r.tx_summary.get("total_errors", 0)
+        phase_reports.append(pr)
+
+    report = {
+        "profile": profile.get("description", ""),
+        "total_elapsed_sec": round(total_elapsed, 1),
+        "phases": phase_reports,
+        "totals": {
+            "rpc_sent": total_rpc_sent,
+            "rpc_errors": total_rpc_errors,
+            "tx_submitted": total_tx_submitted,
+            "tx_errors": total_tx_errors,
+        },
+    }
+
+    return report
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Workload Orchestrator for rippled telemetry validation",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Profiles:
+  full-validation  Full 18-dashboard coverage (~5 min load + 1 min propagation)
+  quick-smoke      Fast CI smoke test (~30s load + 30s propagation)
+  stress           Heavy sustained load for benchmarking (~3.5 min + 1 min)
+
+Examples:
+  python3 workload_orchestrator.py --profile full-validation
+  python3 workload_orchestrator.py --profile quick-smoke --endpoints ws://localhost:6006
+  python3 workload_orchestrator.py --profile stress --report /tmp/report.json
+        """,
+    )
+    parser.add_argument(
+        "--profile",
+        type=str,
+        required=True,
+        help="Named profile from workload-profiles.json",
+    )
+    parser.add_argument(
+        "--endpoints",
+        nargs="+",
+        default=["ws://localhost:6006"],
+        help="WebSocket endpoints (default: ws://localhost:6006)",
+    )
+    parser.add_argument(
+        "--report",
+        type=str,
+        default=None,
+        help="Write combined JSON report to this file",
+    )
+    parser.add_argument(
+        "--report-dir",
+        type=str,
+        default="/tmp/xrpld-validation/reports",
+        help="Directory for per-phase reports",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable debug logging",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Main entry point for the workload orchestrator."""
+    args = parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s [%(name)s] %(levelname)s %(message)s",
+    )
+
+    profile = load_profile(args.profile)
+    report_dir = Path(args.report_dir)
+    report_dir.mkdir(parents=True, exist_ok=True)
+
+    report = asyncio.run(run_profile(profile, args.endpoints, report_dir))
+
+    print(json.dumps(report, indent=2))
+
+    if args.report:
+        with open(args.report, "w") as f:
+            json.dump(report, f, indent=2)
+        logger.info("Combined report written to %s", args.report)
+
+    # Exit with error if either generator had high error rates.
+    totals = report["totals"]
+    rpc_err_rate = (
+        totals["rpc_errors"] / totals["rpc_sent"] * 100 if totals["rpc_sent"] > 0 else 0
+    )
+    tx_err_rate = (
+        totals["tx_errors"] / totals["tx_submitted"] * 100
+        if totals["tx_submitted"] > 0
+        else 0
+    )
+    if rpc_err_rate > 50 or tx_err_rate > 50:
+        logger.error(
+            "High error rates: RPC=%.1f%%, TX=%.1f%%", rpc_err_rate, tx_err_rate
+        )
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

docker/telemetry/workload/xrpld-validator.cfg.template

@@ -0,0 +1,97 @@
+# xrpld validator node configuration template for workload harness.
+#
+# Placeholders (replaced by docker-compose entrypoint):
+#   {{NODE_INDEX}}        — Node number (1-based)
+#   {{RPC_PORT}}          — HTTP RPC port
+#   {{WS_PORT}}           — WebSocket port
+#   {{PEER_PORT}}         — Peer protocol port
+#   {{DATA_DIR}}          — Node data directory
+#   {{VALIDATION_SEED}}   — Validator seed from key generation
+#   {{VALIDATORS_FILE}}   — Path to shared validators.txt
+#   {{IPS_FIXED}}         — Peer addresses (one per line)
+#   {{OTEL_ENDPOINT}}     — OTel Collector OTLP/HTTP traces endpoint
+#   {{OTEL_METRICS_ENDPOINT}} — OTel Collector OTLP/HTTP metrics endpoint
+#   {{LOG_LEVEL}}         — Log level (debug, info, warning, error)
+
+[server]
+port_rpc
+port_ws
+port_peer
+
+[port_rpc]
+port = {{RPC_PORT}}
+ip = 0.0.0.0
+admin = 0.0.0.0
+protocol = http
+
+[port_ws]
+port = {{WS_PORT}}
+ip = 0.0.0.0
+admin = 0.0.0.0
+protocol = ws
+
+[port_peer]
+port = {{PEER_PORT}}
+ip = 0.0.0.0
+protocol = peer
+
+[node_db]
+type=NuDB
+path={{DATA_DIR}}/nudb
+online_delete=256
+
+[database_path]
+{{DATA_DIR}}/db
+
+[debug_logfile]
+{{DATA_DIR}}/debug.log
+
+[validation_seed]
+{{VALIDATION_SEED}}
+
+[validators_file]
+{{VALIDATORS_FILE}}
+
+[ips_fixed]
+{{IPS_FIXED}}
+
+[peer_private]
+1
+
+# --- OpenTelemetry tracing (all categories enabled) ---
+[telemetry]
+enabled=1
+service_instance_id=validator-{{NODE_INDEX}}
+endpoint={{OTEL_ENDPOINT}}
+exporter=otlp_http
+sampling_ratio=1.0
+batch_size=512
+batch_delay_ms=2000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1
+
+# --- Native OTel metrics (beast::insight over OTLP/HTTP) ---
+# The collector has no StatsD receiver (metrics pipeline is [otlp, spanmetrics]),
+# so beast::insight exports natively over OTLP. prefix=xrpld matches the OTel
+# resource service name and the xrpld_* names the dashboards query.
+[insight]
+server=otel
+endpoint={{OTEL_METRICS_ENDPOINT}}
+prefix=xrpld
+
+[rpc_startup]
+{ "command": "log_level", "severity": "{{LOG_LEVEL}}" }
+
+[ssl_verify]
+0
+
+# --- Network tuning for local cluster ---
+[network_id]
+0
+
+[sntp_servers]
+time.google.com

docker/telemetry/xrpld-telemetry-mainnet.cfg

@@ -0,0 +1,139 @@
+# xrpld configuration for Mainnet with full OpenTelemetry tracing.
+#
+# Connects to XRP Ledger Mainnet as an observer-only tracking node
+# (no validator credentials) and exercises ALL instrumented workflows:
+# RPC, transactions, consensus, peer overlay, ledger ops, and
+# pathfinding.
+#
+# Usage:
+#   1. Start the observability stack:
+#        docker compose -f docker/telemetry/docker-compose.yml up -d
+#   2. Run xrpld:
+#        ./xrpld --conf docker/telemetry/xrpld-telemetry.cfg
+#   3. Wait for sync (server_state=full), then exercise workflows:
+#        curl -s http://localhost:5005 -d '{"method":"server_info"}'
+#   4. View traces in Grafana Explore -> Tempo: http://localhost:3000
+
+# --- Server ports -----------------------------------------------------------
+
+[server]
+port_rpc_admin_local
+port_ws_admin_local
+port_ws_public
+port_peer
+
+[port_rpc_admin_local]
+port = 5005
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[port_ws_admin_local]
+port = 6006
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = ws
+
+[port_ws_public]
+port = 6005
+ip = 0.0.0.0
+protocol = ws
+
+[port_peer]
+port = 51235
+ip = 0.0.0.0
+protocol = peer
+
+# --- Network ----------------------------------------------------------------
+
+[network_id]
+main
+
+# [ips] omitted on purpose. xrpld uses its built-in mainnet hub list
+# (r.ripple.com, sahyadri.isrdc.in, hubs.xrpkuwait.com,
+# hub.xrpl-commons.org) for peer discovery.
+
+[validators_file]
+validators-mainnet.txt
+
+[peer_private]
+0
+
+[peers_max]
+21
+
+# --- Pathfinding (exercises ripple_path_find / path_find workflows) ---------
+
+[path_search]
+7
+
+[path_search_fast]
+2
+
+[path_search_max]
+10
+
+# --- Signing (allows sign/sign_for RPC for test tx submission) --------------
+
+[signing_support]
+true
+
+# --- Database ---------------------------------------------------------------
+
+[node_db]
+type=NuDB
+path=docker/telemetry/data/nudb
+online_delete=2000
+advisory_delete=0
+
+[database_path]
+docker/telemetry/data
+
+[ledger_history]
+1000
+
+# --- Logging ----------------------------------------------------------------
+
+[debug_logfile]
+/home/pratik/xrpld-logs/mainnet/debug.log
+
+[rpc_startup]
+{ "command": "log_level", "severity": "debug" }
+
+# --- SSL --------------------------------------------------------------------
+
+[ssl_verify]
+0
+
+# --- Insight (native OTel metrics via beast::insight) -----------------------
+
+[insight]
+server=otel
+endpoint=http://localhost:4318/v1/metrics
+prefix=xrpld
+# Sets the OTel service.instance.id resource attribute, which Prometheus
+# exposes as the `exported_instance` label. Dashboards filter on it via the
+# $node template variable, so without this every insight-backed panel is
+# empty. Matches [telemetry] service_instance_id for a single node identity.
+service_instance_id=xrpld-mainnet
+
+# --- OpenTelemetry tracing --------------------------------------------------
+
+[telemetry]
+enabled=1
+service_instance_id=xrpld-mainnet
+endpoint=http://localhost:4318/v1/traces
+metrics_endpoint=http://localhost:4318/v1/metrics
+exporter=otlp_http
+# Mainnet has high span throughput across peer/ledger/consensus.
+# 0.05 keeps Tempo/collector load sustainable. Raise to 1.0 for
+# short debugging windows only.
+sampling_ratio=1.0
+batch_size=512
+batch_delay_ms=5000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1

docker/telemetry/xrpld-telemetry.cfg

@@ -0,0 +1,134 @@
+# xrpld configuration for Devnet with full OpenTelemetry tracing.
+#
+# Connects to the XRP Ledger Devnet and exercises ALL instrumented
+# workflows: RPC, transactions, consensus, peer overlay, ledger ops,
+# and pathfinding.
+#
+# Usage:
+#   1. Start the observability stack:
+#        docker compose -f docker/telemetry/docker-compose.yml up -d
+#   2. Run xrpld:
+#        ./xrpld --conf docker/telemetry/xrpld-telemetry.cfg
+#   3. Wait for sync (server_state=full), then exercise workflows:
+#        curl -s http://localhost:5005 -d '{"method":"server_info"}'
+#   4. View traces in Grafana Explore -> Tempo: http://localhost:3000
+
+# --- Server ports -----------------------------------------------------------
+
+[server]
+port_rpc_admin_local
+port_ws_admin_local
+port_ws_public
+port_peer
+
+[port_rpc_admin_local]
+port = 5005
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[port_ws_admin_local]
+port = 6006
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = ws
+
+[port_ws_public]
+port = 6005
+ip = 0.0.0.0
+protocol = ws
+
+[port_peer]
+port = 51235
+ip = 0.0.0.0
+protocol = peer
+
+# --- Network ----------------------------------------------------------------
+
+[network_id]
+devnet
+
+[ips]
+s.devnet.rippletest.net 51235
+
+[validators_file]
+validators-devnet.txt
+
+[peer_private]
+0
+
+[peers_max]
+21
+
+# --- Pathfinding (exercises ripple_path_find / path_find workflows) ---------
+
+[path_search]
+7
+
+[path_search_fast]
+2
+
+[path_search_max]
+10
+
+# --- Signing (allows sign/sign_for RPC for test tx submission) --------------
+
+[signing_support]
+true
+
+# --- Database ---------------------------------------------------------------
+
+[node_db]
+type=NuDB
+path=docker/telemetry/data/nudb
+online_delete=2000
+advisory_delete=0
+
+[database_path]
+docker/telemetry/data
+
+[ledger_history]
+1000
+
+# --- Logging ----------------------------------------------------------------
+
+[debug_logfile]
+/tmp/xrpld-integration/devnet/debug.log
+
+[rpc_startup]
+{ "command": "log_level", "severity": "debug" }
+
+# --- SSL --------------------------------------------------------------------
+
+[ssl_verify]
+0
+
+# --- Insight (native OTel metrics via beast::insight) -----------------------
+
+[insight]
+server=otel
+endpoint=http://localhost:4318/v1/metrics
+prefix=xrpld
+# Sets the OTel service.instance.id resource attribute, which Prometheus
+# exposes as the `exported_instance` label. Dashboards filter on it via the
+# $node template variable, so without this every insight-backed panel is
+# empty. Matches [telemetry] service_instance_id for a single node identity.
+service_instance_id=xrpld-devnet
+
+# --- OpenTelemetry tracing --------------------------------------------------
+
+[telemetry]
+enabled=1
+service_instance_id=xrpld-devnet
+endpoint=http://localhost:4318/v1/traces
+metrics_endpoint=http://localhost:4318/v1/metrics
+exporter=otlp_http
+sampling_ratio=1.0
+batch_size=512
+batch_delay_ms=5000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1

docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md

@@ -0,0 +1,655 @@
+# External Dashboard Parity — Design Spec
+
+> **Date**: 2026-03-30
+> **Status**: Draft
+> **Source**: [realgrapedrop/xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard)
+> **Jira Epic**: RIPD-5060
+
+## Summary
+
+Integrate 29 missing metrics, 18 alert rules, and enriched span attributes from the community `xrpl-validator-dashboard` into xrpld's native OpenTelemetry instrumentation. Changes are distributed across phases 2, 3, 4, 6, 7, 9, 10, and 11 of the OTel PR chain.
+
+## Gap Analysis
+
+### Coverage Breakdown (86 external metrics)
+
+| Status            | Count | Notes                                                          |
+| ----------------- | ----- | -------------------------------------------------------------- |
+| Already covered   | 30    | peer_count, load_factor, io_latency, uptime, overlay traffic   |
+| Partially covered | 3     | state_value encoding, NuDB granularity, validation_quorum      |
+| Missing           | 29    | Validation agreement, ledger economy, peer quality, UNL health |
+| N/A (external)    | 24    | Monitor health, realtime duplicates, system metrics            |
+
+### Missing Metrics by Category
+
+| Category             | Metrics                                                                                                                                                                                                                            | Count |
+| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
+| Validation Agreement | `validations_sent_total`, `validations_checked_total`, `validation_agreements_total`, `validation_missed_total`, `validation_agreement_pct_1h/24h`, `validation_agreements_1h/24h`, `validation_missed_1h/24h`, `validation_event` | 11    |
+| Ledger Economy       | `ledgers_closed_total`, `ledger_age_seconds`, `base_fee_xrp`, `reserve_base_xrp`, `reserve_inc_xrp`, `transaction_rate`                                                                                                            | 6     |
+| State Tracking       | `time_in_current_state_seconds`, `state_changes_total`, `validator_state_info`                                                                                                                                                     | 3     |
+| Peer Quality         | `peers_insane`, `peer_latency_p90_ms`                                                                                                                                                                                              | 2     |
+| Validator Health     | `amendment_blocked`, `unl_expiry_days`                                                                                                                                                                                             | 2     |
+| Upgrade Awareness    | `peers_higher_version_pct`, `upgrade_recommended`                                                                                                                                                                                  | 2     |
+| Storage / Other      | `ledger_nudb_bytes`, `jq_trans_overflow_total`, `initial_sync_duration_seconds`                                                                                                                                                    | 3     |
+
+### Alert Rules (18 total, from external dashboard)
+
+| Group       | Count | Rules                                                                                                                   |
+| ----------- | ----- | ----------------------------------------------------------------------------------------------------------------------- |
+| Critical    | 8     | Agreement <90%, not proposing, unhealthy state, amendment blocked, UNL expiring, IO latency, load factor, peer count <5 |
+| Network     | 3     | Peer drop >10%/30%, P90 latency + disconnect correlation                                                                |
+| Performance | 7     | CPU >80%, memory >90%, disk >85%, job queue overflow, upgrade recommended, tx rate drop, stale ledger                   |
+
+---
+
+## Branch-to-Change Mapping
+
+### Phase 2 — `pratik/otel-phase2-rpc-tracing`
+
+> **Ref**: Adds to existing Phase 2 task list. Consumed by Phase 7 (MetricsRegistry) and Phase 10 (validation checks).
+
+**Task 2.8: RPC Span Attribute Enrichment**
+
+Add node-level health context to every `rpc.command.*` span so operators can correlate RPC behavior with node state.
+
+New span attributes on `rpc.command.*`:
+
+| Attribute                     | Type   | Source                               | Value Example         |
+| ----------------------------- | ------ | ------------------------------------ | --------------------- |
+| `xrpl.node.amendment_blocked` | bool   | `app_.getOPs().isAmendmentBlocked()` | `true`                |
+| `xrpl.node.server_state`      | string | `app_.getOPs().strOperatingMode()`   | `"full"`, `"syncing"` |
+
+**File**: `src/xrpld/rpc/detail/RPCHandler.cpp` (in the `rpc.command.*` span creation block, after existing setAttribute calls)
+
+**Rationale**: RPC is the operator's primary interaction point. When a node is amendment-blocked or degraded, every RPC response is suspect. Tagging spans with this state enables Tempo TraceQL queries like `{name=~"rpc.command.*" && span.xrpl.node.amendment_blocked = true}` to find all RPCs served during a blocked period.
+
+**Exit Criteria**:
+
+- [ ] `rpc.command.server_info` spans carry `xrpl.node.amendment_blocked` and `xrpl.node.server_state` attributes
+- [ ] No measurable latency impact (attribute values are cached atomics, not computed per-call)
+
+---
+
+### Phase 3 — `pratik/otel-phase3-tx-tracing`
+
+> **Ref**: Adds to existing Phase 3 task list. Consumed by Phase 10 (validation checks).
+
+**Task 3.7: Transaction Span Peer Version Attribute**
+
+Add the relaying peer's xrpld version to transaction receive spans to enable version-mismatch correlation.
+
+New span attribute on `tx.receive`:
+
+| Attribute           | Type   | Source               | Value Example   |
+| ------------------- | ------ | -------------------- | --------------- |
+| `xrpl.peer.version` | string | `peer->getVersion()` | `"xrpld-2.4.0"` |
+
+**File**: `src/xrpld/overlay/detail/PeerImp.cpp` (in the `tx.receive` span block, after existing `xrpl.peer.id` setAttribute)
+
+**Rationale**: Transaction relay is where version mismatches cause subtle serialization or validation bugs. Tracing "this tx came from a v2.3.0 peer" helps diagnose compatibility issues during network upgrades.
+
+**Exit Criteria**:
+
+- [ ] `tx.receive` spans carry `xrpl.peer.version` attribute with a non-empty version string
+- [ ] Attribute is omitted (not empty-string) when `getVersion()` returns empty
+
+---
+
+### Phase 4 — `pratik/otel-phase4-consensus-tracing`
+
+> **Ref**: Adds to existing Phase 4 task list. Provides the span-level foundation that Phase 7 (ValidationTracker) builds upon. Consumed by Phase 10 (validation checks).
+
+**Task 4.8: Consensus Validation Span Enrichment**
+
+Add ledger hash and validation type to validation spans on both send and receive paths. This enables trace-level agreement analysis — filter by ledger hash to see which validators agreed.
+
+New span attributes on `consensus.validation.send`:
+
+| Attribute                     | Type   | Source                                  | Value Example               |
+| ----------------------------- | ------ | --------------------------------------- | --------------------------- |
+| `xrpl.validation.ledger_hash` | string | Ledger hash from `validate()` call args | `"A1B2C3..."` (64-char hex) |
+| `xrpl.validation.full`        | bool   | Whether this is a full validation       | `true`                      |
+
+New span attributes on `peer.validation.receive`:
+
+| Attribute                          | Type   | Source                                | Value Example               |
+| ---------------------------------- | ------ | ------------------------------------- | --------------------------- |
+| `xrpl.peer.validation.ledger_hash` | string | From deserialized STValidation object | `"A1B2C3..."` (64-char hex) |
+| `xrpl.peer.validation.full`        | bool   | From STValidation flags               | `true`                      |
+
+New span attributes on `consensus.accept`:
+
+| Attribute                            | Type  | Source                                   | Value Example |
+| ------------------------------------ | ----- | ---------------------------------------- | ------------- |
+| `xrpl.consensus.validation_quorum`   | int64 | `app_.validators().quorum()`             | `28`          |
+| `xrpl.consensus.proposers_validated` | int64 | `result.proposers` from consensus result | `35`          |
+
+**Files**:
+
+- `src/xrpld/app/consensus/RCLConsensus.cpp` (validation.send and accept spans)
+- `src/xrpld/overlay/detail/PeerImp.cpp` (peer.validation.receive span)
+
+**Rationale**: The external dashboard's most valuable feature is validation agreement tracking. By recording the ledger hash on both outgoing and incoming validation spans, we create the raw data for agreement analysis at the trace level. Phase 7's ValidationTracker builds the metric-level aggregation on top of this.
+
+**Exit Criteria**:
+
+- [ ] `consensus.validation.send` spans carry `xrpl.validation.ledger_hash` and `xrpl.validation.full`
+- [ ] `peer.validation.receive` spans carry `xrpl.peer.validation.ledger_hash` and `xrpl.peer.validation.full`
+- [ ] `consensus.accept` spans carry `xrpl.consensus.validation_quorum` and `xrpl.consensus.proposers_validated`
+- [ ] Ledger hash attributes match between send and receive for the same ledger
+
+---
+
+### Phase 6 — `pratik/otel-phase6-statsd`
+
+> **Ref**: Adds to existing Phase 6 scope. No separate task list file exists for Phase 6 per project convention.
+
+**Addition: Bridge `peerDisconnectsCharges_` metric**
+
+The overlay already tracks resource-limit disconnects via `OverlayImpl::Stats::peerDisconnectsCharges_` (a `beast::insight::Gauge`). This metric is registered but not included in the StatsD bridge mapping.
+
+**What to do**:
+
+- Ensure `xrpld_Overlay_Peer_Disconnects_Charges` appears in the StatsD-to-Prometheus metric name mapping
+- Verify the metric appears in Prometheus after StatsD bridge is active
+
+**File**: `src/xrpld/overlay/detail/OverlayImpl.cpp`
+
+**Prometheus name**: `xrpld_Overlay_Peer_Disconnects_Charges`
+
+---
+
+### Phase 7 — `pratik/otel-phase7-native-metrics`
+
+> **Ref**: Adds to existing Phase 7 task list. This is the largest addition. Depends on Phase 4 span attributes for validation tracking context. Consumed by Phase 9 (dashboards), Phase 10 (validation), Phase 11 (alerts).
+
+**Task 7.8: ValidationTracker — Validation Agreement Computation**
+
+The most valuable missing component. A stateful class that tracks whether our validator's validations agree with network consensus, maintaining rolling 1h and 24h windows.
+
+**Architecture**:
+
+```
+
+  consensus.validation.send ─────> ValidationTracker ──────> MetricsRegistry
+  (records our validation          (reconciles after         (exports agreement
+   for ledger X)                    8s grace period)          gauges every 10s)
+
+  ledger.validate ───────────────> ValidationTracker
+  (records which ledger            (marks ledger X as
+   network validated)               agreed or missed)
+```
+
+**Design**:
+
+```cpp
+/// Tracks validation agreement between this node and network consensus.
+///
+///  ValidationTracker
+///  ├── recordOurValidation(ledgerHash, ledgerSeq)  // called when we send
+///  ├── recordNetworkValidation(ledgerHash, seq)     // called on ledger validate
+///  ├── reconcile()                                  // called periodically (timer)
+///  ├── agreementPct1h() -> double                   // 0.0-100.0
+///  ├── agreementPct24h() -> double
+///  ├── agreements1h() -> uint64_t
+///  ├── missed1h() -> uint64_t
+///  ├── agreements24h() -> uint64_t
+///  ├── missed24h() -> uint64_t
+///  ├── totalAgreements() -> uint64_t
+///  ├── totalMissed() -> uint64_t
+///  ├── totalValidationsSent() -> uint64_t
+///  └── totalValidationsChecked() -> uint64_t        // all network validations seen
+class ValidationTracker
+{
+    // Ring buffer of pending ledger events (max 1000)
+    struct LedgerEvent {
+        uint256 ledgerHash;
+        LedgerIndex seq;
+        TimePoint closeTime;
+        bool weValidated = false;     // did we send a validation for this ledger?
+        bool networkValidated = false; // did network validate this ledger?
+        bool reconciled = false;       // has 8s grace period elapsed?
+        bool agreed = false;           // after reconciliation: did we agree?
+    };
+
+    // Sliding window deques for pre-computed window stats
+    struct WindowEvent {
+        TimePoint time;
+        bool agreed;
+    };
+    std::deque<WindowEvent> window1h_;  // events in last 1 hour
+    std::deque<WindowEvent> window24h_; // events in last 24 hours
+
+    // Reconciliation: 8s grace period after ledger close.
+    // If our validation hasn't arrived by then, mark as missed.
+    // 5-minute late repair: if a late validation arrives, correct the miss.
+    static constexpr auto kGracePeriod = std::chrono::seconds(8);
+    static constexpr auto kLateRepairWindow = std::chrono::minutes(5);
+};
+```
+
+**Recording sites** (modifications to consensus code from Phase 7 branch):
+
+| Hook Point                   | File                | What to Record                                                          |
+| ---------------------------- | ------------------- | ----------------------------------------------------------------------- |
+| `validate()` in `doAccept()` | RCLConsensus.cpp    | `tracker.recordOurValidation(ledgerHash, seq)`                          |
+| `onValidation()` callback    | RCLValidations path | `tracker.recordNetworkValidation(...)` — increment `validationsChecked` |
+| LedgerMaster fully-validated | LedgerMaster.cpp    | `tracker.recordNetworkValidation(validatedHash, seq)`                   |
+
+**Key new files**:
+
+- `src/xrpld/telemetry/ValidationTracker.h`
+- `src/xrpld/telemetry/detail/ValidationTracker.cpp`
+
+**Key modified files**:
+
+- `src/xrpld/telemetry/MetricsRegistry.h` (add ValidationTracker member)
+- `src/xrpld/telemetry/MetricsRegistry.cpp` (add gauge callback reading from tracker)
+- `src/xrpld/app/consensus/RCLConsensus.cpp` (add recording hooks)
+- `src/xrpld/app/ledger/detail/LedgerMaster.cpp` (add recording hook)
+
+**Exit Criteria**:
+
+- [ ] `ValidationTracker` correctly tracks agreement with 8s grace period
+- [ ] 5-minute late repair corrects false-positive misses
+- [ ] Thread-safe (atomics + mutex for window deques)
+- [ ] Rolling windows correctly evict stale entries
+- [ ] Unit tests for: normal agreement, missed validation, late repair, window eviction
+
+---
+
+**Task 7.9: Validator Health Observable Gauges**
+
+New MetricsRegistry observable gauge for amendment, UNL, and quorum health.
+
+| Gauge Name               | Label `metric=`     | Type   | Source                                            |
+| ------------------------ | ------------------- | ------ | ------------------------------------------------- |
+| `xrpld_validator_health` | `amendment_blocked` | int64  | `app_.getOPs().isAmendmentBlocked()` → 0/1        |
+|                          | `unl_blocked`       | int64  | `app_.getOPs().isUNLBlocked()` → 0/1              |
+|                          | `unl_expiry_days`   | double | `app_.validators().expires()` → days until expiry |
+|                          | `validation_quorum` | int64  | `app_.validators().quorum()`                      |
+
+**File**: `src/xrpld/telemetry/MetricsRegistry.cpp` (new gauge callback in `registerAsyncGauges()`)
+
+**Exit Criteria**:
+
+- [ ] All 4 label values emitted every 10s
+- [ ] `unl_expiry_days` is negative when expired, positive when active
+- [ ] Values visible in Prometheus
+
+---
+
+**Task 7.10: Peer Quality Observable Gauges**
+
+New MetricsRegistry observable gauge for peer health aggregates.
+
+| Gauge Name           | Label `metric=`            | Type   | Source                                     |
+| -------------------- | -------------------------- | ------ | ------------------------------------------ |
+| `xrpld_peer_quality` | `peer_latency_p90_ms`      | double | Iterate peers, compute P90 from `latency_` |
+|                      | `peers_insane_count`       | int64  | Count peers with `tracking_ == diverged`   |
+|                      | `peers_higher_version_pct` | double | Compare `getVersion()` to own version      |
+|                      | `upgrade_recommended`      | int64  | 1 if `peers_higher_version_pct > 60%`      |
+
+**Implementation note**: The callback iterates `app_.overlay().foreach(...)` to collect per-peer latency and version data. This runs every 10s on the metrics reader thread — acceptable overhead for ~50-200 peers.
+
+**File**: `src/xrpld/telemetry/MetricsRegistry.cpp`
+
+**Exit Criteria**:
+
+- [ ] P90 latency computed correctly (sort peer latencies, pick 90th percentile)
+- [ ] Insane count matches `peers` RPC output
+- [ ] Version comparison handles format variations (e.g., "xrpld-2.4.0-rc1")
+- [ ] Values visible in Prometheus
+
+---
+
+**Task 7.11: Ledger Economy Observable Gauges**
+
+New MetricsRegistry observable gauge for fee and ledger metrics.
+
+| Gauge Name             | Label `metric=`      | Type   | Source                                    |
+| ---------------------- | -------------------- | ------ | ----------------------------------------- |
+| `xrpld_ledger_economy` | `base_fee_xrp`       | double | `app_.getFeeTrack().getBaseFee()` → drops |
+|                        | `reserve_base_xrp`   | double | From validated ledger fee settings        |
+|                        | `reserve_inc_xrp`    | double | From validated ledger fee settings        |
+|                        | `ledger_age_seconds` | double | `now - lastValidatedCloseTime`            |
+|                        | `transaction_rate`   | double | Derived: tx count delta / time delta      |
+
+**File**: `src/xrpld/telemetry/MetricsRegistry.cpp`
+
+**Exit Criteria**:
+
+- [ ] Fee values match `server_info` RPC output
+- [ ] `ledger_age_seconds` increases monotonically between ledger closes, resets on close
+- [ ] `transaction_rate` is smoothed (rolling average, not instantaneous)
+
+---
+
+**Task 7.12: State Tracking Observable Gauges**
+
+New MetricsRegistry observable gauge for node state duration.
+
+| Gauge Name             | Label `metric=`                 | Type   | Source                                           |
+| ---------------------- | ------------------------------- | ------ | ------------------------------------------------ |
+| `xrpld_state_tracking` | `state_value`                   | int64  | 0-7 numeric encoding matching external dashboard |
+|                        | `time_in_current_state_seconds` | double | `now - lastModeChangeTime`                       |
+
+**State value encoding**:
+
+xrpld's `OperatingMode` enum maps 0-4 (DISCONNECTED through FULL). The external dashboard extends this to 0-6 by combining operating mode with consensus participation:
+
+| Value | State        | Source                                                      |
+| ----- | ------------ | ----------------------------------------------------------- |
+| 0     | disconnected | `OperatingMode::DISCONNECTED`                               |
+| 1     | connected    | `OperatingMode::CONNECTED`                                  |
+| 2     | syncing      | `OperatingMode::SYNCING`                                    |
+| 3     | tracking     | `OperatingMode::TRACKING`                                   |
+| 4     | full         | `OperatingMode::FULL` and not validating                    |
+| 5     | validating   | `OperatingMode::FULL` and `mConsensus.validating()` is true |
+| 6     | proposing    | `OperatingMode::FULL` and consensus mode is `proposing`     |
+
+**Note**: Values 5-6 require checking both `OperatingMode` and `ConsensusMode`. The callback should derive these from `app_.getOPs().getOperatingMode()` combined with `mConsensus.mode()`. If operating mode is FULL and consensus is proposing → 6; if FULL and validating → 5; otherwise use the raw OperatingMode enum value.
+
+**File**: `src/xrpld/telemetry/MetricsRegistry.cpp`
+
+**Exit Criteria**:
+
+- [ ] `state_value` matches external dashboard encoding
+- [ ] `time_in_current_state_seconds` resets on mode change
+
+---
+
+**Task 7.13: Storage Detail Observable Gauge**
+
+| Gauge Name             | Label `metric=` | Type  | Source                                   |
+| ---------------------- | --------------- | ----- | ---------------------------------------- |
+| `xrpld_storage_detail` | `nudb_bytes`    | int64 | NuDB backend file size (filesystem stat) |
+
+**File**: `src/xrpld/telemetry/MetricsRegistry.cpp`
+
+**Exit Criteria**:
+
+- [ ] NuDB file size reported in bytes
+- [ ] Gracefully returns 0 if NuDB not configured
+
+---
+
+**Task 7.14: New Synchronous Counters**
+
+New counters incremented at event sites. Declared in MetricsRegistry, recording sites added in consensus/overlay/network code.
+
+| Counter Name                        | Increment Site                   | Source File           |
+| ----------------------------------- | -------------------------------- | --------------------- |
+| `xrpld_ledgers_closed_total`        | `onAccept()` in consensus        | RCLConsensus.cpp      |
+| `xrpld_validations_sent_total`      | `validate()` in consensus        | RCLConsensus.cpp      |
+| `xrpld_validations_checked_total`   | Network validation received      | LedgerMaster.cpp      |
+| `xrpld_validation_agreements_total` | ValidationTracker reconciliation | ValidationTracker.cpp |
+| `xrpld_validation_missed_total`     | ValidationTracker reconciliation | ValidationTracker.cpp |
+| `xrpld_state_changes_total`         | `setMode()` in NetworkOPs        | NetworkOPs.cpp        |
+| `xrpld_jq_trans_overflow_total`     | Job queue overflow path          | JobQueue.cpp          |
+
+**Key modified files**:
+
+- `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (counter declarations)
+- `src/xrpld/app/consensus/RCLConsensus.cpp` (recording: ledgers_closed, validations_sent)
+- `src/xrpld/app/ledger/detail/LedgerMaster.cpp` (recording: validations_checked)
+- `src/xrpld/app/misc/NetworkOPs.cpp` (recording: state_changes)
+
+**Exit Criteria**:
+
+- [ ] All 7 counters monotonically increase during normal operation
+- [ ] Counter values match expected rates (e.g., ledgers_closed ≈ 1 per 3-5s)
+- [ ] Values visible in Prometheus
+
+---
+
+**Task 7.15: Validation Agreement Observable Gauge**
+
+Reads from the `ValidationTracker` (Task 7.8) to export rolling window stats.
+
+| Gauge Name                   | Label `metric=`     | Type   | Source                      |
+| ---------------------------- | ------------------- | ------ | --------------------------- |
+| `xrpld_validation_agreement` | `agreement_pct_1h`  | double | `tracker.agreementPct1h()`  |
+|                              | `agreements_1h`     | int64  | `tracker.agreements1h()`    |
+|                              | `missed_1h`         | int64  | `tracker.missed1h()`        |
+|                              | `agreement_pct_24h` | double | `tracker.agreementPct24h()` |
+|                              | `agreements_24h`    | int64  | `tracker.agreements24h()`   |
+|                              | `missed_24h`        | int64  | `tracker.missed24h()`       |
+
+**File**: `src/xrpld/telemetry/MetricsRegistry.cpp`
+
+**Exit Criteria**:
+
+- [ ] Agreement percentages in range [0.0, 100.0]
+- [ ] Window stats match manual count from validation counters
+- [ ] Percentages stabilize after 1h/24h of operation
+
+---
+
+### Phase 9 — `pratik/otel-phase9-metric-gap-fill`
+
+> **Ref**: Adds to existing Phase 9 task list. Depends on Phase 7 gauges/counters. Consumed by Phase 10 (dashboard load checks).
+
+**Task 9.11: Validator Health Dashboard**
+
+New Grafana dashboard: `xrpld-validator-health.json`
+
+| Panel                      | Type       | PromQL                                                         |
+| -------------------------- | ---------- | -------------------------------------------------------------- |
+| Agreement % (1h)           | stat       | `xrpld_validation_agreement{metric="agreement_pct_1h"}`        |
+| Agreement % (24h)          | stat       | `xrpld_validation_agreement{metric="agreement_pct_24h"}`       |
+| Agreements vs Missed (1h)  | bargauge   | `agreements_1h` and `missed_1h` side by side                   |
+| Agreements vs Missed (24h) | bargauge   | `agreements_24h` and `missed_24h` side by side                 |
+| Validation Rate            | stat       | `rate(xrpld_validations_sent_total[5m]) * 60`                  |
+| Validations Checked Rate   | stat       | `rate(xrpld_validations_checked_total[5m]) * 60`               |
+| Amendment Blocked          | stat       | `xrpld_validator_health{metric="amendment_blocked"}`           |
+| UNL Expiry (days)          | stat       | `xrpld_validator_health{metric="unl_expiry_days"}`             |
+| Validation Quorum          | stat       | `xrpld_validator_health{metric="validation_quorum"}`           |
+| State Value Timeline       | timeseries | `xrpld_state_tracking{metric="state_value"}`                   |
+| Time in Current State      | stat       | `xrpld_state_tracking{metric="time_in_current_state_seconds"}` |
+| State Changes Rate         | stat       | `rate(xrpld_state_changes_total[1h])`                          |
+| Ledgers Closed Rate        | stat       | `rate(xrpld_ledgers_closed_total[5m]) * 60`                    |
+
+**Dashboard conventions**: `$node` template variable for `exported_instance` filtering, dark theme, matching existing panel sizes and color schemes.
+
+---
+
+**Task 9.12: Peer Quality Dashboard**
+
+New Grafana dashboard: `xrpld-peer-quality.json`
+
+| Panel                  | Type       | PromQL                                                         |
+| ---------------------- | ---------- | -------------------------------------------------------------- |
+| P90 Peer Latency       | timeseries | `xrpld_peer_quality{metric="peer_latency_p90_ms"}`             |
+| Insane/Diverged Peers  | stat       | `xrpld_peer_quality{metric="peers_insane_count"}`              |
+| Higher Version Peers % | stat       | `xrpld_peer_quality{metric="peers_higher_version_pct"}`        |
+| Upgrade Recommended    | stat       | `xrpld_peer_quality{metric="upgrade_recommended"}`             |
+| Resource Disconnects   | timeseries | `xrpld_Overlay_Peer_Disconnects_Charges`                       |
+| Inbound vs Outbound    | bargauge   | `xrpld_Peer_Finder_Active_Inbound_Peers`, `..._Outbound_Peers` |
+
+---
+
+**Task 9.13: Ledger Economy Dashboard Panels**
+
+Add a "Ledger Economy" row to the existing `system-node-health.json` dashboard:
+
+| Panel                | Type       | PromQL                                              |
+| -------------------- | ---------- | --------------------------------------------------- |
+| Base Fee (drops)     | stat       | `xrpld_ledger_economy{metric="base_fee_xrp"}`       |
+| Reserve Base (drops) | stat       | `xrpld_ledger_economy{metric="reserve_base_xrp"}`   |
+| Reserve Inc (drops)  | stat       | `xrpld_ledger_economy{metric="reserve_inc_xrp"}`    |
+| Ledger Age           | stat       | `xrpld_ledger_economy{metric="ledger_age_seconds"}` |
+| Transaction Rate     | timeseries | `xrpld_ledger_economy{metric="transaction_rate"}`   |
+
+---
+
+### Phase 10 — `pratik/otel-phase10-workload-validation`
+
+> **Ref**: Adds to existing Phase 10 task list. Validates all additions from Phases 2-9.
+
+**Task 10.6: External Dashboard Parity Validation Checks**
+
+Add checks to `validate_telemetry.py` for all new span attributes and metrics.
+
+**New span attribute checks (~8)**:
+
+| Span Name                   | New Attribute                        |
+| --------------------------- | ------------------------------------ |
+| `rpc.command.server_info`   | `xrpl.node.amendment_blocked`        |
+| `rpc.command.server_info`   | `xrpl.node.server_state`             |
+| `tx.receive`                | `xrpl.peer.version`                  |
+| `consensus.validation.send` | `xrpl.validation.ledger_hash`        |
+| `consensus.validation.send` | `xrpl.validation.full`               |
+| `peer.validation.receive`   | `xrpl.peer.validation.ledger_hash`   |
+| `consensus.accept`          | `xrpl.consensus.validation_quorum`   |
+| `consensus.accept`          | `xrpl.consensus.proposers_validated` |
+
+**New metric existence checks (~13)**:
+
+| Metric Name                                              |
+| -------------------------------------------------------- |
+| `xrpld_validation_agreement{metric="agreement_pct_1h"}`  |
+| `xrpld_validation_agreement{metric="agreement_pct_24h"}` |
+| `xrpld_validator_health{metric="amendment_blocked"}`     |
+| `xrpld_validator_health{metric="unl_expiry_days"}`       |
+| `xrpld_peer_quality{metric="peer_latency_p90_ms"}`       |
+| `xrpld_peer_quality{metric="peers_insane_count"}`        |
+| `xrpld_ledger_economy{metric="base_fee_xrp"}`            |
+| `xrpld_ledger_economy{metric="transaction_rate"}`        |
+| `xrpld_state_tracking{metric="state_value"}`             |
+| `xrpld_ledgers_closed_total`                             |
+| `xrpld_validations_sent_total`                           |
+| `xrpld_state_changes_total`                              |
+| `xrpld_storage_detail{metric="nudb_bytes"}`              |
+
+**New dashboard load checks (~3)**:
+
+| Dashboard                      |
+| ------------------------------ |
+| `xrpld-validator-health`       |
+| `xrpld-peer-quality`           |
+| `system-node-health` (updated) |
+
+**New metric value sanity checks (~4)**:
+
+| Check                         | Condition         |
+| ----------------------------- | ----------------- |
+| `validation_agreement_pct_1h` | in [0, 100]       |
+| `unl_expiry_days`             | > 0 (not expired) |
+| `peer_latency_p90_ms`         | > 0 (peers exist) |
+| `state_value`                 | in [0, 7]         |
+
+**Total new checks: ~28** (bringing total from 73 to ~101)
+
+---
+
+### Phase 11 — (future branch)
+
+> **Ref**: Adds to existing Phase 11 task list. Depends on Phase 7 metrics and Phase 9 dashboards.
+
+**Task 11.9: Alert Rules from External Dashboard**
+
+Port 18 alert rules from the external `xrpl-validator-dashboard` to Grafana alerting provisioning.
+
+**Critical Group** (8 rules, eval interval 10s):
+
+| Rule                | Condition                                                     | For |
+| ------------------- | ------------------------------------------------------------- | --- |
+| Agreement Below 90% | `xrpld_validation_agreement{metric="agreement_pct_24h"} < 90` | 30s |
+| Not Proposing       | `xrpld_state_tracking{metric="state_value"} < 6`              | 10s |
+| Unhealthy State     | `xrpld_state_tracking{metric="state_value"} < 4`              | 10s |
+| Amendment Blocked   | `xrpld_validator_health{metric="amendment_blocked"} == 1`     | 1m  |
+| UNL Expiring        | `xrpld_validator_health{metric="unl_expiry_days"} < 14`       | 1h  |
+| High IO Latency     | `histogram_quantile(0.95, xrpld_ios_latency_bucket) > 50`     | 1m  |
+| High Load Factor    | `xrpld_load_factor_metrics{metric="load_factor"} > 1000`      | 1m  |
+| Peer Count Critical | `xrpld_server_info{metric="peers"} < 5`                       | 1m  |
+
+**Network Group** (3 rules, eval interval 10s):
+
+| Rule                      | Condition                                                         | For |
+| ------------------------- | ----------------------------------------------------------------- | --- |
+| Peer Drop >10%            | `delta(xrpld_server_info{metric="peers"}[30s]) / ... * 100 < -10` | 30s |
+| Peer Drop >30%            | Same formula, threshold -30                                       | 30s |
+| P90 Latency + Disconnects | `peer_latency_p90_ms > 500 AND rate(disconnects) > 0`             | 2m  |
+
+**Performance Group** (7 rules, eval interval 10s):
+
+| Rule                | Condition                                                    | For |
+| ------------------- | ------------------------------------------------------------ | --- |
+| CPU High            | Per-core CPU > 80%                                           | 2m  |
+| Memory Critical     | Memory usage > 90%                                           | 1m  |
+| Disk Warning        | Disk usage > 85%                                             | 2m  |
+| Job Queue Overflow  | `rate(xrpld_jq_trans_overflow_total[5m]) > 0`                | 1m  |
+| Upgrade Recommended | `xrpld_peer_quality{metric="peers_higher_version_pct"} > 60` | 1m  |
+| TX Rate Drop        | Transaction rate dropped > 50% in 5m window                  | 5m  |
+| Stale Ledger        | `xrpld_ledger_economy{metric="ledger_age_seconds"} > 30`     | 1m  |
+
+**Notification channels**: Template configs for Email/SMTP, Discord, Slack, PagerDuty.
+
+**Files**:
+
+- `docker/telemetry/grafana/alerting/alert-rules.yaml` (new or extend existing)
+- `docker/telemetry/grafana/alerting/contact-points.yaml`
+- `docker/telemetry/grafana/alerting/notification-policies.yaml`
+
+---
+
+**Task 11.10: Dual-Datasource Architecture Documentation**
+
+Document the external dashboard's "fast path" pattern as a future optimization for real-time panels:
+
+- **Pattern**: A lightweight Prometheus scrape endpoint (separate from OTLP pipeline) that polls critical metrics every 2-5s, bypassing the 10s OTLP metric reader interval and Prometheus scrape interval.
+- **Use case**: Real-time state panels (server state, ledger age, peer count) where 10-15s latency is too slow.
+- **Decision**: Document as a future option, not implement now. Current 10s interval is acceptable for v1.
+
+**File**: `OpenTelemetryPlan/Phase11_taskList.md` (documentation task, no code)
+
+---
+
+## Documentation Updates
+
+### `docs/telemetry-runbook.md` (on Phase 9 branch)
+
+Add new sections after "Phase 9: OTel Metrics Alerting Rules":
+
+1. **Validator Health Monitoring** — explains agreement tracking, amendment blocked, UNL expiry, with example PromQL queries
+2. **Peer Quality Monitoring** — explains P90 latency, insane peers, version awareness
+3. **Ledger Economy Monitoring** — explains fee/reserve gauges, transaction rate, ledger age
+4. **Validation Agreement Explained** — operator-facing explanation of the reconciliation algorithm (8s grace, 5m late repair), what "missed" means, and when to worry
+
+### `OpenTelemetryPlan/09-data-collection-reference.md` (on Phase 9 branch)
+
+Add new metric tables in a "Phase 7+: External Dashboard Parity" section covering all 29 new metrics with their gauge names, label values, types, and sources.
+
+---
+
+## Cross-Phase Dependency Chain
+
+```
+Phase 2 (span attrs: amendment_blocked, server_state)
+Phase 3 (span attrs: peer.version)
+Phase 4 (span attrs: validation.ledger_hash, validation.full, quorum)
+Phase 6 (StatsD bridge: peerDisconnectsCharges)
+    │
+    ├── all above rebase into ──>
+    │
+Phase 7 (ValidationTracker + 7 gauges + 7 counters + agreement gauge)
+    │
+Phase 9 (3 dashboards + ledger economy panels + runbook + data-collection-ref)
+    │
+Phase 10 (28 new validation checks in validate_telemetry.py)
+    │
+Phase 11 (18 alert rules + dual-datasource docs)
+```
+
+## Rebase Strategy
+
+After committing changes to each branch (starting from Phase 2):
+
+1. Commit on `pratik/otel-phase2-rpc-tracing`
+2. Rebase `phase3` onto `phase2`, resolve conflicts (task list files only — low risk)
+3. Commit on `phase3`, rebase `phase4` onto `phase3`
+4. Continue through chain: 4 → 5 → 5b → 6 → 7 → 8 → 9 → 10
+5. Force-push-with-lease all affected branches
+
+Since these are documentation-only changes (task list .md files), merge conflicts should be minimal — each file is unique to its branch.

docs/telemetry-runbook.md

@@ -15,8 +15,9 @@ docker compose -f docker/telemetry/docker-compose.yml up -d
 This starts:
 
 - **OTel Collector** on ports 4317 (gRPC) and 4318 (HTTP)
-- **Jaeger** UI on http://localhost:16686
+- **Tempo** on http://localhost:3200 (trace backend)
 - **Prometheus** on http://localhost:9090
+- **Loki** on http://localhost:3100 (log aggregation)
 - **Grafana** on http://localhost:3000
 
 ### 2. Enable telemetry in xrpld
@@ -74,194 +75,342 @@ All spans instrumented in xrpld, grouped by subsystem:
 
 ### Transaction Spans (Phase 3)
 
-| Span Name    | Source File    | Attributes                                                                             | Description                           |
-| ------------ | -------------- | -------------------------------------------------------------------------------------- | ------------------------------------- |
-| `tx.process` | NetworkOPs.cpp | `xrpl.tx.hash`, `local`, `path`, `tx_type`, `fee`, `sequence`, `ter_result`, `applied` | Transaction submission and processing |
-| `tx.receive` | PeerImp.cpp    | `xrpl.peer.id`, `xrpl.tx.hash`, `tx_type`, `peer_version`, `suppressed`, `tx_status`   | Transaction received from peer relay  |
+| Span Name       | Source File     | Attributes                                                                        | Description                           |
+| --------------- | --------------- | --------------------------------------------------------------------------------- | ------------------------------------- |
+| `tx.process`    | NetworkOPs.cpp  | `tx_hash`, `local`, `path`, `tx_type`, `fee`, `sequence`, `ter_result`, `applied` | Transaction submission and processing |
+| `tx.receive`    | PeerImp.cpp     | `peer_id`, `tx_hash`, `tx_type`, `peer_version`, `suppressed`, `tx_status`        | Transaction received from peer relay  |
+| `tx.apply`      | BuildLedger.cpp | `ledger_seq`, `tx_count`, `tx_failed`                                             | Transaction set applied per ledger    |
+| `tx.preflight`  | applySteps.cpp  | `stage`, `tx_type`, `ter_result`                                                  | Stateless checks stage                |
+| `tx.preclaim`   | applySteps.cpp  | `stage`, `tx_type`, `ter_result`                                                  | Ledger-aware checks stage             |
+| `tx.transactor` | Transactor.cpp  | `stage`, `tx_type`, `ter_result`, `applied`                                       | Apply stage (transactor runs)         |
+
+The three apply-pipeline spans (`tx.preflight`, `tx.preclaim`, `tx.transactor`)
+share a deterministic `trace_id` from `txID[0:16]`, so they group under one
+trace per transaction. The `stage` attribute (`preflight` / `preclaim` /
+`apply`) drives the collector spanmetrics `stage` dimension, giving per-stage
+RED metrics on the _Transaction Overview_ dashboard.
 
 ### Transaction Queue Spans (Phase 3)
 
-| Span Name          | Source File | Attributes                                                    | Description                                        |
-| ------------------ | ----------- | ------------------------------------------------------------- | -------------------------------------------------- |
-| `txq.enqueue`      | TxQ.cpp     | `xrpl.tx.hash`, `tx_type`                                     | Transaction enqueue decision (child of tx.process) |
-| `txq.apply_direct` | TxQ.cpp     | --                                                            | Direct apply attempt (bypassing queue)             |
-| `txq.batch_clear`  | TxQ.cpp     | --                                                            | Batch clear of queued transactions for an account  |
-| `txq.accept`       | TxQ.cpp     | `queue_size`, `ledger_changed`                                | Ledger-close accept loop over queued transactions  |
-| `txq.accept_tx`    | TxQ.cpp     | `xrpl.tx.hash`, `retries_remaining`, `ter_code`, `txq_status` | Per-transaction apply during accept                |
-| `txq.cleanup`      | TxQ.cpp     | `xrpl.ledger.seq`                                             | Post-close cleanup of expired queue entries        |
+| Span Name          | Source File | Attributes                                               | Description                                        |
+| ------------------ | ----------- | -------------------------------------------------------- | -------------------------------------------------- |
+| `txq.enqueue`      | TxQ.cpp     | `tx_hash`, `tx_type`                                     | Transaction enqueue decision (child of tx.process) |
+| `txq.apply_direct` | TxQ.cpp     | --                                                       | Direct apply attempt (bypassing queue)             |
+| `txq.batch_clear`  | TxQ.cpp     | --                                                       | Batch clear of queued transactions for an account  |
+| `txq.accept`       | TxQ.cpp     | `queue_size`, `ledger_changed`                           | Ledger-close accept loop over queued transactions  |
+| `txq.accept_tx`    | TxQ.cpp     | `tx_hash`, `retries_remaining`, `ter_code`, `txq_status` | Per-transaction apply during accept                |
+| `txq.cleanup`      | TxQ.cpp     | `ledger_seq`                                             | Post-close cleanup of expired queue entries        |
 
 ### Consensus Spans (Phase 4)
 
-| Span Name                      | Source File      | Attributes                                                                                                                                                                                                                        | Description                                                                                                                           |
-| ------------------------------ | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- |
-| `consensus.round`              | RCLConsensus.cpp | `xrpl.consensus.ledger_id`, `xrpl.ledger.seq`, `xrpl.consensus.mode`, `trace_strategy`, `xrpl.consensus.round_id`                                                                                                                 | Root span for a consensus round (deterministic or random trace ID)                                                                    |
-| `consensus.phase.open`         | Consensus.h      | --                                                                                                                                                                                                                                | Open phase duration (child of round)                                                                                                  |
-| `consensus.proposal.send`      | RCLConsensus.cpp | `xrpl.consensus.round`, `is_bow_out`                                                                                                                                                                                              | Consensus proposal broadcast                                                                                                          |
-| `consensus.ledger_close`       | RCLConsensus.cpp | `xrpl.ledger.seq`, `xrpl.consensus.mode`                                                                                                                                                                                          | Ledger close event                                                                                                                    |
-| `consensus.establish`          | Consensus.h      | `converge_percent`, `establish_count`, `proposers`                                                                                                                                                                                | Establish phase duration (child of round)                                                                                             |
-| `consensus.update_positions`   | Consensus.h      | `converge_percent`, `proposers`, `disputes_count`                                                                                                                                                                                 | Position update and dispute resolution (see Events below)                                                                             |
-| `consensus.check`              | Consensus.h      | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `consensus_result`                                                                                                         | Consensus threshold check                                                                                                             |
-| `consensus.accept`             | RCLConsensus.cpp | `proposers`, `round_time_ms`, `quorum`, `disputes_count`, `consensus_state`                                                                                                                                                       | Ledger accepted by consensus                                                                                                          |
-| `consensus.accept.apply`       | RCLConsensus.cpp | `xrpl.ledger.seq`, `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction`, `tx_count` | Ledger application with close time details (see Events below)                                                                         |
-| `consensus.validation.send`    | RCLConsensus.cpp | `xrpl.ledger.seq`, `proposing`, `ledger_hash`, `full_validation`, `validation_sign_time`                                                                                                                                          | Validation sent after accept (follows-from link)                                                                                      |
-| `consensus.mode_change`        | RCLConsensus.cpp | `mode_old`, `mode_new`                                                                                                                                                                                                            | Consensus mode transition                                                                                                             |
-| `consensus.proposal.receive`   | PeerImp.cpp      | `trusted`, `xrpl.consensus.round`                                                                                                                                                                                                 | Proposal received from peer (extracts parent context from TraceContext when present; falls back to standalone span for older peers)   |
-| `consensus.validation.receive` | PeerImp.cpp      | `trusted`, `xrpl.ledger.seq`                                                                                                                                                                                                      | Validation received from peer (extracts parent context from TraceContext when present; falls back to standalone span for older peers) |
+| Span Name                      | Source File      | Attributes                                                                                                                                                                                                                   | Description                                                                                                                           |
+| ------------------------------ | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- |
+| `consensus.round`              | RCLConsensus.cpp | `consensus_ledger_id`, `ledger_seq`, `consensus_mode`, `trace_strategy`, `consensus_round_id`                                                                                                                                | Root span for a consensus round (deterministic or random trace ID)                                                                    |
+| `consensus.phase.open`         | Consensus.h      | --                                                                                                                                                                                                                           | Open phase duration (child of round)                                                                                                  |
+| `consensus.proposal.send`      | RCLConsensus.cpp | `consensus_round`, `is_bow_out`                                                                                                                                                                                              | Consensus proposal broadcast                                                                                                          |
+| `consensus.ledger_close`       | RCLConsensus.cpp | `ledger_seq`, `consensus_mode`                                                                                                                                                                                               | Ledger close event                                                                                                                    |
+| `consensus.establish`          | Consensus.h      | `converge_percent`, `establish_count`, `proposers`                                                                                                                                                                           | Establish phase duration (child of round)                                                                                             |
+| `consensus.update_positions`   | Consensus.h      | `converge_percent`, `proposers`, `disputes_count`                                                                                                                                                                            | Position update and dispute resolution (see Events below)                                                                             |
+| `consensus.check`              | Consensus.h      | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `consensus_result`                                                                                                    | Consensus threshold check                                                                                                             |
+| `consensus.accept`             | RCLConsensus.cpp | `proposers`, `round_time_ms`, `quorum`, `disputes_count`, `consensus_state`                                                                                                                                                  | Ledger accepted by consensus                                                                                                          |
+| `consensus.accept.apply`       | RCLConsensus.cpp | `ledger_seq`, `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction`, `tx_count` | Ledger application with close time details (see Events below)                                                                         |
+| `consensus.validation.send`    | RCLConsensus.cpp | `ledger_seq`, `proposing`, `ledger_hash`, `full_validation`, `validation_sign_time`                                                                                                                                          | Validation sent after accept (follows-from link)                                                                                      |
+| `consensus.mode_change`        | RCLConsensus.cpp | `mode_old`, `mode_new`                                                                                                                                                                                                       | Consensus mode transition                                                                                                             |
+| `consensus.proposal.receive`   | PeerImp.cpp      | `trusted`, `consensus_round`                                                                                                                                                                                                 | Proposal received from peer (extracts parent context from TraceContext when present; falls back to standalone span for older peers)   |
+| `consensus.validation.receive` | PeerImp.cpp      | `trusted`, `ledger_seq`                                                                                                                                                                                                      | Validation received from peer (extracts parent context from TraceContext when present; falls back to standalone span for older peers) |
 
 #### Consensus Span Events
 
-| Parent Span                  | Event Name        | Event Attributes                                                 | Description                                             |
-| ---------------------------- | ----------------- | ---------------------------------------------------------------- | ------------------------------------------------------- |
-| `consensus.update_positions` | `dispute.resolve` | `xrpl.tx.id`, `dispute_our_vote`, `dispute_yays`, `dispute_nays` | Emitted per dispute when votes are tallied              |
-| `consensus.accept.apply`     | `tx.included`     | `xrpl.tx.id`                                                     | Emitted per transaction included in the accepted ledger |
+| Parent Span                  | Event Name        | Event Attributes                                            | Description                                             |
+| ---------------------------- | ----------------- | ----------------------------------------------------------- | ------------------------------------------------------- |
+| `consensus.update_positions` | `dispute.resolve` | `tx_id`, `dispute_our_vote`, `dispute_yays`, `dispute_nays` | Emitted per dispute when votes are tallied              |
+| `consensus.accept.apply`     | `tx.included`     | `tx_id`                                                     | Emitted per transaction included in the accepted ledger |
 
 #### Close Time Queries (Tempo TraceQL)
 
+Span attributes are filtered with `span.<attr>` inside `{}`. Combine conditions with `&&`.
+
 ```
 # Find rounds where validators disagreed on close time
-{name="consensus.accept.apply"} | close_time_correct = false
+{name="consensus.accept.apply" && span.close_time_correct = false}
 
 # Find consensus failures (moved_on)
-{name="consensus.accept.apply"} | consensus_state = "moved_on"
+{name="consensus.accept.apply" && span.consensus_state = "moved_on"}
 
 # Find slow ledger applications (>5s)
-{name="consensus.accept.apply"} | duration > 5s
+{name="consensus.accept.apply" && duration > 5000ms}
 
 # Find specific ledger's consensus details
-{name="consensus.accept.apply"} | xrpl.ledger.seq = 92345678
+{name="consensus.accept.apply" && span.ledger_seq = 92345678}
 
 # Find all spans in a consensus round (deterministic trace strategy)
-{name="consensus.round"} | xrpl.consensus.round_id = "<round_id>"
+{name="consensus.round" && span.consensus_round_id = "<round_id>"}
 
 # Find dispute resolutions
 {name="consensus.update_positions"} >> {event:name="dispute.resolve"}
 ```
 
+### Ledger Spans (Phase 6)
+
+| Span Name         | Source File          | Attributes                            | Description                   |
+| ----------------- | -------------------- | ------------------------------------- | ----------------------------- |
+| `ledger.build`    | BuildLedger.cpp:31   | `ledger_seq`, `tx_count`, `tx_failed` | Ledger build during consensus |
+| `ledger.validate` | LedgerMaster.cpp:915 | `ledger_seq`, `validations`           | Ledger promoted to validated  |
+| `ledger.store`    | LedgerMaster.cpp:409 | `ledger_seq`                          | Ledger stored in history      |
+
+### Peer Spans (Phase 6)
+
+| Span Name                 | Source File      | Attributes                      | Description                   |
+| ------------------------- | ---------------- | ------------------------------- | ----------------------------- |
+| `peer.proposal.receive`   | PeerImp.cpp:1667 | `peer_id`, `proposal_trusted`   | Proposal received from peer   |
+| `peer.validation.receive` | PeerImp.cpp:2264 | `peer_id`, `validation_trusted` | Validation received from peer |
+
+---
+
 ## Insights and Sample Queries
 
-This section shows what questions you can now answer using the enriched span attributes, with example Tempo TraceQL queries.
+This section shows what questions you can answer using the span attributes, with example Tempo TraceQL queries.
+
+**TraceQL syntax note:** span attributes must be referenced with the `span.` prefix inside `{}`.
+Conditions are combined with `&&`. The `|` pipeline operator is not supported on this Tempo version.
+
+```
+# General pattern
+{name="<span-name>" && span.<attr> = <value> && span.<attr2> != <value2>}
+
+# Duration filter (no prefix needed)
+{name="<span-name>" && duration > 500ms}
+
+# Regex match
+{name="<span-name>" && span.<attr> =~ "<pattern>.*"}
+
+# Multiple span names
+{name = "<span-a>" || name = "<span-b>"}
+
+# Name regex
+{name =~ "<pattern>.*" && span.<attr> = <value>}
+
+# Structural: find parent spans that have a matching child/event
+{name="<parent>"} >> {event:name="<event-name>"}
+```
 
 ### Transaction Workflow Analysis
 
 ```
-# Find all AMM transactions (AMMDeposit, AMMWithdraw, AMMCreate, etc.)
-{name="tx.process"} | tx_type =~ "AMM.*"
+# Find all AMM transactions (AMMDeposit, AMMWithdraw, AMMVote)
+{name="tx.process" && span.tx_type =~ "AMM.*"}
+
+# Find a specific AMM operation
+{name="tx.process" && span.tx_type = "AMMDeposit"}
+{name="tx.process" && span.tx_type = "AMMWithdraw"}
+{name="tx.process" && span.tx_type = "AMMVote"}
 
 # Find Payment transactions that failed
-{name="tx.process"} | tx_type = "Payment" && ter_result != "tesSUCCESS"
+{name="tx.process" && span.tx_type = "Payment" && span.ter_result != "tesSUCCESS"}
+
+# Find Payment failures due to path issues
+{name="tx.process" && span.tx_type = "Payment" && span.ter_result =~ "tecPATH.*"}
 
 # Compare latency of different transaction types
-{name="tx.process"} | tx_type = "OfferCreate"
-{name="tx.process"} | tx_type = "Payment"
+{name="tx.process" && span.tx_type = "OfferCreate"}
+{name="tx.process" && span.tx_type = "Payment"}
 
 # Find high-fee transactions (fee > 1 XRP = 1000000 drops)
-{name="tx.process"} | fee > 1000000
+{name="tx.process" && span.fee > 1000000}
 
 # Find transactions that were not applied
-{name="tx.process"} | applied = false
+{name="tx.process" && span.applied = false}
 
-# Trace a specific transaction by type across the network
-{name=~"tx\\..*"} | tx_type = "NFTokenMint"
+# Find NFTokenMint across tx and txq spans
+{name =~ "tx.*|txq.*" && span.tx_type = "NFTokenMint"}
+
+# Find all NFT-related activity
+{name =~ "tx.*|txq.*" && span.tx_type =~ "NFToken.*"}
+
+# Find TrustSet transactions (IOU trust lines)
+{name="tx.process" && span.tx_type = "TrustSet"}
+
+# Find oracle price updates
+{name="tx.process" && span.tx_type = "OracleSet"}
 ```
 
+### DEX (OfferCreate / OfferCancel)
+
+```
+# All DEX offer creates
+{name="tx.process" && span.tx_type = "OfferCreate"}
+
+# Offers killed (ImmediateOrCancel/FillOrKill with no fill)
+{name="tx.process" && span.tx_type = "OfferCreate" && span.ter_result = "tecKILLED"}
+
+# Offers that failed due to insufficient funds
+{name="tx.process" && span.tx_type = "OfferCreate" && span.ter_result = "tecUNFUNDED_OFFER"}
+
+# Offers failed due to insufficient reserve to place the offer
+{name="tx.process" && span.tx_type = "OfferCreate" && span.ter_result = "tecINSUF_RESERVE_OFFER"}
+
+# Offer cancellations
+{name="tx.process" && span.tx_type = "OfferCancel"}
+
+# OfferCreate transactions received from peers (cross-node relay)
+{name="tx.receive" && span.tx_type = "OfferCreate"}
+```
+
+### Apply Pipeline by Stage
+
+```
+# All three stages of one transaction (preflight -> preclaim -> apply)
+{name=~"tx.preflight|tx.preclaim|tx.transactor"}
+
+# Transactions that failed at the preclaim stage
+{name="tx.preclaim"} | ter_result != "tesSUCCESS"
+
+# Transactions that hard-failed preflight (never reached preclaim/apply)
+{name="tx.preflight"} | ter_result != "tesSUCCESS"
+```
+
+PromQL on the span-derived metrics (dashboard: _Transaction Overview_):
+
+```
+# Per-stage throughput — the funnel preflight >= preclaim >= apply
+sum by (stage) (rate(traces_span_metrics_calls_total{span_name=~"tx.preflight|tx.preclaim|tx.transactor"}[5m]))
+
+# Per-stage p95 latency
+histogram_quantile(0.95, sum by (le, stage) (rate(traces_span_metrics_duration_milliseconds_bucket{span_name=~"tx.preflight|tx.preclaim|tx.transactor"}[5m])))
+
+# Per-stage failure rate (ter_result != tesSUCCESS; a failing ter completes the
+# span normally, so filter on the attribute, not status_code which only flags exceptions)
+sum by (stage) (rate(traces_span_metrics_calls_total{span_name=~"tx.preflight|tx.preclaim|tx.transactor", ter_result!~"tesSUCCESS|"}[5m]))
+```
+
+> **Alerting**: a rising `tx.preflight` / `tx.preclaim` failure rate points to
+> malformed or stale-sequence submissions (often spam or a misbehaving client);
+> a rising `tx.transactor` failure rate points to apply-time problems. Alert per
+> stage rather than on a single aggregate so the failing stage is obvious.
+
+> **Sampling caveat**: these stage metrics are span-derived and inherit the
+> **tracer head-sampling** ratio (`sampling_ratio`). At `sampling_ratio < 1.0`
+> they undercount proportionally — treat them as relative trends, not absolute
+> transaction counts. Native StatsD metrics are unsampled.
+
 ### Transaction Queue Health
 
 ```
 # Find transactions rejected from the queue
-{name="txq.accept_tx"} | txq_status = "failed"
+{name="txq.accept_tx" && span.txq_status = "failed"}
 
-# Which transaction types get queued most often?
-{name="txq.enqueue"} | tx_type = "Payment"
-{name="txq.enqueue"} | tx_type = "OfferCreate"
-
-# Find ledger closes that applied queued transactions
-{name="txq.accept"} | ledger_changed = true
+# Find transactions being retried
+{name="txq.accept_tx" && span.txq_status = "retried"}
 
 # Find transactions that exhausted retries
-{name="txq.accept_tx"} | txq_status = "retried" && retries_remaining = 0
+{name="txq.accept_tx" && span.txq_status = "retried" && span.retries_remaining = 0}
+
+# Which transaction types get queued most often?
+{name="txq.enqueue" && span.tx_type = "Payment"}
+{name="txq.enqueue" && span.tx_type = "OfferCreate"}
+{name="txq.enqueue" && span.tx_type =~ "NFToken.*"}
+
+# Find ledger closes that applied queued transactions
+{name="txq.accept" && span.ledger_changed = true}
 ```
 
 ### RPC Debugging
 
 ```
 # Find batch RPC requests
-{name="rpc.process"} | is_batch = true
+{name="rpc.process" && span.is_batch = true}
 
 # Find large RPC payloads (>100KB)
-{name="rpc.http_request"} | request_payload_size > 100000
+{name="rpc.http_request" && span.request_payload_size > 100000}
 
 # Find resource-heavy RPC commands (by load_type)
-{name=~"rpc.command.*"} | load_type = "exception_rpc"
+{name =~ "rpc.command.*" && span.load_type = "exceptioned RPC"}
 
 # Find a specific WebSocket command
-{name="rpc.ws_message"} | command = "subscribe"
+{name="rpc.ws_message" && span.command = "subscribe"}
+
+# Find server_info calls
+{name="rpc.command.server_info"}
 
 # Find slow pathfinding with many source assets
-{name="pathfind.discover"} | pathfind_num_source_assets > 10
+{name="pathfind.discover" && span.pathfind_num_source_assets > 10}
 ```
 
 ### PathFinding Performance
 
 ```
 # Find pathfinding for specific currencies
-{name="pathfind.compute"} | pathfind_dest_currency = "USD"
+{name="pathfind.compute" && span.pathfind_dest_currency = "USD"}
 
 # Find expensive pathfinding (many source assets to explore)
-{name="pathfind.discover"} | pathfind_num_source_assets > 20
+{name="pathfind.discover" && span.pathfind_num_source_assets > 20}
 
-# Find large pathfinding requests
-{name="pathfind.compute"} | duration > 1s
+# Find slow pathfinding requests
+{name="pathfind.compute" && duration > 1000ms}
 ```
 
 ### Consensus Health
 
 ```
 # Find rounds where consensus timed out (expired)
-{name="consensus.accept"} | consensus_state = "expired"
+{name="consensus.accept" && span.consensus_state = "expired"}
 
 # Find rounds where we moved on without full agreement
-{name="consensus.accept"} | consensus_state = "moved_on"
+{name="consensus.accept" && span.consensus_state = "moved_on"}
 
 # Find rounds with many disputes
-{name="consensus.accept"} | disputes_count > 5
+{name="consensus.accept" && span.disputes_count > 5}
+
+# Find slow consensus rounds (>5s)
+{name="consensus.accept" && span.round_time_ms > 5000}
 
 # Find bow-out proposals (node resigned from round)
-{name="consensus.proposal.send"} | is_bow_out = true
+{name="consensus.proposal.send" && span.is_bow_out = true}
 
 # Correlate validation with its ledger
-{name="consensus.validation.send"} | ledger_hash = "<hash>"
+{name="consensus.validation.send" && span.ledger_hash = "<hash>"}
 
 # Find rounds where validators disagreed on close time
-{name="consensus.accept.apply"} | close_time_correct = false
+{name="consensus.accept.apply" && span.close_time_correct = false}
+
+# Find both validation send and receive (compare sender vs receiver latency)
+{name = "consensus.validation.send" || name = "consensus.validation.receive"}
 ```
 
 ### Cross-Subsystem Correlation
 
 ```
 # Follow a transaction from receive through queue to ledger
-{name=~"tx\\..*|txq\\..*"} | tx_type = "Payment" && duration > 500ms
+{name =~ "tx.*|txq.*" && span.tx_type = "Payment" && duration > 500ms}
 
-# Find all NFT-related activity
-{name=~"tx\\..*|txq\\..*"} | tx_type =~ "NFToken.*"
+# Find all NFT-related activity across tx and txq spans
+{name =~ "tx.*|txq.*" && span.tx_type =~ "NFToken.*"}
 
-# Find consensus rounds with slow transactions
-{name="consensus.accept"} | round_time_ms > 5000
+# Find all AMM activity across tx and txq spans
+{name =~ "tx.*|txq.*" && span.tx_type =~ "AMM.*"}
+
+# Find cross-node transaction receives (no errors)
+{name="tx.receive" && status != error}
 ```
 
 ### Where to Look (Quick Reference)
 
-| Question                            | Span                        | Key Attributes                 |
-| ----------------------------------- | --------------------------- | ------------------------------ |
-| "Which tx type is slowest?"         | `tx.process`                | `tx_type` + duration           |
-| "Why was my tx rejected?"           | `tx.process`                | `ter_result`, `applied`        |
-| "Is the TxQ backing up?"            | `txq.accept`                | `queue_size`, `ledger_changed` |
-| "Why was my tx dropped from queue?" | `txq.accept_tx`             | `txq_status`, `ter_code`       |
-| "Are batch requests a problem?"     | `rpc.process`               | `is_batch`, `batch_size`       |
-| "Which RPC is expensive?"           | `rpc.command.*`             | `load_type`, duration          |
-| "Did consensus stall?"              | `consensus.check`           | `consensus_stalled`            |
-| "Was consensus outcome normal?"     | `consensus.accept`          | `consensus_state`              |
-| "Did a validator bow out?"          | `consensus.proposal.send`   | `is_bow_out`                   |
-| "Which ledger was validated?"       | `consensus.validation.send` | `ledger_hash`                  |
+| Question                            | Span                        | Key Attributes                           |
+| ----------------------------------- | --------------------------- | ---------------------------------------- |
+| "Which tx type is slowest?"         | `tx.process`                | `span.tx_type` + duration                |
+| "Why was my tx rejected?"           | `tx.process`                | `span.ter_result`, `span.applied`        |
+| "What AMM operations happened?"     | `tx.process`                | `span.tx_type =~ "AMM.*"`                |
+| "What DEX offers failed?"           | `tx.process`                | `span.tx_type`, `span.ter_result`        |
+| "What NFT activity occurred?"       | `tx.process`, `txq.enqueue` | `span.tx_type =~ "NFToken.*"`            |
+| "Is the TxQ backing up?"            | `txq.accept`                | `span.queue_size`, `span.ledger_changed` |
+| "Why was my tx dropped from queue?" | `txq.accept_tx`             | `span.txq_status`, `span.ter_code`       |
+| "Are batch requests a problem?"     | `rpc.process`               | `span.is_batch`, `span.batch_size`       |
+| "Which RPC is expensive?"           | `rpc.command.*`             | `span.load_type`, duration               |
+| "Did consensus reach threshold?"    | `consensus.check`           | `span.consensus_result`                  |
+| "Was consensus outcome normal?"     | `consensus.accept`          | `span.consensus_state`                   |
+| "Did a validator bow out?"          | `consensus.proposal.send`   | `span.is_bow_out`                        |
+| "Which ledger was validated?"       | `consensus.validation.send` | `span.ledger_hash`                       |
+| "Did close time agreement fail?"    | `consensus.accept.apply`    | `span.close_time_correct`                |
 
 ---
 
@@ -330,20 +479,20 @@ all its normal attributes, it just lacks a cross-node parent link.
 ### Example Tempo Queries
 
 ```
-# Find cross-node transaction traces (tx.process -> tx.receive across nodes)
-{name="tx.receive"} && status != error
+# Find cross-node transaction traces (tx.receive spans with no errors)
+{name="tx.receive" && status != error}
 
 # Find proposals received with cross-node parent context
-{name="consensus.proposal.receive"} && nestedSetParent > 0
+{name="consensus.proposal.receive"}
 
 # Trace a transaction across the network by its hash
-{name=~"tx\\..*"} | xrpl.tx.hash = "<hash>"
+{name =~ "tx.*" && span.tx_hash = "<hash>"}
 
 # Find all spans in a cross-node consensus trace
-{rootServiceName="xrpld"} | xrpl.consensus.round_id = "<round_id>"
+{resource.service.name="xrpld" && span.consensus_round_id = "<round_id>"}
 
 # Compare latency between sender and receiver for validations
-{name="consensus.validation.send" || name="consensus.validation.receive"}
+{name = "consensus.validation.send" || name = "consensus.validation.receive"}
 ```
 
 ## Prometheus Metrics (Spanmetrics)
@@ -370,14 +519,16 @@ Every metric carries these standard labels:
 | `service_name` | Resource attribute | `xrpld`                                  |
 | `span_kind`    | Span kind          | `SPAN_KIND_INTERNAL`                     |
 
-Additionally, span attributes configured as dimensions in the collector become metric labels (dots → underscores):
+Additionally, span attributes configured as dimensions in the collector become metric labels. The collector dimensions use the bare attribute keys emitted by the code, so the label name equals the attribute name:
 
-| Span Attribute        | Metric Label          | Applies To                     |
-| --------------------- | --------------------- | ------------------------------ |
-| `command`             | `xrpl_rpc_command`    | `rpc.command.*` spans          |
-| `rpc_status`          | `xrpl_rpc_status`     | `rpc.command.*` spans          |
-| `xrpl.consensus.mode` | `xrpl_consensus_mode` | `consensus.ledger_close` spans |
-| `local`               | `xrpl_tx_local`       | `tx.process` spans             |
+| Span Attribute       | Metric Label         | Applies To                      |
+| -------------------- | -------------------- | ------------------------------- |
+| `command`            | `command`            | `rpc.command.*` spans           |
+| `rpc_status`         | `rpc_status`         | `rpc.command.*` spans           |
+| `consensus_mode`     | `consensus_mode`     | `consensus.ledger_close` spans  |
+| `local`              | `local`              | `tx.process` spans              |
+| `proposal_trusted`   | `proposal_trusted`   | `peer.proposal.receive` spans   |
+| `validation_trusted` | `validation_trusted` | `peer.validation.receive` spans |
 
 ### Histogram Buckets
 
@@ -387,50 +538,226 @@ Configured in `otel-collector-config.yaml`:
 1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 5s
 ```
 
+## System Metrics (OTel native -- beast::insight)
+
+xrpld has a built-in metrics framework (`beast::insight`) that exports metrics natively via OTLP to the OTel Collector. These complement the span-derived RED metrics by providing system-level gauges, counters, and timers that don't map to individual trace spans.
+
+### Configuration
+
+Add to `xrpld.cfg`:
+
+```ini
+[insight]
+server=otel
+endpoint=http://localhost:4318/v1/metrics
+prefix=xrpld
+```
+
+The `OTelCollector` implementation exports metrics via OTLP/HTTP to the same OTel Collector that receives traces. No separate StatsD receiver is needed.
+
+> **Fallback**: Set `server=statsd` and `address=127.0.0.1:8125` to use the legacy StatsD UDP path. This requires re-enabling the `statsd` receiver in `otel-collector-config.yaml` and uncommenting port 8125 in `docker-compose.yml`.
+
+### Metric Reference
+
+#### Gauges
+
+| Prometheus Metric                           | Source                    | Description                                                                |
+| ------------------------------------------- | ------------------------- | -------------------------------------------------------------------------- |
+| `xrpld_LedgerMaster_Validated_Ledger_Age`   | LedgerMaster.h:373        | Age of validated ledger (seconds)                                          |
+| `xrpld_LedgerMaster_Published_Ledger_Age`   | LedgerMaster.h:374        | Age of published ledger (seconds)                                          |
+| `xrpld_State_Accounting_{Mode}_duration`    | NetworkOPs.cpp:774        | Time in each operating mode (Disconnected/Connected/Syncing/Tracking/Full) |
+| `xrpld_State_Accounting_{Mode}_transitions` | NetworkOPs.cpp:780        | Transition count per mode                                                  |
+| `xrpld_Peer_Finder_Active_Inbound_Peers`    | PeerfinderManager.cpp:214 | Active inbound peer connections                                            |
+| `xrpld_Peer_Finder_Active_Outbound_Peers`   | PeerfinderManager.cpp:215 | Active outbound peer connections                                           |
+| `xrpld_Overlay_Peer_Disconnects`            | OverlayImpl.h:557         | Peer disconnect count                                                      |
+| `xrpld_job_count`                           | JobQueue.cpp:26           | Current job queue depth                                                    |
+| `xrpld_{category}_Bytes_In/Out`             | OverlayImpl.h:535         | Overlay traffic bytes per category (57 categories)                         |
+| `xrpld_{category}_Messages_In/Out`          | OverlayImpl.h:535         | Overlay traffic messages per category                                      |
+
+#### OTel MetricsRegistry Gauges (Phase 9)
+
+These gauges are exported via the OTel Metrics SDK `PeriodicMetricReader` (10s interval), NOT through beast::insight.
+
+| Prometheus Metric                                         | Source              | Description                                  |
+| --------------------------------------------------------- | ------------------- | -------------------------------------------- |
+| `xrpld_server_info{metric="server_state"}`                | MetricsRegistry.cpp | Operating mode (0=DISCONNECTED .. 4=FULL)    |
+| `xrpld_server_info{metric="uptime"}`                      | MetricsRegistry.cpp | Seconds since server start                   |
+| `xrpld_server_info{metric="peers"}`                       | MetricsRegistry.cpp | Total connected peers                        |
+| `xrpld_server_info{metric="validated_ledger_seq"}`        | MetricsRegistry.cpp | Validated ledger sequence number             |
+| `xrpld_server_info{metric="ledger_current_index"}`        | MetricsRegistry.cpp | Current open ledger sequence                 |
+| `xrpld_server_info{metric="peer_disconnects_resources"}`  | MetricsRegistry.cpp | Cumulative resource-related peer disconnects |
+| `xrpld_server_info{metric="last_close_proposers"}`        | MetricsRegistry.cpp | Proposers in last closed round               |
+| `xrpld_server_info{metric="last_close_converge_time_ms"}` | MetricsRegistry.cpp | Last close convergence time (ms)             |
+| `xrpld_build_info{version="<ver>"}`                       | MetricsRegistry.cpp | Info-style metric (always 1)                 |
+| `xrpld_complete_ledgers{bound="start\|end",index="<N>"}`  | MetricsRegistry.cpp | Complete ledger range start/end pairs        |
+| `xrpld_db_metrics{metric="db_kb_total"}`                  | MetricsRegistry.cpp | Total database size (KB)                     |
+| `xrpld_db_metrics{metric="db_kb_ledger"}`                 | MetricsRegistry.cpp | Ledger database size (KB)                    |
+| `xrpld_db_metrics{metric="db_kb_transaction"}`            | MetricsRegistry.cpp | Transaction database size (KB)               |
+| `xrpld_db_metrics{metric="historical_perminute"}`         | MetricsRegistry.cpp | Historical ledger fetches per minute         |
+| `xrpld_cache_metrics{metric="AL_size"}`                   | MetricsRegistry.cpp | AcceptedLedger cache size                    |
+| `xrpld_nodestore_state{metric="node_reads_duration_us"}`  | MetricsRegistry.cpp | Cumulative read time (microseconds)          |
+| `xrpld_nodestore_state{metric="read_request_bundle"}`     | MetricsRegistry.cpp | Read request bundle count                    |
+| `xrpld_nodestore_state{metric="read_threads_running"}`    | MetricsRegistry.cpp | Active read threads                          |
+| `xrpld_nodestore_state{metric="read_threads_total"}`      | MetricsRegistry.cpp | Total read threads configured                |
+
+#### Counters
+
+| Prometheus Metric               | Source                | Description                    |
+| ------------------------------- | --------------------- | ------------------------------ |
+| `xrpld_rpc_requests`            | ServerHandler.cpp:108 | Total RPC request count        |
+| `xrpld_ledger_fetches`          | InboundLedgers.cpp:44 | Ledger fetch request count     |
+| `xrpld_ledger_history_mismatch` | LedgerHistory.cpp:16  | Ledger hash mismatch count     |
+| `xrpld_warn`                    | Logic.h:33            | Resource manager warning count |
+| `xrpld_drop`                    | Logic.h:34            | Resource manager drop count    |
+
+#### Histograms
+
+| Prometheus Metric     | Source                | Description                    |
+| --------------------- | --------------------- | ------------------------------ |
+| `xrpld_rpc_time`      | ServerHandler.cpp:110 | RPC response time (ms)         |
+| `xrpld_rpc_size`      | ServerHandler.cpp:109 | RPC response size (bytes)      |
+| `xrpld_ios_latency`   | Application.cpp:438   | I/O service loop latency (ms)  |
+| `xrpld_pathfind_fast` | PathRequests.h:23     | Fast pathfinding duration (ms) |
+| `xrpld_pathfind_full` | PathRequests.h:24     | Full pathfinding duration (ms) |
+
 ## Grafana Dashboards
 
-Three dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`:
+Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`:
 
 ### RPC Performance (`xrpld-rpc-perf`)
 
-| Panel                       | Type       | PromQL                                                                                                                                             | Labels Used                       |
-| --------------------------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------- |
-| RPC Request Rate by Command | timeseries | `sum by (xrpl_rpc_command) (rate(traces_span_metrics_calls_total{span_name=~"rpc.command.*"}[5m]))`                                                | `xrpl_rpc_command`                |
-| RPC Latency p95 by Command  | timeseries | `histogram_quantile(0.95, sum by (le, xrpl_rpc_command) (rate(traces_span_metrics_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m])))` | `xrpl_rpc_command`                |
-| RPC Error Rate              | bargauge   | Error spans / total spans × 100, grouped by `xrpl_rpc_command`                                                                                     | `xrpl_rpc_command`, `status_code` |
-| RPC Latency Heatmap         | heatmap    | `sum(increase(traces_span_metrics_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m])) by (le)`                                          | `le` (bucket boundaries)          |
+| Panel                       | Type       | PromQL                                                                                                                                    | Labels Used              |
+| --------------------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ |
+| RPC Request Rate by Command | timeseries | `sum by (command) (rate(traces_span_metrics_calls_total{span_name=~"rpc.command.*"}[5m]))`                                                | `command`                |
+| RPC Latency p95 by Command  | timeseries | `histogram_quantile(0.95, sum by (le, command) (rate(traces_span_metrics_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m])))` | `command`                |
+| RPC Error Rate              | bargauge   | Error spans / total spans × 100, grouped by `command`                                                                                     | `command`, `status_code` |
+| RPC Latency Heatmap         | heatmap    | `sum(increase(traces_span_metrics_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m])) by (le)`                                 | `le` (bucket boundaries) |
+| Overall RPC Throughput      | timeseries | `rpc.request` + `rpc.process` rate                                                                                                        | —                        |
+| RPC Success vs Error        | timeseries | by `status_code` (UNSET vs ERROR)                                                                                                         | `status_code`            |
+| Top Commands by Volume      | bargauge   | `topk(10, ...)` by `command`                                                                                                              | `command`                |
+| WebSocket Message Rate      | stat       | `rpc.ws_message` rate                                                                                                                     | —                        |
 
 ### Transaction Overview (`xrpld-transactions`)
 
-| Panel                             | Type       | PromQL                                                                                       | Labels Used     |
-| --------------------------------- | ---------- | -------------------------------------------------------------------------------------------- | --------------- |
-| Transaction Processing Rate       | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m])` and `tx.receive`         | `span_name`     |
-| Transaction Processing Latency    | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="tx.process"})`                              | —               |
-| Transaction Path Distribution     | piechart   | `sum by (xrpl_tx_local) (rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m]))` | `xrpl_tx_local` |
-| Transaction Receive vs Suppressed | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.receive"}[5m])`                          | —               |
+| Panel                             | Type       | PromQL                                                                               | Labels Used   |
+| --------------------------------- | ---------- | ------------------------------------------------------------------------------------ | ------------- |
+| Transaction Processing Rate       | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m])` and `tx.receive` | `span_name`   |
+| Transaction Processing Latency    | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="tx.process"})`                      | —             |
+| Transaction Path Distribution     | piechart   | `sum by (local) (rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m]))` | `local`       |
+| Transaction Receive vs Suppressed | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.receive"}[5m])`                  | —             |
+| TX Processing Duration Heatmap    | heatmap    | `tx.process` histogram buckets                                                       | `le`          |
+| TX Apply Duration per Ledger      | timeseries | p95/p50 of `tx.apply`                                                                | —             |
+| Peer TX Receive Rate              | timeseries | `tx.receive` rate                                                                    | —             |
+| TX Apply Failed Rate              | stat       | `tx.apply` with `STATUS_CODE_ERROR`                                                  | `status_code` |
 
 ### Consensus Health (`xrpld-consensus`)
 
-| Panel                         | Type       | PromQL                                                                             | Labels Used |
-| ----------------------------- | ---------- | ---------------------------------------------------------------------------------- | ----------- |
-| Consensus Round Duration      | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="consensus.accept"})`              | —           |
-| Consensus Proposals Sent Rate | timeseries | `rate(traces_span_metrics_calls_total{span_name="consensus.proposal.send"}[5m])`   | —           |
-| Ledger Close Duration         | timeseries | `histogram_quantile(0.95, ... {span_name="consensus.ledger_close"})`               | —           |
-| Validation Send Rate          | stat       | `rate(traces_span_metrics_calls_total{span_name="consensus.validation.send"}[5m])` | —           |
-| Ledger Apply Duration         | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="consensus.accept.apply"})`        | —           |
-| Close Time Agreement          | timeseries | `rate(traces_span_metrics_calls_total{span_name="consensus.accept.apply"}[5m])`    | —           |
+| Panel                         | Type       | PromQL                                                                             | Labels Used      |
+| ----------------------------- | ---------- | ---------------------------------------------------------------------------------- | ---------------- |
+| Consensus Round Duration      | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="consensus.accept"})`              | —                |
+| Consensus Proposals Sent Rate | timeseries | `rate(traces_span_metrics_calls_total{span_name="consensus.proposal.send"}[5m])`   | —                |
+| Ledger Close Duration         | timeseries | `histogram_quantile(0.95, ... {span_name="consensus.ledger_close"})`               | —                |
+| Validation Send Rate          | stat       | `rate(traces_span_metrics_calls_total{span_name="consensus.validation.send"}[5m])` | —                |
+| Ledger Apply Duration         | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="consensus.accept.apply"})`        | —                |
+| Close Time Agreement          | timeseries | `rate(traces_span_metrics_calls_total{span_name="consensus.accept.apply"}[5m])`    | —                |
+| Consensus Mode Over Time      | timeseries | `consensus.ledger_close` by `consensus_mode`                                       | `consensus_mode` |
+| Accept vs Close Rate          | timeseries | `consensus.accept` vs `consensus.ledger_close` rate                                | —                |
+| Validation vs Close Rate      | timeseries | `consensus.validation.send` vs `consensus.ledger_close`                            | —                |
+| Accept Duration Heatmap       | heatmap    | `consensus.accept` histogram buckets                                               | `le`             |
+
+### Ledger Operations (`xrpld-ledger-ops`)
+
+| Panel                   | Type       | PromQL                                         | Labels Used |
+| ----------------------- | ---------- | ---------------------------------------------- | ----------- |
+| Ledger Build Rate       | stat       | `ledger.build` call rate                       | —           |
+| Ledger Build Duration   | timeseries | p95/p50 of `ledger.build`                      | —           |
+| Ledger Validation Rate  | stat       | `ledger.validate` call rate                    | —           |
+| Build Duration Heatmap  | heatmap    | `ledger.build` histogram buckets               | `le`        |
+| TX Apply Duration       | timeseries | p95/p50 of `tx.apply`                          | —           |
+| TX Apply Rate           | timeseries | `tx.apply` call rate                           | —           |
+| Ledger Store Rate       | stat       | `ledger.store` call rate                       | —           |
+| Build vs Close Duration | timeseries | p95 `ledger.build` vs `consensus.ledger_close` | —           |
+
+### Peer Network (`xrpld-peer-net`)
+
+Requires `trace_peer=1` in the `[telemetry]` config section.
+
+| Panel                            | Type       | PromQL                         | Labels Used          |
+| -------------------------------- | ---------- | ------------------------------ | -------------------- |
+| Proposal Receive Rate            | timeseries | `peer.proposal.receive` rate   | —                    |
+| Validation Receive Rate          | timeseries | `peer.validation.receive` rate | —                    |
+| Proposals Trusted vs Untrusted   | piechart   | by `proposal_trusted`          | `proposal_trusted`   |
+| Validations Trusted vs Untrusted | piechart   | by `validation_trusted`        | `validation_trusted` |
+
+### Node Health -- System Metrics (`xrpld-system-node-health`)
+
+| Panel                                  | Type       | PromQL                                                          | Labels Used      |
+| -------------------------------------- | ---------- | --------------------------------------------------------------- | ---------------- |
+| Validated Ledger Age                   | stat       | `xrpld_LedgerMaster_Validated_Ledger_Age`                       | —                |
+| Published Ledger Age                   | stat       | `xrpld_LedgerMaster_Published_Ledger_Age`                       | —                |
+| Operating Mode Duration                | timeseries | `xrpld_State_Accounting_*_duration`                             | —                |
+| Operating Mode Transitions             | timeseries | `xrpld_State_Accounting_*_transitions`                          | —                |
+| I/O Latency                            | timeseries | `histogram_quantile(0.95, xrpld_ios_latency_bucket)`            | —                |
+| Job Queue Depth                        | timeseries | `xrpld_job_count`                                               | —                |
+| Ledger Fetch Rate                      | stat       | `rate(xrpld_ledger_fetches[5m])`                                | —                |
+| Ledger History Mismatches              | stat       | `rate(xrpld_ledger_history_mismatch[5m])`                       | —                |
+| Key Jobs Execution Time                | timeseries | `xrpld_acceptLedger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile`       |
+| Key Jobs Dequeue Wait Time             | timeseries | `xrpld_acceptLedger_q{quantile="$quantile"}` (+ 10 more)        | `quantile`       |
+| FullBelowCache Size                    | timeseries | `xrpld_Node_family_full_below_cache_size`                       | —                |
+| FullBelowCache Hit Rate                | gauge      | `xrpld_Node_family_full_below_cache_hit_rate`                   | —                |
+| Ledger Publish Gap                     | stat       | `Published_Ledger_Age - Validated_Ledger_Age`                   | —                |
+| State Duration Rate (Full vs Tracking) | timeseries | `rate(xrpld_State_Accounting_Full_duration[5m]) / 1000000`      | —                |
+| All Jobs Execution Time (Detail)       | timeseries | `{__name__=~"xrpld_<all_jobs>", quantile="$quantile"}`          | `quantile`       |
+| All Jobs Dequeue Wait (Detail)         | timeseries | `{__name__=~"xrpld_<all_jobs>_q", quantile="$quantile"}`        | `quantile`       |
+| Server State                           | stat       | `xrpld_server_info{metric="server_state"}`                      | `metric`         |
+| Uptime                                 | stat       | `xrpld_server_info{metric="uptime"}`                            | `metric`         |
+| Peer Count                             | stat       | `xrpld_server_info{metric="peers"}`                             | `metric`         |
+| Validated Ledger Seq                   | stat       | `xrpld_server_info{metric="validated_ledger_seq"}`              | `metric`         |
+| Build Version                          | stat       | `xrpld_build_info`                                              | `version`        |
+| Complete Ledger Ranges                 | table      | `xrpld_complete_ledgers`                                        | `bound`, `index` |
+| Database Sizes                         | timeseries | `xrpld_db_metrics{metric=~"db_kb_.*"}`                          | `metric`         |
+| Historical Fetch Rate                  | stat       | `xrpld_db_metrics{metric="historical_perminute"}`               | `metric`         |
+
+### Network Traffic -- System Metrics (`xrpld-system-network`)
+
+| Panel                                | Type       | PromQL                                     | Labels Used |
+| ------------------------------------ | ---------- | ------------------------------------------ | ----------- |
+| Active Peers                         | timeseries | `xrpld_Peer_Finder_Active_*_Peers`         | —           |
+| Peer Disconnects                     | timeseries | `xrpld_Overlay_Peer_Disconnects`           | —           |
+| Total Network Bytes                  | timeseries | `rate(xrpld_total_Bytes_In/Out[5m])`       | —           |
+| Total Network Messages               | timeseries | `xrpld_total_Messages_In/Out`              | —           |
+| Transaction Traffic                  | timeseries | `xrpld_transactions_Messages_In/Out`       | —           |
+| Proposal Traffic                     | timeseries | `xrpld_proposals_Messages_In/Out`          | —           |
+| Validation Traffic                   | timeseries | `xrpld_validations_Messages_In/Out`        | —           |
+| Traffic by Category                  | bargauge   | `topk(10, xrpld_*_Bytes_In)`               | —           |
+| Duplicate Traffic (Wasted Bandwidth) | timeseries | `rate(xrpld_*_duplicate_Bytes_In/Out[5m])` | —           |
+| All Traffic Categories (Detail)      | timeseries | `topk(15, rate(xrpld_*_Bytes_In[5m]))`     | —           |
+
+### RPC & Pathfinding -- System Metrics (`xrpld-system-rpc`)
+
+| Panel                     | Type       | PromQL                                                 | Labels Used |
+| ------------------------- | ---------- | ------------------------------------------------------ | ----------- |
+| RPC Request Rate          | stat       | `rate(xrpld_rpc_requests[5m])`                         | —           |
+| RPC Response Time         | timeseries | `histogram_quantile(0.95, xrpld_rpc_time_bucket)`      | —           |
+| RPC Response Size         | timeseries | `histogram_quantile(0.95, xrpld_rpc_size_bucket)`      | —           |
+| RPC Response Time Heatmap | heatmap    | `xrpld_rpc_time_bucket`                                | —           |
+| Pathfinding Fast Duration | timeseries | `histogram_quantile(0.95, xrpld_pathfind_fast_bucket)` | —           |
+| Pathfinding Full Duration | timeseries | `histogram_quantile(0.95, xrpld_pathfind_full_bucket)` | —           |
+| Resource Warnings Rate    | stat       | `rate(xrpld_warn[5m])`                                 | —           |
+| Resource Drops Rate       | stat       | `rate(xrpld_drop[5m])`                                 | —           |
 
 ### Span → Metric → Dashboard Summary
 
 | Span Name                      | Prometheus Metric Filter                     | Grafana Dashboard                             |
 | ------------------------------ | -------------------------------------------- | --------------------------------------------- |
-| `rpc.http_request`             | `{span_name="rpc.http_request"}`             | -- (available but not paneled)                |
+| `rpc.http_request`             | `{span_name="rpc.http_request"}`             | RPC Performance (Overall Throughput)          |
 | `rpc.ws_upgrade`               | `{span_name="rpc.ws_upgrade"}`               | -- (available but not paneled)                |
-| `rpc.ws_message`               | `{span_name="rpc.ws_message"}`               | -- (available but not paneled)                |
-| `rpc.process`                  | `{span_name="rpc.process"}`                  | -- (available but not paneled)                |
-| `rpc.command.*`                | `{span_name=~"rpc.command.*"}`               | RPC Performance (all 4 panels)                |
-| `tx.process`                   | `{span_name="tx.process"}`                   | Transaction Overview (3 panels)               |
-| `tx.receive`                   | `{span_name="tx.receive"}`                   | Transaction Overview (2 panels)               |
+| `rpc.ws_message`               | `{span_name="rpc.ws_message"}`               | RPC Performance (WebSocket Rate)              |
+| `rpc.process`                  | `{span_name="rpc.process"}`                  | RPC Performance (Overall Throughput)          |
+| `rpc.command.*`                | `{span_name=~"rpc.command.*"}`               | RPC Performance (Rate, Latency, Error, Top)   |
+| `tx.process`                   | `{span_name="tx.process"}`                   | Transaction Overview (Rate, Latency, Heatmap) |
+| `tx.receive`                   | `{span_name="tx.receive"}`                   | Transaction Overview (Rate, Receive)          |
+| `tx.apply`                     | `{span_name="tx.apply"}`                     | Transaction Overview + Ledger Ops (Apply)     |
 | `txq.enqueue`                  | `{span_name="txq.enqueue"}`                  | -- (available but not paneled)                |
 | `txq.apply_direct`             | `{span_name="txq.apply_direct"}`             | -- (available but not paneled)                |
 | `txq.batch_clear`              | `{span_name="txq.batch_clear"}`              | -- (available but not paneled)                |
@@ -442,14 +769,68 @@ Three dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`:
 | `consensus.establish`          | `{span_name="consensus.establish"}`          | -- (available but not paneled)                |
 | `consensus.update_positions`   | `{span_name="consensus.update_positions"}`   | -- (available but not paneled)                |
 | `consensus.check`              | `{span_name="consensus.check"}`              | -- (available but not paneled)                |
-| `consensus.accept`             | `{span_name="consensus.accept"}`             | Consensus Health (Round Duration)             |
+| `consensus.accept`             | `{span_name="consensus.accept"}`             | Consensus Health (Duration, Rate, Heatmap)    |
 | `consensus.proposal.send`      | `{span_name="consensus.proposal.send"}`      | Consensus Health (Proposals Rate)             |
-| `consensus.ledger_close`       | `{span_name="consensus.ledger_close"}`       | Consensus Health (Close Duration)             |
+| `consensus.ledger_close`       | `{span_name="consensus.ledger_close"}`       | Consensus Health (Close, Mode)                |
 | `consensus.validation.send`    | `{span_name="consensus.validation.send"}`    | Consensus Health (Validation Rate)            |
 | `consensus.accept.apply`       | `{span_name="consensus.accept.apply"}`       | Consensus Health (Apply Duration, Close Time) |
 | `consensus.mode_change`        | `{span_name="consensus.mode_change"}`        | -- (available but not paneled)                |
 | `consensus.proposal.receive`   | `{span_name="consensus.proposal.receive"}`   | -- (available but not paneled)                |
 | `consensus.validation.receive` | `{span_name="consensus.validation.receive"}` | -- (available but not paneled)                |
+| `ledger.build`                 | `{span_name="ledger.build"}`                 | Ledger Ops (Build Rate, Duration, Heatmap)    |
+| `ledger.validate`              | `{span_name="ledger.validate"}`              | Ledger Ops (Validation Rate)                  |
+| `ledger.store`                 | `{span_name="ledger.store"}`                 | Ledger Ops (Store Rate)                       |
+| `peer.proposal.receive`        | `{span_name="peer.proposal.receive"}`        | Peer Network (Rate, Trusted/Untrusted)        |
+| `peer.validation.receive`      | `{span_name="peer.validation.receive"}`      | Peer Network (Rate, Trusted/Untrusted)        |
+
+## Log-Trace Correlation (Phase 8)
+
+When xrpld is built with `telemetry=ON`, log lines emitted within an active OpenTelemetry span automatically include `trace_id` and `span_id` fields:
+
+```
+2024-01-15T10:30:45.123Z LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42
+```
+
+This enables bidirectional navigation between logs and traces in Grafana:
+
+- **Tempo -> Loki**: Click "Logs for this trace" on any trace in Grafana Tempo to see all log lines from that trace.
+- **Loki -> Tempo**: Click the `TraceID` derived field link on any log line containing `trace_id=` to jump to the full trace in Tempo.
+
+### Log Ingestion Pipeline
+
+Log files are ingested by the OTel Collector's `filelog` receiver, which tails `debug.log` files and parses them with a regex that extracts `timestamp`, `partition`, `severity`, `trace_id`, `span_id`, and `message` fields. Parsed entries are exported to Grafana Loki.
+
+### LogQL Query Examples
+
+The OTel Collector emits logs to Loki with `service_name="xrpld"` (not `job="xrpld"`).
+
+```logql
+# Find all logs for a specific trace
+{service_name="xrpld"} |= "trace_id=abc123def456789012345678abcdef01"
+
+# Error logs with trace context (log lines with ERR severity that have a trace_id)
+{service_name="xrpld"} |= "ERR" |= "trace_id="
+
+# All logs from a specific partition that were emitted during a span
+{service_name="xrpld"} |= "LedgerMaster" | regexp `trace_id=(?P<trace_id>[a-f0-9]+)` | trace_id != ""
+
+# Logs from a specific subsystem during a span (e.g. LedgerConsensus)
+{service_name="xrpld"} |= "LedgerConsensus" |= "trace_id="
+
+# Logs from the last hour containing trace context
+{service_name="xrpld"} |= "trace_id=" | regexp `(?P<partition>\S+):(?P<sev>\S+)\s+trace_id=(?P<tid>[a-f0-9]+)`
+
+# Count of traced vs untraced log lines
+count_over_time({service_name="xrpld"} |= "trace_id=" [5m])
+```
+
+### Verifying Log Correlation
+
+1. Start the observability stack and xrpld with telemetry enabled.
+2. Send an RPC request: `curl http://localhost:5005 -d '{"method":"server_info"}'`
+3. Check the debug.log for `trace_id=` entries: `grep trace_id= /path/to/debug.log`
+4. Open Grafana at http://localhost:3000 -> Explore -> Loki and search for `{service_name="xrpld"} |= "trace_id="`.
+5. Click the TraceID link to navigate to the corresponding trace in Tempo.
 
 ## Troubleshooting
 
@@ -462,6 +843,25 @@ Three dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`:
 5. Verify Tempo is receiving data: open Grafana → Explore → select Tempo datasource → search by `service.name = xrpld`
 6. Check Tempo logs: `docker compose -f docker/telemetry/docker-compose.yml logs tempo`
 
+### No system metrics in Prometheus
+
+1. Check xrpld logs for `OTelCollector starting` message
+2. Verify `server=otel` in the `[insight]` config section
+3. Verify the endpoint in `[insight]` points to the OTLP/HTTP port (default: `http://localhost:4318/v1/metrics`)
+4. Check that the `otlp` receiver is in the metrics pipeline receivers in `otel-collector-config.yaml`
+5. Query Prometheus directly: `curl 'http://localhost:9090/api/v1/query?query=xrpld_job_count'`
+
+### Server info gauge shows server_state=0
+
+This is normal during startup. The server starts in DISCONNECTED mode (0) and
+progresses through CONNECTED (1), SYNCING (2), TRACKING (3), to FULL (4).
+Wait for the node to sync with the network.
+
+### Database metrics showing zero
+
+The `getKBUsed*()` methods require SQLite databases to exist. If running with
+`--standalone` or before the first ledger is stored, these will be zero.
+
 ### High memory usage
 
 - Reduce `sampling_ratio` (e.g., `0.1` for 10% sampling)
@@ -474,6 +874,20 @@ Three dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`:
 - Check firewall rules for ports 4317/4318
 - If using TLS, verify certificate path with `tls_ca_cert`
 
+### No trace_id in log output
+
+- Verify xrpld was built with `telemetry=ON` (the `XRPL_ENABLE_TELEMETRY` preprocessor flag)
+- Verify `enabled=1` in the `[telemetry]` config section
+- Log lines only contain `trace_id`/`span_id` when emitted inside an active span — background logs outside of RPC/consensus/transaction processing will not have trace context
+- Check that the specific trace category is enabled (e.g., `trace_rpc=1`)
+
+### No logs in Loki
+
+- Verify the log file mount in docker-compose.yml points to the correct xrpld log directory
+- Check OTel Collector logs for filelog receiver errors: `docker compose logs otel-collector`
+- Verify Loki is running: `curl http://localhost:3100/ready`
+- Check the filelog receiver glob pattern matches your log file paths
+
 ## Performance Tuning
 
 | Scenario                 | Recommendation                                    |
@@ -492,3 +906,77 @@ cmake --preset default -Dtelemetry=OFF
 ```
 
 When telemetry is compiled out, all trace macros expand to no-ops with zero overhead.
+
+## Validating Telemetry Stack
+
+After deploying telemetry, use the Phase 10 workload tools to validate the full stack end-to-end.
+
+### Quick Validation
+
+```bash
+# Run the full validation suite (starts cluster, generates load, validates):
+docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld
+
+# Check the report:
+cat /tmp/xrpld-validation/reports/validation-report.json | jq '.summary'
+```
+
+### What Gets Validated
+
+| Category   | Checks         | Description                                             |
+| ---------- | -------------- | ------------------------------------------------------- |
+| Spans      | 16+ span types | All span names appear in Tempo with required attributes |
+| Metrics    | 30+ metrics    | SpanMetrics, StatsD gauges/counters, Phase 9 metrics    |
+| Logs       | 2 checks       | trace_id/span_id present in Loki, cross-reference works |
+| Dashboards | 10 dashboards  | All Grafana dashboards load without errors              |
+
+### Running Individual Tools
+
+```bash
+# RPC load only:
+python3 docker/telemetry/workload/rpc_load_generator.py \
+    --endpoints ws://localhost:6006 --rate 50 --duration 120
+
+# Transaction mix only:
+python3 docker/telemetry/workload/tx_submitter.py \
+    --endpoint ws://localhost:6006 --tps 5 --duration 120
+
+# Validation only (assumes load already ran):
+python3 docker/telemetry/workload/validate_telemetry.py \
+    --report /tmp/report.json
+```
+
+### Interpreting Failures
+
+- **Span failures**: Check that the relevant trace category is enabled in `[telemetry]` config (e.g., `trace_rpc=1`).
+- **Metric failures**: Verify the OTel Collector is running and Prometheus is scraping port 8889. Check `docker compose logs otel-collector`.
+- **Dashboard failures**: Ensure Grafana provisioning is mounted correctly. Check `docker compose logs grafana`.
+
+## Performance Benchmarking
+
+Measure the overhead of the telemetry stack against a baseline:
+
+```bash
+docker/telemetry/workload/benchmark.sh --xrpld .build/xrpld --duration 300
+```
+
+### Benchmark Thresholds
+
+| Metric            | Target | Description                            |
+| ----------------- | ------ | -------------------------------------- |
+| CPU overhead      | < 3%   | Average CPU increase across nodes      |
+| Memory overhead   | < 5MB  | Peak RSS increase per node             |
+| RPC p99 latency   | < 2ms  | Additional p99 latency for server_info |
+| Throughput impact | < 5%   | Reduction in ledger close rate         |
+| Consensus impact  | < 1%   | Increase in consensus round time       |
+
+### Tuning for Production
+
+If benchmarks exceed thresholds:
+
+1. **Reduce sampling**: `sampling_ratio=0.01` (1% of traces)
+2. **Disable peer tracing**: `trace_peer=0` (highest volume category)
+3. **Increase batch delay**: `batch_delay_ms=10000` (less frequent exports)
+4. **Reduce queue size**: `max_queue_size=1024` (back-pressure earlier)
+
+See `docker/telemetry/workload/README.md` for full documentation.

include/xrpl/beast/insight/Insight.h

@@ -12,4 +12,5 @@
 #include <xrpl/beast/insight/Hook.h>
 #include <xrpl/beast/insight/HookImpl.h>
 #include <xrpl/beast/insight/NullCollector.h>
+#include <xrpl/beast/insight/OTelCollector.h>
 #include <xrpl/beast/insight/StatsDCollector.h>

include/xrpl/beast/insight/OTelCollector.h

@@ -0,0 +1,91 @@
+#pragma once
+
+/**
+ * @file OTelCollector.h
+ * @brief OpenTelemetry-based implementation of the beast::insight::Collector
+ *        interface for native OTLP metric export.
+ *
+ * When XRPL_ENABLE_TELEMETRY is defined, OTelCollector maps each
+ * beast::insight instrument type (Counter, Gauge, Event, Meter, Hook) to
+ * the corresponding OpenTelemetry Metrics SDK instrument and exports
+ * them via OTLP/HTTP to an OpenTelemetry Collector.
+ *
+ * When XRPL_ENABLE_TELEMETRY is NOT defined, OTelCollector::New() returns
+ * a NullCollector so the binary compiles without OTel dependencies.
+ *
+ * Dependency diagram:
+ *
+ *   +-----------------+     +-------------------+
+ *   | Collector (ABC) |<----| OTelCollector     |
+ *   +-----------------+     | (public header)   |
+ *         ^                 +-------------------+
+ *         |                          |
+ *   +-----------------+     +-------------------+
+ *   | NullCollector   |     | OTelCollectorImp  |
+ *   | (fallback when  |     | (impl in .cpp,    |
+ *   |  no telemetry)  |     |  uses OTel SDK)   |
+ *   +-----------------+     +-------------------+
+ *                                    |
+ *                           +-------------------+
+ *                           | OTel Metrics SDK  |
+ *                           | MeterProvider     |
+ *                           | OTLP HTTP Metric  |
+ *                           | Exporter          |
+ *                           +-------------------+
+ */
+
+#include <xrpl/beast/insight/Collector.h>
+#include <xrpl/beast/utility/Journal.h>
+
+#include <memory>
+#include <string>
+
+namespace beast::insight {
+
+/**
+ * @brief A Collector that exports metrics via OpenTelemetry OTLP/HTTP.
+ *
+ * Replaces StatsD-based metric collection with native OTel Metrics SDK
+ * instruments. Each beast::insight instrument maps to an OTel equivalent:
+ *
+ *   - Counter  -> OTel Counter<uint64_t>
+ *   - Gauge    -> OTel ObservableGauge<int64_t> (async callback)
+ *   - Event    -> OTel Histogram<double> (duration in milliseconds)
+ *   - Meter    -> OTel Counter<uint64_t> (monotonic, unsigned)
+ *   - Hook     -> Called by PeriodicMetricReader at collection time
+ *
+ * @see StatsDCollector for the StatsD-based alternative.
+ * @see NullCollector for the no-op fallback.
+ */
+class OTelCollector : public Collector
+{
+public:
+    explicit OTelCollector() = default;
+
+    /**
+     * @brief Factory method to create an OTelCollector instance.
+     *
+     * When XRPL_ENABLE_TELEMETRY is defined, creates a real OTel-backed
+     * collector that exports metrics via OTLP/HTTP. When telemetry is
+     * disabled at compile time, returns a NullCollector.
+     *
+     * @param endpoint    OTLP/HTTP metrics endpoint URL
+     *                    (e.g. "http://localhost:4318/v1/metrics").
+     * @param prefix      Prefix prepended to all metric names
+     *                    (e.g. "xrpld").
+     * @param instanceId  Unique identifier for this node instance,
+     *                    emitted as the `service.instance.id` OTel
+     *                    resource attribute. Defaults to empty string
+     *                    (attribute omitted when empty).
+     * @param journal     Journal for logging.
+     * @return Shared pointer to the created Collector.
+     */
+    static std::shared_ptr<Collector>
+    // NOLINTNEXTLINE(readability-identifier-naming)
+    New(std::string const& endpoint,
+        std::string const& prefix,
+        std::string const& instanceId,
+        Journal journal);
+};
+
+}  // namespace beast::insight

include/xrpl/core/ServiceRegistry.h

@@ -20,6 +20,7 @@ class PerfLog;
 }  // namespace perf
 namespace telemetry {
 class Telemetry;
+class MetricsRegistry;
 }  // namespace telemetry
 
 // This is temporary until we migrate all code to use ServiceRegistry.
@@ -224,6 +225,12 @@ public:
     virtual telemetry::Telemetry&
     getTelemetry() = 0;
 
+    /** Return the MetricsRegistry, or nullptr if telemetry is disabled.
+        Used by PerfLog and other hot paths to record OTel metrics.
+    */
+    virtual telemetry::MetricsRegistry*
+    getMetricsRegistry() = 0;
+
     // Configuration and state
     [[nodiscard]] virtual bool
     isStopping() const = 0;

include/xrpl/proto/xrpl.proto

@@ -102,6 +102,10 @@ message TraceContext {
   optional bytes trace_id = 1;      // 16-byte trace identifier
   optional bytes span_id = 2;       // 8-byte parent span identifier
   optional uint32 trace_flags = 3;  // bit 0 = sampled
+  // TODO: trace_state is reserved for W3C tracestate vendor-specific
+  // key-value pairs but is not yet read or written by
+  // TraceContextPropagator. Wire it when cross-vendor trace
+  // propagation is needed.
   optional string trace_state = 4;  // RESERVED — see TraceContext header note
 }
 

include/xrpl/telemetry/SpanNames.h

@@ -125,6 +125,7 @@ inline constexpr auto ledgerSeq = makeStr("ledger_seq");
 inline constexpr auto closeTime = makeStr("close_time");
 inline constexpr auto closeTimeCorrect = makeStr("close_time_correct");
 inline constexpr auto closeResolutionMs = makeStr("close_resolution_ms");
+inline constexpr auto ledgerHash = join(join(seg::xrpl, seg::ledger), makeStr("hash"));
 }  // namespace attr
 
 // ===== Shared attribute values =============================================

src/libxrpl/basics/Log.cpp

@@ -7,6 +7,15 @@
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/filesystem/path.hpp>
 
+#ifdef XRPL_ENABLE_TELEMETRY
+#include <opentelemetry/context/runtime_context.h>
+#include <opentelemetry/nostd/shared_ptr.h>
+#include <opentelemetry/nostd/span.h>
+#include <opentelemetry/nostd/variant.h>
+#include <opentelemetry/trace/span.h>
+#include <opentelemetry/trace/span_metadata.h>
+#endif  // XRPL_ENABLE_TELEMETRY
+
 #include <chrono>
 #include <cstring>
 #include <fstream>
@@ -290,6 +299,34 @@ Logs::format(
             break;
     }
 
+#ifdef XRPL_ENABLE_TELEMETRY
+    // Inject OTel trace context when an active span exists on this thread.
+    // Checks the thread-local context value directly to avoid the heap
+    // allocation that GetSpan() performs on the no-span path.
+    {
+        auto context = opentelemetry::context::RuntimeContext::GetCurrent();
+        auto spanValue = context.GetValue(opentelemetry::trace::kSpanKey);
+        if (opentelemetry::nostd::holds_alternative<
+                opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>>(spanValue))
+        {
+            auto span = opentelemetry::nostd::get<
+                opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>>(spanValue);
+            auto spanCtx = span->GetContext();
+            if (spanCtx.IsValid())
+            {
+                char traceId[32], spanId[16];
+                spanCtx.trace_id().ToLowerBase16(opentelemetry::nostd::span<char, 32>{traceId});
+                spanCtx.span_id().ToLowerBase16(opentelemetry::nostd::span<char, 16>{spanId});
+                output += "trace_id=";
+                output.append(traceId, 32);
+                output += " span_id=";
+                output.append(spanId, 16);
+                output += ' ';
+            }
+        }
+    }
+#endif  // XRPL_ENABLE_TELEMETRY
+
     output += message;
 
     // Limit the maximum length of the output

src/libxrpl/beast/insight/OTelCollector.cpp

@@ -0,0 +1,892 @@
+/**
+ * @file OTelCollector.cpp
+ * @brief OpenTelemetry Metrics SDK implementation of beast::insight::Collector.
+ *
+ * Compiled only when XRPL_ENABLE_TELEMETRY is defined (via CMake
+ * telemetry=ON). Maps beast::insight instruments to OTel SDK instruments
+ * and exports them via OTLP/HTTP using a PeriodicMetricReader.
+ *
+ * When XRPL_ENABLE_TELEMETRY is not defined, OTelCollector::New() returns
+ * a NullCollector so the build succeeds without OTel dependencies.
+ *
+ * Data flow:
+ *
+ *   beast::insight callers
+ *       |
+ *       v
+ *   OTelCounterImpl / OTelGaugeImpl / OTelEventImpl / OTelMeterImpl
+ *       |                    |                |              |
+ *       v                    v                v              v
+ *   Counter<uint64_t>  ObservableGauge  Histogram<double>  Counter<uint64_t>
+ *       |                    |                |              |
+ *       +--------------------+----------------+--------------+
+ *       |
+ *       v
+ *   PeriodicMetricReader (1s interval)
+ *       |
+ *       v
+ *   OtlpHttpMetricExporter -> OTel Collector -> Prometheus
+ */
+
+#ifdef XRPL_ENABLE_TELEMETRY
+#include <xrpl/beast/insight/OTelCollector.h>
+
+#include <xrpl/beast/insight/Collector.h>
+#include <xrpl/beast/insight/CounterImpl.h>
+#include <xrpl/beast/insight/EventImpl.h>
+#include <xrpl/beast/insight/GaugeImpl.h>
+#include <xrpl/beast/insight/Hook.h>
+#include <xrpl/beast/insight/HookImpl.h>
+#include <xrpl/beast/insight/MeterImpl.h>
+#include <xrpl/beast/utility/Journal.h>
+
+#include <opentelemetry/exporters/otlp/otlp_http_metric_exporter_factory.h>
+#include <opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h>
+#include <opentelemetry/metrics/async_instruments.h>
+#include <opentelemetry/metrics/meter.h>
+#include <opentelemetry/metrics/meter_provider.h>
+#include <opentelemetry/metrics/observer_result.h>
+#include <opentelemetry/metrics/sync_instruments.h>
+#include <opentelemetry/nostd/shared_ptr.h>
+#include <opentelemetry/nostd/unique_ptr.h>
+#include <opentelemetry/nostd/variant.h>
+#include <opentelemetry/sdk/metrics/aggregation/aggregation_config.h>
+#include <opentelemetry/sdk/metrics/export/periodic_exporting_metric_reader_factory.h>
+#include <opentelemetry/sdk/metrics/export/periodic_exporting_metric_reader_options.h>
+#include <opentelemetry/sdk/metrics/instruments.h>
+#include <opentelemetry/sdk/metrics/meter_provider.h>
+#include <opentelemetry/sdk/metrics/meter_provider_factory.h>
+#include <opentelemetry/sdk/metrics/view/instrument_selector_factory.h>
+#include <opentelemetry/sdk/metrics/view/meter_selector_factory.h>
+#include <opentelemetry/sdk/metrics/view/view_factory.h>
+#include <opentelemetry/sdk/metrics/view/view_registry.h>
+#include <opentelemetry/sdk/resource/resource.h>
+#include <opentelemetry/semconv/incubating/service_attributes.h>
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace beast::insight {
+
+namespace detail {
+
+namespace metrics_api = opentelemetry::metrics;
+namespace metrics_sdk = opentelemetry::sdk::metrics;
+namespace otlp_http = opentelemetry::exporter::otlp;
+namespace resource = opentelemetry::sdk::resource;
+
+class OTelCollectorImp;
+
+//------------------------------------------------------------------------------
+
+/**
+ * @brief OTel-backed implementation of beast::insight::HookImpl.
+ *
+ * Stores a handler function that is invoked during each periodic
+ * metric collection cycle. This mirrors the StatsDHookImpl pattern
+ * where hooks are called at each 1-second timer tick, but here the
+ * invocation is triggered by the OTel PeriodicMetricReader's
+ * observable callback mechanism.
+ */
+class OTelHookImpl : public HookImpl
+{
+public:
+    /**
+     * @param handler  Callback invoked at each collection interval.
+     * @param impl     Owning collector (prevents premature destruction).
+     */
+    OTelHookImpl(HandlerType handler, std::shared_ptr<OTelCollectorImp> impl);
+
+    ~OTelHookImpl() override;
+
+    OTelHookImpl&
+    operator=(OTelHookImpl const&) = delete;
+
+    /**
+     * @brief Invoke the stored handler.
+     *
+     * Called by the collector during observable gauge callbacks to give
+     * metric producers a chance to update gauge values before export.
+     */
+    void
+    callHandler();
+
+private:
+    /** Owning collector. Prevents collector destruction while hook alive. */
+    std::shared_ptr<OTelCollectorImp> impl_;
+
+    /** User-supplied handler called at each collection interval. */
+    HandlerType handler_;
+};
+
+//------------------------------------------------------------------------------
+
+/**
+ * @brief OTel-backed implementation of beast::insight::CounterImpl.
+ *
+ * Wraps an OTel Counter<uint64_t> instrument. Each increment() call
+ * is forwarded directly to the OTel counter's Add() method. The
+ * PeriodicMetricReader collects and exports the accumulated delta.
+ *
+ * Thread safety: OTel Counter::Add() is thread-safe by specification.
+ */
+class OTelCounterImpl : public CounterImpl
+{
+public:
+    /**
+     * @param name   Fully-qualified metric name (prefix.group.name).
+     * @param meter  OTel Meter used to create the counter instrument.
+     */
+    OTelCounterImpl(
+        std::string const& name,
+        opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter);
+
+    ~OTelCounterImpl() override = default;
+
+    OTelCounterImpl&
+    operator=(OTelCounterImpl const&) = delete;
+
+    /**
+     * @brief Add amount to the counter.
+     * @param amount  Value to add (must be non-negative for OTel counters).
+     */
+    void
+    increment(value_type amount) override;
+
+private:
+    /** OTel synchronous counter instrument. */
+    opentelemetry::nostd::unique_ptr<metrics_api::Counter<uint64_t>> counter_;
+};
+
+//------------------------------------------------------------------------------
+
+/**
+ * @brief OTel-backed implementation of beast::insight::EventImpl.
+ *
+ * Wraps an OTel Histogram<double> instrument. Each notify() call
+ * records the duration in milliseconds. Uses explicit bucket boundaries
+ * matching the SpanMetrics connector configuration:
+ *   [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms
+ *
+ * Thread safety: OTel Histogram::Record() is thread-safe by specification.
+ */
+class OTelEventImpl : public EventImpl
+{
+public:
+    /**
+     * @param name   Fully-qualified metric name (prefix.group.name).
+     * @param meter  OTel Meter used to create the histogram instrument.
+     */
+    OTelEventImpl(
+        std::string const& name,
+        opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter);
+
+    ~OTelEventImpl() override = default;
+
+    OTelEventImpl&
+    operator=(OTelEventImpl const&) = delete;
+
+    /**
+     * @brief Record a duration measurement.
+     * @param value  Duration in milliseconds.
+     */
+    void
+    notify(value_type const& value) override;
+
+private:
+    /** OTel histogram instrument for recording durations. */
+    opentelemetry::nostd::unique_ptr<metrics_api::Histogram<double>> histogram_;
+};
+
+//------------------------------------------------------------------------------
+
+/**
+ * @brief OTel-backed implementation of beast::insight::GaugeImpl.
+ *
+ * Uses an atomic int64_t to store the current gauge value. The OTel SDK
+ * reads this value via an ObservableGauge async callback during each
+ * collection cycle. The set() and increment() methods update the
+ * atomic value without blocking the collection thread.
+ *
+ * Design note: OTel gauges are asynchronous (observable) instruments.
+ * The SDK calls a registered callback to read the value rather than
+ * accepting push-style updates. We bridge the beast::insight push-style
+ * API to OTel's pull-style API via the atomic variable.
+ *
+ * Thread safety: std::atomic operations are lock-free on all platforms.
+ */
+class OTelGaugeImpl : public GaugeImpl
+{
+public:
+    /**
+     * @param name       Fully-qualified metric name (prefix.group.name).
+     * @param meter      OTel Meter used to create the observable gauge.
+     * @param collector  Owning collector, used to invoke hooks before reads.
+     */
+    OTelGaugeImpl(
+        std::string const& name,
+        opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter,
+        std::shared_ptr<OTelCollectorImp> const& collector);
+
+    ~OTelGaugeImpl() override;
+
+    /**
+     * @brief Set the gauge to an absolute value.
+     * @param value  New gauge value.
+     */
+    void
+    set(value_type value) override;
+
+    /**
+     * @brief Increment (or decrement) the gauge by a signed amount.
+     *
+     * Clamps the result to [0, INT64_MAX] to match StatsDGaugeImpl
+     * behavior.
+     *
+     * @param amount  Signed amount to add to the current value.
+     */
+    void
+    increment(difference_type amount) override;
+
+    /**
+     * @brief Return the current gauge value for the OTel callback.
+     * @return The most recently set/incremented value.
+     */
+    int64_t
+    currentValue() const;
+
+    OTelGaugeImpl&
+    operator=(OTelGaugeImpl const&) = delete;
+
+    /** Static callback registered with the OTel SDK observable gauge. */
+    static void
+    gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state);
+
+private:
+    /** Current gauge value, updated atomically by set()/increment(). */
+    std::atomic<int64_t> value_{0};
+
+    /** OTel observable gauge handle (prevents deregistration). */
+    opentelemetry::nostd::shared_ptr<metrics_api::ObservableInstrument> gauge_;
+
+    /** Owning collector, used to invoke hooks before reading gauge values. */
+    std::shared_ptr<OTelCollectorImp> collector_;
+};
+
+//------------------------------------------------------------------------------
+
+/**
+ * @brief OTel-backed implementation of beast::insight::MeterImpl.
+ *
+ * Wraps an OTel Counter<uint64_t> instrument. Semantically identical
+ * to Counter but uses unsigned values. The OTel SDK accumulates deltas
+ * and exports them via the PeriodicMetricReader.
+ *
+ * Note: In StatsD, Meter used the non-standard "|m" type which was
+ * silently dropped by the OTel StatsD receiver. With native OTel,
+ * Meter values are properly captured as counter deltas.
+ *
+ * Thread safety: OTel Counter::Add() is thread-safe by specification.
+ */
+class OTelMeterImpl : public MeterImpl
+{
+public:
+    /**
+     * @param name   Fully-qualified metric name (prefix.group.name).
+     * @param meter  OTel Meter used to create the counter instrument.
+     */
+    OTelMeterImpl(
+        std::string const& name,
+        opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter);
+
+    ~OTelMeterImpl() override = default;
+
+    OTelMeterImpl&
+    operator=(OTelMeterImpl const&) = delete;
+
+    /**
+     * @brief Add amount to the meter.
+     * @param amount  Value to add (unsigned).
+     */
+    void
+    increment(value_type amount) override;
+
+private:
+    /** OTel synchronous counter instrument (unsigned). */
+    opentelemetry::nostd::unique_ptr<metrics_api::Counter<uint64_t>> counter_;
+};
+
+//------------------------------------------------------------------------------
+
+/**
+ * @brief Main OTel Collector implementation.
+ *
+ * Creates an OTel MeterProvider with a PeriodicMetricReader that
+ * exports metrics via OTLP/HTTP at 1-second intervals. Implements
+ * all Collector::make_*() factory methods to create OTel-backed
+ * instrument wrappers.
+ *
+ * Class diagram:
+ *
+ *   +------------------+      +------------------+
+ *   | Collector (ABC)  |<-----| OTelCollector    |
+ *   +------------------+      | (public header)  |
+ *          ^                  +------------------+
+ *          |                          ^
+ *   +------------------+             |
+ *   | OTelCollectorImp |-------------+
+ *   +------------------+
+ *   | - journal_       |
+ *   | - prefix_        |
+ *   | - provider_      |     +---------------------+
+ *   | - otelMeter_     |---->| OTel MeterProvider   |
+ *   | - hooks_[]       |     | + PeriodicReader     |
+ *   | - gauges_[]      |     | + OtlpHttpExporter   |
+ *   +------------------+     +---------------------+
+ *
+ * Lifecycle:
+ *   1. Constructor creates MeterProvider + exporter pipeline.
+ *   2. make_*() methods create instruments registered with the provider.
+ *   3. PeriodicMetricReader collects every 1s, calling observable callbacks.
+ *   4. Observable callbacks invoke hooks, read gauge atomics.
+ *   5. Destructor shuts down MeterProvider (flushes pending exports).
+ *
+ * Caveats:
+ *   - Observable gauge callbacks run on the SDK's internal thread. Hook
+ *     handlers must be thread-safe.
+ *   - Metric names are formed as "prefix_name" with dots replaced by
+ *     underscores to match StatsD->Prometheus naming conventions.
+ *   - The OTel Prometheus exporter appends "_total" to counters. The
+ *     metric names we register do NOT include this suffix — Prometheus
+ *     adds it automatically.
+ *
+ * Example usage:
+ * @code
+ *   auto collector = OTelCollector::New(
+ *       "http://localhost:4318/v1/metrics", "xrpld", journal);
+ *   auto counter = collector->makeCounter("rpc.requests");
+ *   counter.increment(1);
+ *   // Metric "xrpld_rpc_requests" exported via OTLP every 1s.
+ * @endcode
+ */
+class OTelCollectorImp : public OTelCollector, public std::enable_shared_from_this<OTelCollectorImp>
+{
+public:
+    /**
+     * @brief Construct the OTel collector and initialize the export pipeline.
+     *
+     * @param endpoint    OTLP/HTTP metrics endpoint URL.
+     * @param prefix      Prefix for all metric names.
+     * @param instanceId  Value for the service.instance.id resource attribute.
+     *                    When empty, the attribute is omitted.
+     * @param journal     Journal for logging.
+     */
+    OTelCollectorImp(
+        std::string const& endpoint,
+        std::string prefix,
+        std::string const& instanceId,
+        Journal journal);
+
+    /**
+     * @brief Shut down the MeterProvider, flushing any pending exports.
+     */
+    ~OTelCollectorImp() override;
+
+    /** @name Collector interface implementation */
+    /** @{ */
+    Hook
+    makeHook(HookImpl::HandlerType const& handler) override;
+
+    Counter
+    makeCounter(std::string const& name) override;
+
+    Event
+    makeEvent(std::string const& name) override;
+
+    Gauge
+    makeGauge(std::string const& name) override;
+
+    Meter
+    makeMeter(std::string const& name) override;
+    /** @} */
+
+    /** @name Hook management for observable callbacks */
+    /** @{ */
+
+    /**
+     * @brief Register a hook for periodic invocation.
+     * @param hook  Pointer to the hook to register.
+     */
+    void
+    addHook(OTelHookImpl* hook);
+
+    /**
+     * @brief Unregister a hook.
+     * @param hook  Pointer to the hook to unregister.
+     */
+    void
+    removeHook(OTelHookImpl* hook);
+
+    /**
+     * @brief Invoke all registered hooks.
+     *
+     * Called from observable gauge callbacks before reading gauge values,
+     * so that hook handlers have a chance to update metrics.
+     */
+    void
+    callHooks();
+    /** @} */
+
+    /** @name Gauge registration for observable callbacks */
+    /** @{ */
+
+    /**
+     * @brief Register a gauge for observable callback reading.
+     * @param gauge  Pointer to the gauge to register.
+     */
+    void
+    addGauge(OTelGaugeImpl* gauge);
+
+    /**
+     * @brief Unregister a gauge.
+     * @param gauge  Pointer to the gauge to unregister.
+     */
+    void
+    removeGauge(OTelGaugeImpl* gauge);
+    /** @} */
+
+    /**
+     * @brief Get the OTel Meter instance for creating instruments.
+     * @return Shared pointer to the OTel Meter.
+     */
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter> const&
+    otelMeter() const;
+
+    /**
+     * @brief Format a metric name with the configured prefix.
+     *
+     * Replaces dots with underscores to match StatsD->Prometheus naming.
+     * Example: prefix="xrpld", name="LedgerMaster.Validated_Ledger_Age"
+     *   -> "xrpld_LedgerMaster_Validated_Ledger_Age"
+     *
+     * @param name  Raw metric name from beast::insight callers.
+     * @return Fully-qualified metric name.
+     */
+    std::string
+    formatName(std::string const& name) const;
+
+private:
+    /** Journal for log output. */
+    Journal journal_;
+
+    /** Prefix for all metric names (e.g., "xrpld"). */
+    std::string prefix_;
+
+    /** OTel SDK MeterProvider owning the export pipeline. RAII lifecycle. */
+    std::shared_ptr<metrics_sdk::MeterProvider> provider_;
+
+    /** OTel Meter used to create all instruments. */
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter> otelMeter_;
+
+    /** Mutex protecting hook and gauge registration lists. */
+    std::mutex mutex_;
+
+    /** Registered hooks called during observable callbacks. */
+    std::vector<OTelHookImpl*> hooks_;
+
+    /** Registered gauges read during observable callbacks. */
+    std::vector<OTelGaugeImpl*> gauges_;
+
+    /**
+     * @brief Debounce timestamp for callHooks().
+     *
+     * Multiple gauge callbacks fire during the same collection cycle.
+     * This atomic tracks the last time hooks were invoked (ms since epoch).
+     * Hooks are called at most once per 500ms window to avoid redundant
+     * invocations while still ensuring fresh values each collection cycle.
+     */
+    std::atomic<int64_t> lastHookCallMs_{0};
+};
+
+//==============================================================================
+// Implementation
+//==============================================================================
+
+//------------------------------------------------------------------------------
+// OTelHookImpl
+//------------------------------------------------------------------------------
+
+OTelHookImpl::OTelHookImpl(HandlerType handler, std::shared_ptr<OTelCollectorImp> impl)
+    : impl_(std::move(impl)), handler_(std::move(handler))
+{
+    impl_->addHook(this);
+}
+
+OTelHookImpl::~OTelHookImpl()
+{
+    impl_->removeHook(this);
+}
+
+void
+OTelHookImpl::callHandler()
+{
+    handler_();
+}
+
+//------------------------------------------------------------------------------
+// OTelCounterImpl
+//------------------------------------------------------------------------------
+
+OTelCounterImpl::OTelCounterImpl(
+    std::string const& name,
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter)
+    : counter_(meter->CreateUInt64Counter(name))
+{
+}
+
+void
+OTelCounterImpl::increment(value_type amount)
+{
+    // OTel counters require non-negative values. beast::insight CounterImpl
+    // uses int64_t, so clamp negative values to 0 and cast to uint64_t.
+    if (amount > 0)
+        counter_->Add(static_cast<uint64_t>(amount));
+}
+
+//------------------------------------------------------------------------------
+// OTelEventImpl
+//------------------------------------------------------------------------------
+
+OTelEventImpl::OTelEventImpl(
+    std::string const& name,
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter)
+    : histogram_(meter->CreateDoubleHistogram(name, "Duration in ms", "ms"))
+{
+}
+
+void
+OTelEventImpl::notify(value_type const& value)
+{
+    histogram_->Record(static_cast<double>(value.count()), opentelemetry::context::Context{});
+}
+
+//------------------------------------------------------------------------------
+// OTelGaugeImpl
+//------------------------------------------------------------------------------
+
+OTelGaugeImpl::OTelGaugeImpl(
+    std::string const& name,
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter,
+    std::shared_ptr<OTelCollectorImp> const& collector)
+    : gauge_(meter->CreateInt64ObservableGauge(name)), collector_(collector)
+{
+    collector_->addGauge(this);
+    gauge_->AddCallback(gaugeCallback, this);
+}
+
+void
+OTelGaugeImpl::gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state)
+{
+    auto* self = static_cast<OTelGaugeImpl*>(state);
+    self->collector_->callHooks();
+    if (auto intResult = opentelemetry::nostd::get_if<
+            opentelemetry::nostd::shared_ptr<opentelemetry::metrics::ObserverResultT<int64_t>>>(
+            &result))
+    {
+        (*intResult)->Observe(self->currentValue());
+    }
+}
+
+OTelGaugeImpl::~OTelGaugeImpl()
+{
+    gauge_->RemoveCallback(gaugeCallback, this);
+    collector_->removeGauge(this);
+}
+
+void
+OTelGaugeImpl::set(value_type value)
+{
+    value_.store(static_cast<int64_t>(value), std::memory_order_relaxed);
+}
+
+void
+OTelGaugeImpl::increment(difference_type amount)
+{
+    // Use compare-exchange loop to safely clamp to [0, MAX].
+    int64_t current = value_.load(std::memory_order_relaxed);
+    int64_t desired = 0;
+    do
+    {
+        desired = current + amount;
+        // Clamp to 0 on underflow.
+        desired = std::max(desired, int64_t{0});
+    } while (!value_.compare_exchange_weak(current, desired, std::memory_order_relaxed));
+}
+
+int64_t
+OTelGaugeImpl::currentValue() const
+{
+    return value_.load(std::memory_order_relaxed);
+}
+
+//------------------------------------------------------------------------------
+// OTelMeterImpl
+//------------------------------------------------------------------------------
+
+OTelMeterImpl::OTelMeterImpl(
+    std::string const& name,
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter)
+    : counter_(meter->CreateUInt64Counter(name))
+{
+}
+
+void
+OTelMeterImpl::increment(value_type amount)
+{
+    counter_->Add(amount);
+}
+
+//------------------------------------------------------------------------------
+// OTelCollectorImp
+//------------------------------------------------------------------------------
+
+OTelCollectorImp::OTelCollectorImp(
+    std::string const& endpoint,
+    std::string prefix,
+    std::string const& instanceId,
+    Journal journal)
+    : journal_(journal), prefix_(std::move(prefix))
+{
+    if (journal_.info())
+    {
+        journal_.info() << "OTelCollector starting: endpoint=" << endpoint << " prefix=" << prefix_;
+    }
+
+    // Configure OTLP HTTP metric exporter.
+    otlp_http::OtlpHttpMetricExporterOptions exporterOpts;
+    exporterOpts.url = endpoint;
+
+    auto exporter = otlp_http::OtlpHttpMetricExporterFactory::Create(exporterOpts);
+
+    // Configure periodic metric reader (1-second export interval).
+    metrics_sdk::PeriodicExportingMetricReaderOptions readerOpts;
+    readerOpts.export_interval_millis = std::chrono::milliseconds(1000);
+    readerOpts.export_timeout_millis = std::chrono::milliseconds(500);
+
+    auto reader =
+        metrics_sdk::PeriodicExportingMetricReaderFactory::Create(std::move(exporter), readerOpts);
+
+    // Configure resource attributes matching the trace exporter.
+    // Include service.instance.id when provided so Prometheus
+    // exported_instance labels distinguish multi-node deployments.
+    resource::ResourceAttributes attrs;
+    attrs[opentelemetry::semconv::service::kServiceName] = "xrpld";
+    if (!instanceId.empty())
+    {
+        attrs[opentelemetry::semconv::service::kServiceInstanceId] = instanceId;
+    }
+    auto resourceAttrs = resource::Resource::Create(attrs);
+
+    // Create MeterProvider with resource, then attach the metric reader.
+    provider_ = metrics_sdk::MeterProviderFactory::Create(
+        std::make_unique<metrics_sdk::ViewRegistry>(), resourceAttrs);
+    provider_->AddMetricReader(std::move(reader));
+
+    // Configure histogram bucket boundaries for Event instruments.
+    // These match the SpanMetrics connector buckets for consistency.
+    auto histogramSelector = metrics_sdk::InstrumentSelectorFactory::Create(
+        metrics_sdk::InstrumentType::kHistogram, "*", "ms");
+    auto meterSelector = metrics_sdk::MeterSelectorFactory::Create("xrpld_metrics", "", "");
+    auto histogramConfig = std::make_shared<metrics_sdk::HistogramAggregationConfig>();
+    histogramConfig->boundaries_ =
+        std::vector<double>{1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 5000.0};
+    auto histogramView = metrics_sdk::ViewFactory::Create(
+        "default_histogram",
+        "Default histogram view with SpanMetrics-compatible buckets",
+        metrics_sdk::AggregationType::kHistogram,
+        std::move(histogramConfig));
+
+    provider_->AddView(
+        std::move(histogramSelector), std::move(meterSelector), std::move(histogramView));
+
+    // Create the OTel Meter for creating instruments.
+    otelMeter_ = provider_->GetMeter("xrpld_metrics", "1.0.0");
+
+    if (journal_.info())
+    {
+        journal_.info() << "OTelCollector started successfully";
+    }
+}
+
+OTelCollectorImp::~OTelCollectorImp()
+{
+    if (journal_.info())
+    {
+        journal_.info() << "OTelCollector shutting down";
+    }
+    if (provider_)
+    {
+        provider_->ForceFlush(std::chrono::milliseconds(2000));
+        provider_->Shutdown();
+    }
+    if (journal_.info())
+    {
+        journal_.info() << "OTelCollector stopped";
+    }
+}
+
+Hook
+OTelCollectorImp::makeHook(HookImpl::HandlerType const& handler)
+{
+    return Hook(std::make_shared<OTelHookImpl>(handler, shared_from_this()));
+}
+
+Counter
+OTelCollectorImp::makeCounter(std::string const& name)
+{
+    return Counter(std::make_shared<OTelCounterImpl>(formatName(name), otelMeter_));
+}
+
+Event
+OTelCollectorImp::makeEvent(std::string const& name)
+{
+    return Event(std::make_shared<OTelEventImpl>(formatName(name), otelMeter_));
+}
+
+Gauge
+OTelCollectorImp::makeGauge(std::string const& name)
+{
+    return Gauge(std::make_shared<OTelGaugeImpl>(formatName(name), otelMeter_, shared_from_this()));
+}
+
+Meter
+OTelCollectorImp::makeMeter(std::string const& name)
+{
+    return Meter(std::make_shared<OTelMeterImpl>(formatName(name), otelMeter_));
+}
+
+void
+OTelCollectorImp::addHook(OTelHookImpl* hook)
+{
+    std::scoped_lock const lock(mutex_);
+    hooks_.push_back(hook);
+}
+
+void
+OTelCollectorImp::removeHook(OTelHookImpl* hook)
+{
+    std::scoped_lock const lock(mutex_);
+    std::erase(hooks_, hook);
+}
+
+void
+OTelCollectorImp::callHooks()
+{
+    // Debounce: hooks run at most once per 500ms. Multiple gauge callbacks
+    // fire during the same collection cycle — only the first one triggers
+    // hooks. Subsequent callbacks within the window read already-updated
+    // gauge values.
+    auto now = std::chrono::duration_cast<std::chrono::milliseconds>(
+                   std::chrono::steady_clock::now().time_since_epoch())
+                   .count();
+    auto last = lastHookCallMs_.load(std::memory_order_acquire);
+    if (now - last < 500)
+        return;
+    if (!lastHookCallMs_.compare_exchange_strong(last, now, std::memory_order_acq_rel))
+        return;  // Another thread won the race.
+
+    std::scoped_lock const lock(mutex_);
+    for (auto* hook : hooks_)
+        hook->callHandler();
+}
+
+void
+OTelCollectorImp::addGauge(OTelGaugeImpl* gauge)
+{
+    std::scoped_lock const lock(mutex_);
+    gauges_.push_back(gauge);
+}
+
+void
+OTelCollectorImp::removeGauge(OTelGaugeImpl* gauge)
+{
+    std::scoped_lock const lock(mutex_);
+    std::erase(gauges_, gauge);
+}
+
+opentelemetry::nostd::shared_ptr<metrics_api::Meter> const&
+OTelCollectorImp::otelMeter() const
+{
+    return otelMeter_;
+}
+
+std::string
+OTelCollectorImp::formatName(std::string const& name) const
+{
+    // StatsD uses "prefix.group.name" format. The OTel StatsD receiver
+    // converts dots to underscores for Prometheus. We replicate this
+    // to preserve metric name compatibility.
+    //
+    // Example: prefix="xrpld", name="LedgerMaster.Validated_Ledger_Age"
+    //   -> "xrpld_LedgerMaster_Validated_Ledger_Age"
+    std::string result;
+    if (!prefix_.empty())
+    {
+        result = prefix_;
+        result += '_';
+    }
+    for (char const c : name)
+    {
+        result += (c == '.') ? '_' : c;
+    }
+    return result;
+}
+
+}  // namespace detail
+
+//------------------------------------------------------------------------------
+
+std::shared_ptr<Collector>
+OTelCollector::New(
+    std::string const& endpoint,
+    std::string const& prefix,
+    std::string const& instanceId,
+    Journal journal)
+{
+    return std::make_shared<detail::OTelCollectorImp>(endpoint, prefix, instanceId, journal);
+}
+
+}  // namespace beast::insight
+
+#else  // !XRPL_ENABLE_TELEMETRY
+
+// When telemetry is disabled at compile time, OTelCollector::New()
+// returns a NullCollector so callers do not need conditional logic.
+
+#include <xrpl/beast/insight/Collector.h>
+#include <xrpl/beast/insight/NullCollector.h>
+#include <xrpl/beast/insight/OTelCollector.h>
+#include <xrpl/beast/utility/Journal.h>
+
+namespace beast::insight {
+
+std::shared_ptr<Collector>
+OTelCollector::New(
+    std::string const& /* endpoint */,
+    std::string const& /* prefix */,
+    std::string const& /* instanceId */,
+    Journal /* journal */)
+{
+    return NullCollector::make();
+}
+
+}  // namespace beast::insight
+
+#endif  // XRPL_ENABLE_TELEMETRY

src/libxrpl/beast/insight/StatsDCollector.cpp

@@ -166,7 +166,7 @@ private:
     std::string name_;
     GaugeImpl::value_type lastValue_{0};
     GaugeImpl::value_type value_{0};
-    bool dirty_{false};
+    bool dirty_{true};
 };
 
 //------------------------------------------------------------------------------
@@ -580,6 +580,9 @@ StatsDEventImpl::doNotify(EventImpl::value_type const& value)
 StatsDGaugeImpl::StatsDGaugeImpl(std::string name, std::shared_ptr<StatsDCollectorImp> const& impl)
     : impl_(impl), name_(std::move(name))
 {
+    // Start dirty so the initial value (0) is emitted on the first flush.
+    // Without this, gauges whose value never changes from 0 would never
+    // appear in downstream metric stores (e.g. Prometheus via StatsD).
     impl_->add(*this);
 }
 

src/libxrpl/telemetry/SpanGuard.cpp

@@ -466,6 +466,7 @@ SpanGuard::addEvent(std::string_view name, std::initializer_list<EventAttribute>
     // std::vector<std::pair<...>> doesn't satisfy is_key_value_iterable.
     // Wrap in nostd::span over the vector's storage so the SDK accepts it.
     std::vector<std::pair<opentelemetry::nostd::string_view, opentelemetry::common::AttributeValue>>
+
         otelAttrs;
     otelAttrs.reserve(attrs.size());
     for (auto const& [k, v] : attrs)

src/test/csf/Peer.h

@@ -12,6 +12,10 @@
 #include <xrpld/consensus/Consensus.h>
 #include <xrpld/consensus/Validations.h>
 
+#ifdef XRPL_ENABLE_TELEMETRY
+#include <xrpl/telemetry/Telemetry.h>
+#endif
+
 #include <xrpl/beast/utility/WrappedSink.h>
 #include <xrpl/protocol/PublicKey.h>
 
@@ -631,6 +635,22 @@ struct Peer
     {
     }
 
+#ifdef XRPL_ENABLE_TELEMETRY
+    /** Provide telemetry access for the Consensus template.
+     *
+     *  The test Peer adaptor uses a static disabled NullTelemetry instance
+     *  so that all shouldTrace*() checks return false and no spans are
+     *  created during simulation tests.
+     */
+    telemetry::Telemetry&
+    getTelemetry()
+    {
+        static auto tel = telemetry::makeTelemetry(
+            telemetry::Telemetry::Setup{}, beast::Journal{beast::Journal::getNullSink()});
+        return *tel;
+    }
+#endif
+
     // Share a message by broadcasting to all connected peers
     template <class M>
     void

src/tests/libxrpl/CMakeLists.txt

@@ -53,10 +53,25 @@ endif()
 xrpl_add_test(telemetry)
 target_link_libraries(xrpl.test.telemetry PRIVATE xrpl.imports.test)
 target_include_directories(xrpl.test.telemetry PRIVATE ${CMAKE_SOURCE_DIR}/src)
+# ValidationTracker lives in xrpld but has no OTel SDK dependency —
+# compile its .cpp directly so the test can link without all of xrpld.
+target_sources(
+    xrpl.test.telemetry
+    PRIVATE ${CMAKE_SOURCE_DIR}/src/xrpld/telemetry/detail/ValidationTracker.cpp
+)
 if(telemetry)
     target_link_libraries(
         xrpl.test.telemetry
         PRIVATE opentelemetry-cpp::opentelemetry-cpp
     )
+else()
+    # MetricsRegistry lives in xrpld; compile its .cpp directly into the test
+    # target so the no-op path can be tested without linking all of xrpld.
+    # When telemetry=ON, XRPL_ENABLE_TELEMETRY is globally defined and the
+    # .cpp pulls in xrpld symbols we cannot satisfy here.
+    target_sources(
+        xrpl.test.telemetry
+        PRIVATE ${CMAKE_SOURCE_DIR}/src/xrpld/telemetry/MetricsRegistry.cpp
+    )
 endif()
 add_dependencies(xrpl.tests xrpl.test.telemetry)

src/tests/libxrpl/helpers/TestServiceRegistry.h

@@ -334,6 +334,12 @@ public:
         throw std::logic_error("TestServiceRegistry::getTelemetry() not implemented");
     }
 
+    telemetry::MetricsRegistry*
+    getMetricsRegistry() override
+    {
+        return nullptr;
+    }
+
     // Configuration and state
     bool
     isStopping() const override

src/tests/libxrpl/telemetry/MetricsRegistry.cpp

@@ -0,0 +1,355 @@
+/** GTest unit tests for MetricsRegistry (no-op / telemetry-disabled path).
+ *
+ *  Tests cover:
+ *  - Construction with telemetry disabled (no-op behavior).
+ *  - start()/stop() lifecycle when disabled.
+ *  - Synchronous instrument recording methods do not crash when disabled.
+ *  - Double stop() is safe.
+ *  - Destructor handles cleanup without crash.
+ *
+ *  NOTE: These tests only exercise the no-op path (telemetry disabled).
+ *  When XRPL_ENABLE_TELEMETRY is defined, MetricsRegistry.cpp pulls in
+ *  xrpld symbols that cannot be linked into this standalone test binary,
+ *  so the tests are compiled out.
+ */
+
+// When telemetry is globally enabled, MetricsRegistry.cpp requires xrpld
+// link dependencies we cannot satisfy in a standalone GTest binary.
+#ifndef XRPL_ENABLE_TELEMETRY
+
+#include <xrpld/telemetry/MetricsRegistry.h>
+
+#include <xrpl/basics/Log.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/beast/utility/Journal.h>
+#include <xrpl/core/ServiceRegistry.h>
+
+#include <boost/asio/io_context.hpp>
+
+#include <gtest/gtest.h>
+
+#include <optional>
+#include <stdexcept>
+#include <string>
+
+using namespace xrpl;
+
+namespace {
+
+/** Minimal mock ServiceRegistry for MetricsRegistry testing.
+ *
+ *  Only the getMetricsRegistry() call is used in the tests; other methods
+ *  are not invoked because the registry is disabled (enabled=false) so no
+ *  gauge callbacks execute.
+ *
+ *  All pure virtual methods throw to catch accidental calls during tests.
+ */
+class MockServiceRegistry : public ServiceRegistry
+{
+    [[noreturn]] static void
+    throwUnimplemented()
+    {
+        throw std::logic_error("MockServiceRegistry: method not implemented");
+    }
+
+public:
+    // ServiceRegistry interface — stubs that should never be called.
+    CollectorManager&
+    getCollectorManager() override
+    {
+        throwUnimplemented();
+    }
+    Family&
+    getNodeFamily() override
+    {
+        throwUnimplemented();
+    }
+    TimeKeeper&
+    getTimeKeeper() override
+    {
+        throwUnimplemented();
+    }
+    JobQueue&
+    getJobQueue() override
+    {
+        throwUnimplemented();
+    }
+    NodeCache&
+    getTempNodeCache() override
+    {
+        throwUnimplemented();
+    }
+    CachedSLEs&
+    getCachedSLEs() override
+    {
+        throwUnimplemented();
+    }
+    NetworkIDService&
+    getNetworkIDService() override
+    {
+        throwUnimplemented();
+    }
+    AmendmentTable&
+    getAmendmentTable() override
+    {
+        throwUnimplemented();
+    }
+    HashRouter&
+    getHashRouter() override
+    {
+        throwUnimplemented();
+    }
+    LoadFeeTrack&
+    getFeeTrack() override
+    {
+        throwUnimplemented();
+    }
+    LoadManager&
+    getLoadManager() override
+    {
+        throwUnimplemented();
+    }
+    RCLValidations&
+    getValidations() override
+    {
+        throwUnimplemented();
+    }
+    ValidatorList&
+    getValidators() override
+    {
+        throwUnimplemented();
+    }
+    ValidatorSite&
+    getValidatorSites() override
+    {
+        throwUnimplemented();
+    }
+    ManifestCache&
+    getValidatorManifests() override
+    {
+        throwUnimplemented();
+    }
+    ManifestCache&
+    getPublisherManifests() override
+    {
+        throwUnimplemented();
+    }
+    Overlay&
+    getOverlay() override
+    {
+        throwUnimplemented();
+    }
+    Cluster&
+    getCluster() override
+    {
+        throwUnimplemented();
+    }
+    PeerReservationTable&
+    getPeerReservations() override
+    {
+        throwUnimplemented();
+    }
+    Resource::Manager&
+    getResourceManager() override
+    {
+        throwUnimplemented();
+    }
+    NodeStore::Database&
+    getNodeStore() override
+    {
+        throwUnimplemented();
+    }
+    SHAMapStore&
+    getSHAMapStore() override
+    {
+        throwUnimplemented();
+    }
+    RelationalDatabase&
+    getRelationalDatabase() override
+    {
+        throwUnimplemented();
+    }
+    InboundLedgers&
+    getInboundLedgers() override
+    {
+        throwUnimplemented();
+    }
+    InboundTransactions&
+    getInboundTransactions() override
+    {
+        throwUnimplemented();
+    }
+    TaggedCache<uint256, AcceptedLedger>&
+    getAcceptedLedgerCache() override
+    {
+        throwUnimplemented();
+    }
+    LedgerMaster&
+    getLedgerMaster() override
+    {
+        throwUnimplemented();
+    }
+    LedgerCleaner&
+    getLedgerCleaner() override
+    {
+        throwUnimplemented();
+    }
+    LedgerReplayer&
+    getLedgerReplayer() override
+    {
+        throwUnimplemented();
+    }
+    PendingSaves&
+    getPendingSaves() override
+    {
+        throwUnimplemented();
+    }
+    [[nodiscard]] OpenLedger&
+    getOpenLedger() override
+    {
+        throwUnimplemented();
+    }
+    [[nodiscard]] OpenLedger const&
+    getOpenLedger() const override
+    {
+        throwUnimplemented();
+    }
+    NetworkOPs&
+    getOPs() override
+    {
+        throwUnimplemented();
+    }
+    OrderBookDB&
+    getOrderBookDB() override
+    {
+        throwUnimplemented();
+    }
+    TransactionMaster&
+    getMasterTransaction() override
+    {
+        throwUnimplemented();
+    }
+    TxQ&
+    getTxQ() override
+    {
+        throwUnimplemented();
+    }
+    PathRequestManager&
+    getPathRequestManager() override
+    {
+        throwUnimplemented();
+    }
+    ServerHandler&
+    getServerHandler() override
+    {
+        throwUnimplemented();
+    }
+    perf::PerfLog&
+    getPerfLog() override
+    {
+        throwUnimplemented();
+    }
+    telemetry::Telemetry&
+    getTelemetry() override
+    {
+        throwUnimplemented();
+    }
+    telemetry::MetricsRegistry*
+    getMetricsRegistry() override
+    {
+        return nullptr;
+    }
+    [[nodiscard]] bool
+    isStopping() const override
+    {
+        return false;
+    }
+    beast::Journal
+    getJournal(std::string const&) override
+    {
+        return beast::Journal(beast::Journal::getNullSink());
+    }
+    boost::asio::io_context&
+    getIOContext() override
+    {
+        throwUnimplemented();
+    }
+    Logs&
+    getLogs() override
+    {
+        throwUnimplemented();
+    }
+    [[nodiscard]] std::optional<uint256> const&
+    getTrapTxID() const override
+    {
+        static std::optional<uint256> const empty;
+        return empty;
+    }
+    DatabaseCon&
+    getWalletDB() override
+    {
+        throwUnimplemented();
+    }
+    Application&
+    getApp() override
+    {
+        throwUnimplemented();
+    }
+};
+
+/// Test fixture that provides a MockServiceRegistry and null Journal.
+class MetricsRegistryTest : public ::testing::Test
+{
+protected:
+    MockServiceRegistry mockApp_;
+    beast::Journal j_{beast::Journal::getNullSink()};
+};
+
+}  // namespace
+
+TEST_F(MetricsRegistryTest, disabled_construction)
+{
+    // Construct with enabled=false; should be a no-op.
+    telemetry::MetricsRegistry const registry(false, mockApp_, j_);
+    EXPECT_FALSE(registry.isEnabled());
+}
+
+TEST_F(MetricsRegistryTest, disabled_start_stop)
+{
+    telemetry::MetricsRegistry registry(false, mockApp_, j_);
+
+    // start() and stop() should be no-ops when disabled.
+    registry.start("http://localhost:4318/v1/metrics");
+    registry.stop();
+
+    // Double stop should be safe.
+    registry.stop();
+}
+
+TEST_F(MetricsRegistryTest, disabled_recording_methods)
+{
+    telemetry::MetricsRegistry registry(false, mockApp_, j_);
+    registry.start("http://localhost:4318/v1/metrics");
+
+    // All recording methods should be no-ops (not crash).
+    registry.recordRpcStarted("server_info");
+    registry.recordRpcFinished("server_info", 1000);
+    registry.recordRpcErrored("ledger", 500);
+    registry.recordJobQueued("ledgerData");
+    registry.recordJobStarted("ledgerData", 200);
+    registry.recordJobFinished("ledgerData", 3000);
+
+    registry.stop();
+}
+
+TEST_F(MetricsRegistryTest, destructor_calls_stop)
+{
+    {
+        // Let the destructor handle cleanup.
+        telemetry::MetricsRegistry registry(false, mockApp_, j_);
+        registry.start("http://localhost:4318/v1/metrics");
+    }
+    // If we get here without crash, the destructor handled stop.
+}
+
+#endif  // !XRPL_ENABLE_TELEMETRY

src/tests/libxrpl/telemetry/SpanGuardFactory.cpp

@@ -30,8 +30,8 @@ TEST(SpanGuardFactory, category_span_returns_null_when_disabled)
     auto span = SpanGuard::span(TraceCategory::Rpc, "rpc", "test");
     EXPECT_FALSE(span);
 
-    span.setAttribute("xrpl.rpc.command", "test");
-    span.setAttribute("xrpl.rpc.status", "success");
+    span.setAttribute("command", "test");
+    span.setAttribute("rpc_status", "success");
 }
 
 TEST(SpanGuardFactory, child_span_null_when_no_parent)

src/tests/libxrpl/telemetry/TraceContextPropagator.cpp

@@ -17,6 +17,8 @@
 #include <opentelemetry/trace/trace_flags.h>
 #include <opentelemetry/trace/trace_id.h>
 
+#include <xrpl.pb.h>
+
 #include <cstdint>
 #include <cstring>
 
@@ -127,8 +129,8 @@ TEST(TraceContextPropagator, inject_invalid_span)
 
 TEST(TraceContextPropagator, flags_preservation)
 {
-    std::uint8_t traceIdBuf[16] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
-    std::uint8_t spanIdBuf[8] = {1, 2, 3, 4, 5, 6, 7, 8};
+    std::uint8_t const traceIdBuf[16] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+    std::uint8_t const spanIdBuf[8] = {1, 2, 3, 4, 5, 6, 7, 8};
 
     // Test with flags NOT sampled (flags = 0)
     trace::TraceFlags const flags(0);

src/tests/libxrpl/telemetry/ValidationTracker.cpp

@@ -0,0 +1,374 @@
+/** @file ValidationTracker.cpp
+    Unit tests for xrpl::telemetry::ValidationTracker.
+*/
+
+#include <xrpld/telemetry/ValidationTracker.h>
+
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/Protocol.h>
+
+#include <gtest/gtest.h>
+
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <thread>
+
+using namespace xrpl;
+using namespace xrpl::telemetry;
+
+/// Helper to create a unique uint256 from an integer seed.
+static uint256
+makeHash(std::uint64_t n)
+{
+    return uint256(n);
+}
+
+/// Test fixture providing a fresh ValidationTracker per test.
+class ValidationTrackerTest : public ::testing::Test
+{
+protected:
+    ValidationTracker tracker_;
+};
+
+// ---------------------------------------------------------------
+// 1. Normal agreement
+//    Record both our validation and network validation for the
+//    same hash, then reconcile after the grace period elapses.
+// ---------------------------------------------------------------
+TEST_F(ValidationTrackerTest, NormalAgreement)
+{
+    auto const hash = makeHash(1);
+    LedgerIndex const seq = 100;
+
+    tracker_.recordOurValidation(hash, seq);
+    tracker_.recordNetworkValidation(hash, seq);
+
+    // Immediately after recording, nothing is reconciled yet
+    // (grace period has not elapsed).
+    tracker_.reconcile();
+    EXPECT_EQ(tracker_.totalValidationsSent(), 1u);
+    EXPECT_EQ(tracker_.totalValidationsChecked(), 1u);
+
+    // Wait for the grace period (8 seconds) to elapse, then reconcile.
+    std::this_thread::sleep_for(std::chrono::seconds(9));
+    tracker_.reconcile();
+
+    EXPECT_EQ(tracker_.totalAgreements(), 1u);
+    EXPECT_EQ(tracker_.totalMissed(), 0u);
+    EXPECT_EQ(tracker_.agreements1h(), 1u);
+    EXPECT_EQ(tracker_.missed1h(), 0u);
+    EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 100.0);
+}
+
+// ---------------------------------------------------------------
+// 2. Missed validation
+//    Only the network validates; we never do. After grace period
+//    the event should be reconciled as a miss.
+// ---------------------------------------------------------------
+TEST_F(ValidationTrackerTest, MissedValidation)
+{
+    auto const hash = makeHash(2);
+    LedgerIndex const seq = 200;
+
+    tracker_.recordNetworkValidation(hash, seq);
+
+    // Wait for grace period then reconcile.
+    std::this_thread::sleep_for(std::chrono::seconds(9));
+    tracker_.reconcile();
+
+    EXPECT_EQ(tracker_.totalAgreements(), 0u);
+    EXPECT_EQ(tracker_.totalMissed(), 1u);
+    EXPECT_EQ(tracker_.agreements1h(), 0u);
+    EXPECT_EQ(tracker_.missed1h(), 1u);
+    EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 0.0);
+}
+
+// ---------------------------------------------------------------
+// 3. Late repair
+//    Network validates first, grace period elapses (miss), then
+//    our validation arrives within the 5-minute repair window and
+//    the miss is flipped to an agreement.
+// ---------------------------------------------------------------
+TEST_F(ValidationTrackerTest, LateRepair)
+{
+    auto const hash = makeHash(3);
+    LedgerIndex const seq = 300;
+
+    // Network validates, but we do not (yet).
+    tracker_.recordNetworkValidation(hash, seq);
+
+    // Grace period elapses -- reconciled as a miss.
+    std::this_thread::sleep_for(std::chrono::seconds(9));
+    tracker_.reconcile();
+    EXPECT_EQ(tracker_.totalMissed(), 1u);
+    EXPECT_EQ(tracker_.totalAgreements(), 0u);
+    EXPECT_EQ(tracker_.missed1h(), 1u);
+
+    // Late arrival of our validation (within repair window).
+    tracker_.recordOurValidation(hash, seq);
+    tracker_.reconcile();
+
+    // Miss should be repaired to agreement.
+    EXPECT_EQ(tracker_.totalAgreements(), 1u);
+    EXPECT_EQ(tracker_.totalMissed(), 0u);
+    EXPECT_EQ(tracker_.agreements1h(), 1u);
+    EXPECT_EQ(tracker_.missed1h(), 0u);
+    EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 100.0);
+}
+
+// ---------------------------------------------------------------
+// 4. Empty window returns 0%
+//    When no events have been recorded the percentage methods
+//    must return 0.0, not NaN or any other value.
+// ---------------------------------------------------------------
+TEST_F(ValidationTrackerTest, EmptyWindowReturnsZero)
+{
+    EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 0.0);
+    EXPECT_DOUBLE_EQ(tracker_.agreementPct24h(), 0.0);
+    EXPECT_EQ(tracker_.agreements1h(), 0u);
+    EXPECT_EQ(tracker_.missed1h(), 0u);
+    EXPECT_EQ(tracker_.agreements24h(), 0u);
+    EXPECT_EQ(tracker_.missed24h(), 0u);
+    EXPECT_EQ(tracker_.totalAgreements(), 0u);
+    EXPECT_EQ(tracker_.totalMissed(), 0u);
+    EXPECT_EQ(tracker_.totalAgreementsEver(), 0u);
+    EXPECT_EQ(tracker_.totalMissedEver(), 0u);
+    EXPECT_EQ(tracker_.totalValidationsSent(), 0u);
+    EXPECT_EQ(tracker_.totalValidationsChecked(), 0u);
+}
+
+// ---------------------------------------------------------------
+// 5. Grace period boundary
+//    Events recorded less than 8 seconds ago must NOT be
+//    reconciled. Verify that an immediate reconcile is a no-op.
+// ---------------------------------------------------------------
+TEST_F(ValidationTrackerTest, GracePeriodBoundary)
+{
+    auto const hash = makeHash(5);
+    LedgerIndex const seq = 500;
+
+    tracker_.recordOurValidation(hash, seq);
+    tracker_.recordNetworkValidation(hash, seq);
+
+    // Reconcile immediately -- grace period has not elapsed.
+    tracker_.reconcile();
+
+    // Nothing should be reconciled yet.
+    EXPECT_EQ(tracker_.totalAgreements(), 0u);
+    EXPECT_EQ(tracker_.totalMissed(), 0u);
+    EXPECT_EQ(tracker_.agreements1h(), 0u);
+    EXPECT_EQ(tracker_.missed1h(), 0u);
+
+    // Lifetime send/check counters should still be incremented.
+    EXPECT_EQ(tracker_.totalValidationsSent(), 1u);
+    EXPECT_EQ(tracker_.totalValidationsChecked(), 1u);
+}
+
+// ---------------------------------------------------------------
+// 6. Max pending events -- trimming
+//    Add more than kMaxPendingEvents (1000) events. After
+//    reconciliation and a second reconcile pass the pending map
+//    should be trimmed. Lifetime totals must remain consistent.
+// ---------------------------------------------------------------
+TEST_F(ValidationTrackerTest, MaxPendingEventsTrimming)
+{
+    constexpr std::size_t kCount = 1100;
+
+    for (std::size_t i = 0; i < kCount; ++i)
+    {
+        auto const hash = makeHash(i + 1);
+        LedgerIndex const seq = static_cast<LedgerIndex>(i + 1);
+        tracker_.recordOurValidation(hash, seq);
+        tracker_.recordNetworkValidation(hash, seq);
+    }
+
+    EXPECT_EQ(tracker_.totalValidationsSent(), kCount);
+    EXPECT_EQ(tracker_.totalValidationsChecked(), kCount);
+
+    // Wait for grace period so all events can be reconciled.
+    std::this_thread::sleep_for(std::chrono::seconds(9));
+    tracker_.reconcile();
+
+    // All events should be reconciled as agreements.
+    EXPECT_EQ(tracker_.totalAgreements(), kCount);
+    EXPECT_EQ(tracker_.totalMissed(), 0u);
+
+    // Reconcile again to trigger pending eviction / trimming.
+    // The pending map should be trimmed, but totals remain correct.
+    tracker_.reconcile();
+    EXPECT_EQ(tracker_.totalAgreements(), kCount);
+    EXPECT_EQ(tracker_.totalMissed(), 0u);
+}
+
+// ---------------------------------------------------------------
+// 7. Multiple distinct ledgers -- mixed results
+//    Record a mix of agreements and misses to verify that window
+//    counts and percentages are computed correctly.
+// ---------------------------------------------------------------
+TEST_F(ValidationTrackerTest, MixedAgreementsAndMisses)
+{
+    // 3 agreements: both sides validate.
+    for (int i = 1; i <= 3; ++i)
+    {
+        auto const hash = makeHash(static_cast<std::uint64_t>(i));
+        tracker_.recordOurValidation(hash, static_cast<LedgerIndex>(i));
+        tracker_.recordNetworkValidation(hash, static_cast<LedgerIndex>(i));
+    }
+
+    // 2 misses: only network validates.
+    for (int i = 4; i <= 5; ++i)
+    {
+        auto const hash = makeHash(static_cast<std::uint64_t>(i));
+        tracker_.recordNetworkValidation(hash, static_cast<LedgerIndex>(i));
+    }
+
+    // Wait for grace period then reconcile.
+    std::this_thread::sleep_for(std::chrono::seconds(9));
+    tracker_.reconcile();
+
+    EXPECT_EQ(tracker_.totalAgreements(), 3u);
+    EXPECT_EQ(tracker_.totalMissed(), 2u);
+    EXPECT_EQ(tracker_.agreements1h(), 3u);
+    EXPECT_EQ(tracker_.missed1h(), 2u);
+
+    // 3 out of 5 = 60%
+    EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 60.0);
+}
+
+// ---------------------------------------------------------------
+// 8. Duplicate recording for same hash
+//    Recording the same hash multiple times should not create
+//    duplicate pending entries or double-count totals beyond the
+//    per-call increments.
+// ---------------------------------------------------------------
+TEST_F(ValidationTrackerTest, DuplicateRecordingSameHash)
+{
+    auto const hash = makeHash(42);
+    LedgerIndex const seq = 42;
+
+    // Record our validation twice for the same hash.
+    tracker_.recordOurValidation(hash, seq);
+    tracker_.recordOurValidation(hash, seq);
+    tracker_.recordNetworkValidation(hash, seq);
+
+    // Each call increments the lifetime counter.
+    EXPECT_EQ(tracker_.totalValidationsSent(), 2u);
+    EXPECT_EQ(tracker_.totalValidationsChecked(), 1u);
+
+    // But only one pending event exists, so only one agreement.
+    std::this_thread::sleep_for(std::chrono::seconds(9));
+    tracker_.reconcile();
+
+    EXPECT_EQ(tracker_.totalAgreements(), 1u);
+    EXPECT_EQ(tracker_.totalMissed(), 0u);
+}
+
+// ---------------------------------------------------------------
+// 9. Only-we-validated scenario
+//    We validate but the network does not. After grace period
+//    this should be a miss (not an agreement).
+// ---------------------------------------------------------------
+TEST_F(ValidationTrackerTest, OnlyWeValidated)
+{
+    auto const hash = makeHash(99);
+    LedgerIndex const seq = 99;
+
+    tracker_.recordOurValidation(hash, seq);
+
+    std::this_thread::sleep_for(std::chrono::seconds(9));
+    tracker_.reconcile();
+
+    EXPECT_EQ(tracker_.totalAgreements(), 0u);
+    EXPECT_EQ(tracker_.totalMissed(), 1u);
+    EXPECT_EQ(tracker_.missed1h(), 1u);
+    EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 0.0);
+}
+
+// ---------------------------------------------------------------
+// 10. Gross miss tally is monotonic across a late repair
+//     The gross lifetime tallies (totalAgreementsEver/totalMissedEver)
+//     back the monotonic Prometheus _total counters. A late repair must
+//     move the NET totals (miss -> agreement) but must NOT move the gross
+//     tallies: a miss already counted stays counted, and the repair does
+//     not add a second (agreement) count for the same ledger.
+// ---------------------------------------------------------------
+TEST_F(ValidationTrackerTest, GrossMissedNeverDecrementsOnRepair)
+{
+    auto const hash = makeHash(10);
+    LedgerIndex const seq = 1000;
+
+    // Network validates, we do not (yet).
+    tracker_.recordNetworkValidation(hash, seq);
+
+    // Grace period elapses -- reconciled as a miss.
+    std::this_thread::sleep_for(std::chrono::seconds(9));
+    tracker_.reconcile();
+
+    // Net and gross both show exactly one initial miss, zero agreements.
+    EXPECT_EQ(tracker_.totalMissed(), 1u);
+    EXPECT_EQ(tracker_.totalMissedEver(), 1u);
+    EXPECT_EQ(tracker_.totalAgreements(), 0u);
+    EXPECT_EQ(tracker_.totalAgreementsEver(), 0u);
+
+    // Late arrival of our validation repairs the miss to an agreement.
+    tracker_.recordOurValidation(hash, seq);
+    tracker_.reconcile();
+
+    // Net totals reflect the repair...
+    EXPECT_EQ(tracker_.totalMissed(), 0u);
+    EXPECT_EQ(tracker_.totalAgreements(), 1u);
+    // ...but the gross tallies are frozen at first classification: the miss
+    // stays counted and no agreement was added (repair path excluded).
+    EXPECT_EQ(tracker_.totalMissedEver(), 1u);
+    EXPECT_EQ(tracker_.totalAgreementsEver(), 0u);
+}
+
+// ---------------------------------------------------------------
+// 11. Gross tallies count initial classification only (additive)
+//     With a mix of initial agreements and misses the gross tallies equal
+//     the net totals. A subsequent repair shifts the net totals but leaves
+//     the gross tallies unchanged, and the gross sum equals the number of
+//     reconciled ledgers (the additive invariant the _total counters rely on).
+// ---------------------------------------------------------------
+TEST_F(ValidationTrackerTest, GrossAgreementsCountInitialOnly)
+{
+    // 3 initial agreements: both sides validate.
+    for (int i = 1; i <= 3; ++i)
+    {
+        auto const h = makeHash(static_cast<std::uint64_t>(i));
+        tracker_.recordOurValidation(h, static_cast<LedgerIndex>(i));
+        tracker_.recordNetworkValidation(h, static_cast<LedgerIndex>(i));
+    }
+
+    // 2 initial misses: only network validates.
+    for (int i = 4; i <= 5; ++i)
+    {
+        auto const h = makeHash(static_cast<std::uint64_t>(i));
+        tracker_.recordNetworkValidation(h, static_cast<LedgerIndex>(i));
+    }
+
+    // Grace period elapses -- all five reconciled at first classification.
+    std::this_thread::sleep_for(std::chrono::seconds(9));
+    tracker_.reconcile();
+
+    // Before any repair, gross equals net.
+    EXPECT_EQ(tracker_.totalAgreements(), 3u);
+    EXPECT_EQ(tracker_.totalAgreementsEver(), 3u);
+    EXPECT_EQ(tracker_.totalMissed(), 2u);
+    EXPECT_EQ(tracker_.totalMissedEver(), 2u);
+
+    // Repair one of the misses (hash 4) within the repair window.
+    tracker_.recordOurValidation(makeHash(4), 4);
+    tracker_.reconcile();
+
+    // Net totals shift by the repair...
+    EXPECT_EQ(tracker_.totalAgreements(), 4u);
+    EXPECT_EQ(tracker_.totalMissed(), 1u);
+    // ...gross tallies stay at the initial classification.
+    EXPECT_EQ(tracker_.totalAgreementsEver(), 3u);
+    EXPECT_EQ(tracker_.totalMissedEver(), 2u);
+
+    // Additive invariant: gross agree + gross miss == ledgers reconciled.
+    EXPECT_EQ(tracker_.totalAgreementsEver() + tracker_.totalMissedEver(), 5u);
+}

src/xrpld/app/consensus/RCLConsensus.cpp

@@ -22,6 +22,7 @@
 #include <xrpld/consensus/ConsensusTypes.h>
 #include <xrpld/overlay/Overlay.h>
 #include <xrpld/overlay/predicates.h>
+#include <xrpld/telemetry/MetricsRegistry.h>
 
 #include <xrpl/basics/Log.h>
 #include <xrpl/basics/Slice.h>
@@ -722,6 +723,10 @@ RCLConsensus::Adaptor::doAccept(
     // See if we can accept a ledger as fully-validated
     ledgerMaster_.consensusBuilt(built.ledger, result.txns.id(), std::move(consensusJson));
 
+    // Record ledger close for OTel dashboard parity counter.
+    if (auto* mr = app_.getMetricsRegistry())
+        mr->incrementLedgersClosed();
+
     //-------------------------------------------------------------------------
     {
         // Apply disputed transactions that didn't get in
@@ -1054,6 +1059,15 @@ RCLConsensus::Adaptor::validate(RCLCxLedger const& ledger, RCLTxSet const& txns,
 
     // Publish to all our subscribers:
     app_.getOPs().pubValidation(v);
+
+    // Record validation sent for OTel dashboard parity counter.
+    if (auto* mr = app_.getMetricsRegistry())
+    {
+        mr->incrementValidationsSent();
+        // Record our validation for the agreement tracker so it can
+        // compare against network-validated ledgers.
+        mr->getValidationTracker().recordOurValidation(ledger.id(), ledger.seq());
+    }
 }
 
 void