Merge otel-phase10 (all otel-phase changes) into perf/allocator-benchmark-notrim

- Add tcmalloc (gperftools) option to conanfile.py + cmake wiring
- Extend gold-linker carveout for tcmalloc
- Disable proactive mallocTrim() in doSweep for allocator benchmarking

.codecov.yml

@@ -36,3 +36,8 @@ ignore:
   - "src/tests/"
   - "include/xrpl/beast/test/"
   - "include/xrpl/beast/unit_test/"
+  # Telemetry modules — conditionally compiled behind XRPL_ENABLE_TELEMETRY,
+  # which is not enabled in coverage builds.
+  - "src/xrpld/telemetry/"
+  - "src/libxrpl/beast/insight/OTelCollector.cpp"
+  - "include/xrpl/beast/insight/OTelCollector.h"

.cspell.config.yaml

@@ -97,6 +97,8 @@ words:
   - dearmor
   - decryptor
   - dedented
+  - Dedup
+  - dedup
   - deleteme
   - demultiplexer
   - deserializaton
@@ -112,6 +114,7 @@ words:
   - enablerepo
   - endmacro
   - envrc
+  - EOCFG
   - exceptioned
   - EXPECT_STREQ
   - exfiltration
@@ -132,6 +135,7 @@ words:
   - godexsoft
   - gpgcheck
   - gpgkey
+  - havetxset
   - hotwallet
   - hicpp
   - htpasswd
@@ -166,6 +170,7 @@ words:
   - libxrpl
   - llection
   - LOCALGOOD
+  - logql
   - logwstream
   - lseq
   - lsmf
@@ -206,6 +211,7 @@ words:
   - nixfmt
   - nixos
   - nixpkgs
+  - NETOP
   - NOLINT
   - NOLINTNEXTLINE
   - nonxrp
@@ -229,9 +235,12 @@ words:
   - permdex
   - perminute
   - permissioned
+  - pgrep
+  - pkill
   - pimpl
   - pointee
   - populator
+  - pratik
   - preauth
   - preauthorization
   - preauthorize
@@ -251,6 +260,7 @@ words:
   - rcflags
   - reparent
   - replayer
+  - reqps
   - rerandomize
   - rerandomization
   - rerandomized
@@ -293,6 +303,7 @@ words:
   - statsd
   - STATSDCOLLECTOR
   - stissue
+  - stledgerentry
   - stnum
   - stobj
   - stobject
@@ -368,5 +379,5 @@ words:
   - xxhasher
   - xychart
   - zpages
-  - pratik
-  - dedup
+  - ripplex
+  - mseconds

.github/scripts/levelization/results/loops.txt

@@ -13,6 +13,12 @@ Loop: xrpld.app xrpld.rpc
 Loop: xrpld.app xrpld.shamap
   xrpld.shamap > xrpld.app
 
+Loop: xrpld.app xrpld.telemetry
+  xrpld.app > xrpld.telemetry
+
 Loop: xrpld.overlay xrpld.rpc
   xrpld.rpc ~= xrpld.overlay
 
+Loop: xrpld.overlay xrpld.telemetry
+  xrpld.telemetry ~= xrpld.overlay
+

.github/scripts/levelization/results/ordering.txt

@@ -109,6 +109,7 @@ test.csf > xrpl.basics
 test.csf > xrpld.consensus
 test.csf > xrpl.json
 test.csf > xrpl.ledger
+test.csf > xrpl.telemetry
 test.json > test.jtx
 test.json > xrpl.json
 test.jtx > test.unit_test
@@ -194,6 +195,7 @@ test.unit_test > xrpl.protocol
 tests.libxrpl > xrpl.basics
 tests.libxrpl > xrpl.config
 tests.libxrpl > xrpl.core
+tests.libxrpl > xrpld.telemetry
 tests.libxrpl > xrpl.json
 tests.libxrpl > xrpl.ledger
 tests.libxrpl > xrpl.net
@@ -253,7 +255,6 @@ xrpld.app > xrpl.config
 xrpld.app > xrpl.core
 xrpld.app > xrpld.consensus
 xrpld.app > xrpld.core
-xrpld.app > xrpld.telemetry
 xrpld.app > xrpl.json
 xrpld.app > xrpl.ledger
 xrpld.app > xrpl.net
@@ -282,7 +283,6 @@ xrpld.overlay > xrpl.core
 xrpld.overlay > xrpld.consensus
 xrpld.overlay > xrpld.core
 xrpld.overlay > xrpld.peerfinder
-xrpld.overlay > xrpld.telemetry
 xrpld.overlay > xrpl.json
 xrpld.overlay > xrpl.ledger
 xrpld.overlay > xrpl.protocol
@@ -301,6 +301,7 @@ xrpld.perflog > xrpl.config
 xrpld.perflog > xrpl.core
 xrpld.perflog > xrpld.app
 xrpld.perflog > xrpld.rpc
+xrpld.perflog > xrpld.telemetry
 xrpld.perflog > xrpl.json
 xrpld.perflog > xrpl.nodestore
 xrpld.perflog > xrpl.protocol
@@ -324,5 +325,12 @@ xrpld.shamap > xrpl.nodestore
 xrpld.shamap > xrpl.protocol
 xrpld.shamap > xrpl.shamap
 xrpld.telemetry > xrpl.basics
+xrpld.telemetry > xrpl.core
 xrpld.telemetry > xrpld.consensus
+xrpld.telemetry > xrpld.core
+xrpld.telemetry > xrpl.json
+xrpld.telemetry > xrpl.nodestore
+xrpld.telemetry > xrpl.protocol
+xrpld.telemetry > xrpl.rdb
+xrpld.telemetry > xrpl.server
 xrpld.telemetry > xrpl.telemetry

.github/scripts/otel-naming/check_otel_naming.py

@@ -814,6 +814,7 @@ def run_rule_d_dashboards(
         "service_version",
         "service_instance_id",
         "job",
+        "job_type",  # standard Prometheus label for job-queue metrics
         "instance",
     }
     # A dashboard label is valid if it is a span attribute (L1), a native-metric

.github/workflows/telemetry-validation.yml

@@ -0,0 +1,310 @@
+# Telemetry Validation CI Workflow
+#
+# Builds rippled with telemetry enabled, runs the multi-node workload
+# harness, validates all telemetry data, and runs performance benchmarks.
+#
+# This is a separate workflow from the main CI. It runs:
+#   - On manual dispatch (workflow_dispatch)
+#   - On pushes to telemetry-related branches
+#
+# The workflow is intentionally heavyweight (builds rippled, starts Docker
+# services, runs a multi-node cluster) — it validates the full telemetry
+# stack end-to-end rather than individual unit tests.
+#
+# Architecture: two jobs to leverage cached dependencies:
+#   1. build-xrpld — runs on a self-hosted runner inside the same container
+#      image the main CI uses (debian-bookworm-gcc-13). This ensures Conan
+#      packages are fetched from the XRPLF remote instead of built from
+#      source, and ccache hits the remote cache.
+#   2. validate-telemetry — runs on ubuntu-latest (which has Docker) to
+#      launch the telemetry stack (OTel collector, Prometheus, Tempo, etc.)
+#      and validate the full pipeline end-to-end.
+
+name: Telemetry Validation
+
+on:
+  workflow_dispatch:
+    inputs:
+      rpc_rate:
+        description: "RPC load rate (requests per second)"
+        required: false
+        default: "50"
+      rpc_duration:
+        description: "RPC load duration (seconds)"
+        required: false
+        default: "120"
+      tx_tps:
+        description: "Transaction submit rate (TPS)"
+        required: false
+        default: "5"
+      tx_duration:
+        description: "Transaction submit duration (seconds)"
+        required: false
+        default: "120"
+      run_benchmark:
+        description: "Run performance benchmarks"
+        required: false
+        type: boolean
+        default: false
+
+  push:
+    branches:
+      - "pratik/otel-phase*"
+      - "feature/otel-*"
+      - "feature/telemetry-*"
+    paths:
+      - ".github/workflows/telemetry-validation.yml"
+      - "docker/telemetry/**"
+      - "include/xrpl/basics/Telemetry*.h"
+      - "src/xrpld/app/misc/Telemetry*"
+
+concurrency:
+  group: telemetry-validation-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  BUILD_DIR: build
+
+jobs:
+  # ── Job 1: Build xrpld in the same container the main CI uses ──────
+  # This ensures Conan binary packages are fetched from the XRPLF remote
+  # (matching package IDs) and ccache hits the remote compilation cache.
+  build-xrpld:
+    name: Build xrpld
+    runs-on: [self-hosted, Linux, X64, heavy]
+    container: ghcr.io/xrplf/ci/debian-bookworm:gcc-13-sha-ab4d1f0
+    timeout-minutes: 60
+    env:
+      CCACHE_NAMESPACE: telemetry-validation
+      CCACHE_REMOTE_ONLY: true
+      CCACHE_REMOTE_STORAGE: http://cache.dev.ripplex.io:8080|layout=bazel
+      CCACHE_SLOPPINESS: include_file_ctime,include_file_mtime
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Prepare runner
+        uses: XRPLF/actions/prepare-runner@90f11ee655d1687824fb8793db770477d52afbab
+        with:
+          enable_ccache: ${{ github.repository_owner == 'XRPLF' }}
+
+      - name: Print build environment
+        uses: XRPLF/actions/print-build-env@59dec886e4afb05a1724443af08baccbc045b574
+
+      - name: Get number of processors
+        uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf
+        id: nproc
+        with:
+          subtract: 2
+
+      - name: Setup Conan
+        uses: ./.github/actions/setup-conan
+
+      - name: Build dependencies
+        uses: ./.github/actions/build-deps
+        with:
+          build_nproc: ${{ steps.nproc.outputs.nproc }}
+          build_type: Release
+          log_verbosity: verbose
+
+      - name: Configure CMake
+        working-directory: ${{ env.BUILD_DIR }}
+        run: |
+          cmake \
+              -G Ninja \
+              -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
+              -DCMAKE_BUILD_TYPE=Release \
+              ..
+
+      - name: Build xrpld
+        working-directory: ${{ env.BUILD_DIR }}
+        env:
+          BUILD_NPROC: ${{ steps.nproc.outputs.nproc }}
+        run: |
+          cmake \
+              --build . \
+              --config Release \
+              --parallel "${BUILD_NPROC}" \
+              --target xrpld
+
+      - name: Show ccache statistics
+        if: ${{ github.repository_owner == 'XRPLF' }}
+        run: ccache --show-stats -vv
+
+      - name: Upload xrpld binary
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: xrpld-telemetry
+          path: ${{ env.BUILD_DIR }}/xrpld
+          retention-days: 1
+          if-no-files-found: error
+
+  # ── Job 2: Run telemetry validation on ubuntu-latest (has Docker) ──
+  validate-telemetry:
+    name: Telemetry Stack Validation
+    needs: build-xrpld
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Install Python dependencies
+        run: pip3 install -r docker/telemetry/workload/requirements.txt
+
+      - name: Download xrpld binary
+        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
+        with:
+          name: xrpld-telemetry
+          path: ${{ env.BUILD_DIR }}
+
+      - name: Make binaries and scripts executable
+        run: |
+          chmod +x ${{ env.BUILD_DIR }}/xrpld
+          chmod +x docker/telemetry/workload/*.sh
+
+      - name: Run full telemetry validation
+        id: validation
+        env:
+          RPC_RATE: ${{ github.event.inputs.rpc_rate || '50' }}
+          RPC_DURATION: ${{ github.event.inputs.rpc_duration || '120' }}
+          TX_TPS: ${{ github.event.inputs.tx_tps || '5' }}
+          TX_DURATION: ${{ github.event.inputs.tx_duration || '120' }}
+          RUN_BENCHMARK: ${{ github.event.inputs.run_benchmark }}
+        run: |
+          ARGS="--xrpld ${{ env.BUILD_DIR }}/xrpld --skip-loki"
+          ARGS="$ARGS --rpc-rate $RPC_RATE"
+          ARGS="$ARGS --rpc-duration $RPC_DURATION"
+          ARGS="$ARGS --tx-tps $TX_TPS"
+          ARGS="$ARGS --tx-duration $TX_DURATION"
+          if [ "$RUN_BENCHMARK" = "true" ]; then
+              ARGS="$ARGS --with-benchmark"
+          fi
+          docker/telemetry/workload/run-full-validation.sh $ARGS
+        # continue-on-error allows subsequent steps (artifact upload,
+        # summary printing) to run even if validation fails. The final
+        # "Check validation result" step re-checks steps.validation.outcome
+        # (the pre-continue-on-error result) and fails the job properly.
+        continue-on-error: true
+
+      - name: Upload validation reports
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: telemetry-validation-reports
+          path: /tmp/xrpld-validation/reports/
+          retention-days: 30
+
+      - name: Upload node logs
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: xrpld-node-logs
+          path: /tmp/xrpld-validation/node*/debug.log
+          retention-days: 7
+
+      - name: Print validation summary
+        if: always()
+        run: |
+          REPORT="/tmp/xrpld-validation/reports/validation-report.json"
+          if [ -f "$REPORT" ]; then
+              echo "## Telemetry Validation Results" >>"$GITHUB_STEP_SUMMARY"
+              echo "" >>"$GITHUB_STEP_SUMMARY"
+              TOTAL=$(jq '.summary.total' "$REPORT")
+              PASSED=$(jq '.summary.passed' "$REPORT")
+              FAILED=$(jq '.summary.failed' "$REPORT")
+              echo "| Metric | Value |" >>"$GITHUB_STEP_SUMMARY"
+              echo "|--------|-------|" >>"$GITHUB_STEP_SUMMARY"
+              echo "| Total Checks | $TOTAL |" >>"$GITHUB_STEP_SUMMARY"
+              echo "| Passed | $PASSED |" >>"$GITHUB_STEP_SUMMARY"
+              echo "| Failed | $FAILED |" >>"$GITHUB_STEP_SUMMARY"
+              echo "" >>"$GITHUB_STEP_SUMMARY"
+              if [ "$FAILED" -gt 0 ]; then
+                  echo "### Failed Checks" >>"$GITHUB_STEP_SUMMARY"
+                  echo "" >>"$GITHUB_STEP_SUMMARY"
+                  jq -r '.checks[] | select(.passed == false) | "- **\(.name)**: \(.message)"' "$REPORT" >>"$GITHUB_STEP_SUMMARY"
+              fi
+          fi
+
+      # Publishes captured OTel timings + regression report to the Step Summary.
+      # When the committed baseline is a placeholder, emits a fenced JSON block
+      # that can be copy-pasted directly into baselines/baseline-timings.json.
+      # When the baseline is populated, summarises the top regressions so the
+      # PR author sees the failure reason without downloading artifacts.
+      - name: Print regression summary
+        if: always()
+        run: |
+          set -euo pipefail
+          TIMINGS="/tmp/xrpld-validation/reports/timings.json"
+          REGRESSION="/tmp/xrpld-validation/reports/regression-report.json"
+          BASELINE="docker/telemetry/workload/baselines/baseline-timings.json"
+
+          if [ ! -f "$TIMINGS" ]; then
+              echo "## Regression Gate: no timings captured" >>"$GITHUB_STEP_SUMMARY"
+              echo "::warning::capture_timings.py did not produce timings.json — regression gate was not evaluated."
+              exit 0
+          fi
+
+          if [ ! -f "$BASELINE" ]; then
+              echo "## Regression Gate: baseline file missing" >>"$GITHUB_STEP_SUMMARY"
+              echo "::error::baselines/baseline-timings.json not found in checkout"
+              exit 1
+          fi
+
+          # NOTE: do NOT use `jq -e` here. With -e, jq exits non-zero when the
+          # filter's result is boolean false — which is the normal case for a
+          # populated (non-placeholder) baseline — and that would be
+          # misreported as a parse failure. Plain `jq -r` exits 0 on any valid
+          # JSON, so a real non-zero exit genuinely means malformed JSON.
+          IS_PLACEHOLDER=$(jq -r '.placeholder == true or (.metrics | length == 0)' "$BASELINE") || {
+              echo "::error::Failed to parse baseline JSON"
+              exit 1
+          }
+
+          echo "## OTel Timings Regression Gate" >>"$GITHUB_STEP_SUMMARY"
+          echo "" >>"$GITHUB_STEP_SUMMARY"
+
+          if [ "$IS_PLACEHOLDER" = "true" ]; then
+              echo "### Paste into \`baselines/baseline-timings.json\`" >>"$GITHUB_STEP_SUMMARY"
+              echo "" >>"$GITHUB_STEP_SUMMARY"
+              echo "The committed baseline is a placeholder. Open a PR replacing" \
+                  "its contents with the JSON block below to activate the" \
+                  "regression gate." >>"$GITHUB_STEP_SUMMARY"
+              echo "" >>"$GITHUB_STEP_SUMMARY"
+              echo '```json' >>"$GITHUB_STEP_SUMMARY"
+              cat "$TIMINGS" >>"$GITHUB_STEP_SUMMARY"
+              echo '```' >>"$GITHUB_STEP_SUMMARY"
+          elif [ -f "$REGRESSION" ]; then
+              REGR_COUNT=$(jq -e '.summary.regressions' "$REGRESSION") || REGR_COUNT=0
+              IMPR_COUNT=$(jq -e '.summary.improvements' "$REGRESSION") || IMPR_COUNT=0
+              TOTAL=$(jq -e '.summary.total' "$REGRESSION") || TOTAL=0
+              echo "| Stat | Count |" >>"$GITHUB_STEP_SUMMARY"
+              echo "|------|-------|" >>"$GITHUB_STEP_SUMMARY"
+              echo "| Metrics compared | $TOTAL |" >>"$GITHUB_STEP_SUMMARY"
+              echo "| Regressions | $REGR_COUNT |" >>"$GITHUB_STEP_SUMMARY"
+              echo "| Improvements | $IMPR_COUNT |" >>"$GITHUB_STEP_SUMMARY"
+              echo "" >>"$GITHUB_STEP_SUMMARY"
+              if [ "$REGR_COUNT" -gt 0 ]; then
+                  echo "### Regressions" >>"$GITHUB_STEP_SUMMARY"
+                  echo "" >>"$GITHUB_STEP_SUMMARY"
+                  echo "| Metric | Baseline | Current | Δ | % | Unit |" >>"$GITHUB_STEP_SUMMARY"
+                  echo "|--------|---------:|--------:|--:|--:|------|" >>"$GITHUB_STEP_SUMMARY"
+                  jq -r '.metrics[] | select(.regressed) | "| \(.key) | \(.baseline) | \(.current) | \(.delta) | \(.pct_change)% | \(.unit) |"' \
+                      "$REGRESSION" >>"$GITHUB_STEP_SUMMARY"
+              fi
+          fi
+
+      - name: Cleanup
+        if: always()
+        run: |
+          docker/telemetry/workload/run-full-validation.sh --cleanup 2>/dev/null || true
+
+      - name: Check validation result
+        if: steps.validation.outcome == 'failure'
+        run: |
+          echo "Telemetry validation failed. Check the uploaded reports for details."
+          exit 1

.gitignore

@@ -86,6 +86,5 @@ __pycache__
 
 # clangd cache
 /.cache
-
-# Env. file carrying environmental setup data for local or cloud runs.
-.env.*
+docker/telemetry/workload/__pycache__/
+.claude/

OpenTelemetryPlan/02-design-decisions.md

@@ -639,6 +639,8 @@ parent through the active context.
 
 ### 2.6.4 Coexistence Strategy
 
+> **Note**: Phase 7 replaces the StatsD bridge with native OTel Metrics SDK export. The diagram below shows the Phase 6 intermediate state. See [Phase7_taskList.md](./Phase7_taskList.md) for the migration design where Beast Insight emits via OTLP instead of StatsD.
+
 ```mermaid
 flowchart TB
     subgraph xrpld["xrpld Process"]
@@ -667,6 +669,8 @@ flowchart TB
 - **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/HTTP to a Collector, which then forwards to a trace backend (Tempo). (OTLP/gRPC is future work — §2.2.2.)
 - **Grafana (red, unified UI)**: All three data streams converge in Grafana, enabling operators to correlate logs, metrics, and traces in a single dashboard.
 
+**Phase 7 target state**: Beast Insight routes to `OTelCollector` (new `Collector` implementation) which exports via OTLP/HTTP to the same collector endpoint as traces. StatsD UDP path becomes a deprecated fallback (`[insight] server=statsd`). See [06-implementation-phases.md §6.8](./06-implementation-phases.md) and [Phase7_taskList.md](./Phase7_taskList.md) for details.
+
 ### 2.6.5 Correlation with PerfLog
 
 Trace IDs can be correlated with existing PerfLog entries for comprehensive

OpenTelemetryPlan/05-configuration-reference.md

@@ -244,13 +244,17 @@ Modify PerfLog so its JSON output includes a `trace_id` field whenever a valid s
 
 In the Tempo datasource, set the `tracesToLogs` derived field to link to Loki on the `trace_id` and `tx_hash` tags, with `filterByTraceID: true`.
 
-### 5.8.7 Correlation with Insight/StatsD Metrics
+### 5.8.7 Correlation with Insight/OTel System Metrics
 
-To correlate traces with existing Beast Insight metrics:
+To correlate traces with Beast Insight system metrics:
 
 **Step 1: Export Insight metrics to Prometheus**
 
-Add a Prometheus scrape job (`prometheus.yaml`) named `xrpld-statsd` targeting the StatsD exporter at `statsd-exporter:9102`.
+Beast Insight metrics are exported natively via OTLP to the OTel Collector,
+which exposes them on the Prometheus endpoint alongside spanmetrics. Configure
+the `[insight]` section of `xrpld.cfg` with `server=otel`,
+`endpoint=http://localhost:4318/v1/metrics`, and `prefix=xrpld`; no separate
+StatsD exporter or Prometheus scrape job is needed when using `server=otel`.
 
 **Step 2: Add exemplars to metrics**
 
@@ -262,7 +266,7 @@ In the Prometheus datasource, set `exemplarTraceIdDestinations` to map the `trac
 
 **Step 4: Dashboard panel with exemplars**
 
-Add a timeseries panel over Prometheus (e.g. `histogram_quantile(0.99, rate(xrpld_rpc_duration_seconds_bucket[5m]))`) with `exemplar: true` enabled.
+Add a timeseries panel over Prometheus (e.g. `histogram_quantile(0.99, rate(rpc_duration_seconds_bucket[5m]))`) with `exemplar: true` enabled.
 
 This allows clicking on metric data points to jump directly to the related trace.
 

OpenTelemetryPlan/08-appendix.md

@@ -43,6 +43,18 @@
 | **LoadManager**   | Dynamic fee escalation based on network load                  |
 | **SHAMap**        | SHA-256 hash-based map (Merkle trie variant) for ledger state |
 
+### Phase 9–11 Terms
+
+| Term                        | Definition                                                                |
+| --------------------------- | ------------------------------------------------------------------------- |
+| **MetricsRegistry**         | Centralized class for OTel async gauge registrations (Phase 9)            |
+| **ObservableGauge**         | OTel Metrics SDK async instrument polled via callback at fixed intervals  |
+| **PeriodicMetricReader**    | OTel SDK component that invokes gauge callbacks at configurable intervals |
+| **CountedObject**           | xrpld template that tracks live instance counts via atomic counters       |
+| **TxQ**                     | Transaction queue managing fee escalation and ordering                    |
+| **Load Factor**             | Combined multiplier affecting transaction cost (local, cluster, network)  |
+| **OTel Collector Receiver** | Custom Go plugin that polls xrpld RPC and emits OTel metrics (Phase 11)   |
+
 ---
 
 ## 8.2 Span Hierarchy Visualization
@@ -162,7 +174,8 @@ flowchart TB
 | ------- | ---------- | ------ | -------------------------------------------------------------- |
 | 1.0     | 2026-02-12 | -      | Initial implementation plan                                    |
 | 1.1     | 2026-02-13 | -      | Refactored into modular documents                              |
-| 1.2     | 2026-03-24 | -      | Review fixes: accuracy corrections, cross-document consistency |
+| 1.2     | 2026-03-09 | -      | Added Phases 9–11 (future enhancement plans)                   |
+| 1.3     | 2026-03-24 | -      | Review fixes: accuracy corrections, cross-document consistency |
 
 ---
 
@@ -170,27 +183,79 @@ flowchart TB
 
 ### Plan Documents
 
-| Document                                                         | Description                                        |
-| ---------------------------------------------------------------- | -------------------------------------------------- |
-| [OpenTelemetryPlan.md](./OpenTelemetryPlan.md)                   | Master overview and executive summary              |
-| [00-tracing-fundamentals.md](./00-tracing-fundamentals.md)       | Distributed tracing concepts and OTel primer       |
-| [01-architecture-analysis.md](./01-architecture-analysis.md)     | xrpld architecture and trace points                |
-| [02-design-decisions.md](./02-design-decisions.md)               | SDK selection, exporters, span conventions         |
-| [03-implementation-strategy.md](./03-implementation-strategy.md) | Directory structure, performance analysis          |
-| [05-configuration-reference.md](./05-configuration-reference.md) | xrpld config, CMake, Collector configs             |
-| [06-implementation-phases.md](./06-implementation-phases.md)     | Timeline, tasks, risks, success metrics            |
-| [07-observability-backends.md](./07-observability-backends.md)   | Backend selection and architecture                 |
-| [08-appendix.md](./08-appendix.md)                               | Glossary, references, version history              |
-| [secure-OTel.md](./secure-OTel.md)                               | Threat model and hardening (mTLS, peer validation) |
+| Document                                                             | Description                                        |
+| -------------------------------------------------------------------- | -------------------------------------------------- |
+| [OpenTelemetryPlan.md](./OpenTelemetryPlan.md)                       | Master overview and executive summary              |
+| [00-tracing-fundamentals.md](./00-tracing-fundamentals.md)           | Distributed tracing concepts and OTel primer       |
+| [01-architecture-analysis.md](./01-architecture-analysis.md)         | xrpld architecture and trace points                |
+| [02-design-decisions.md](./02-design-decisions.md)                   | SDK selection, exporters, span conventions         |
+| [03-implementation-strategy.md](./03-implementation-strategy.md)     | Directory structure, performance analysis          |
+| [05-configuration-reference.md](./05-configuration-reference.md)     | xrpld config, CMake, Collector configs             |
+| [06-implementation-phases.md](./06-implementation-phases.md)         | Timeline, tasks, risks, success metrics            |
+| [07-observability-backends.md](./07-observability-backends.md)       | Backend selection and architecture                 |
+| [08-appendix.md](./08-appendix.md)                                   | Glossary, references, version history              |
+| [secure-OTel.md](./secure-OTel.md)                                   | Threat model and hardening (mTLS, peer validation) |
+| [09-data-collection-reference.md](./09-data-collection-reference.md) | Span/metric/dashboard inventory                    |
 
 ### Task Lists
 
-| Document                                   | Description                          |
-| ------------------------------------------ | ------------------------------------ |
-| [Phase2_taskList.md](./Phase2_taskList.md) | RPC layer trace instrumentation      |
-| [Phase3_taskList.md](./Phase3_taskList.md) | Peer overlay & consensus tracing     |
-| [Phase4_taskList.md](./Phase4_taskList.md) | Transaction lifecycle tracing        |
-| [Phase5_taskList.md](./Phase5_taskList.md) | Ledger processing & advanced tracing |
+| Document                                                                   | Description                                         |
+| -------------------------------------------------------------------------- | --------------------------------------------------- |
+| [Phase2_taskList.md](./Phase2_taskList.md)                                 | RPC layer trace instrumentation                     |
+| [Phase3_taskList.md](./Phase3_taskList.md)                                 | Peer overlay & consensus tracing                    |
+| [Phase4_taskList.md](./Phase4_taskList.md)                                 | Transaction lifecycle tracing                       |
+| [Phase5_taskList.md](./Phase5_taskList.md)                                 | Ledger processing & advanced tracing                |
+| [Phase5_IntegrationTest_taskList.md](./Phase5_IntegrationTest_taskList.md) | Observability stack integration tests               |
+| [Phase7_taskList.md](./Phase7_taskList.md)                                 | Native OTel metrics migration                       |
+| [Phase8_taskList.md](./Phase8_taskList.md)                                 | Log-trace correlation                               |
+| [Phase9_taskList.md](./Phase9_taskList.md)                                 | Internal metric instrumentation gap fill (future)   |
+| [Phase10_taskList.md](./Phase10_taskList.md)                               | Synthetic workload generation & validation (future) |
+| [Phase11_taskList.md](./Phase11_taskList.md)                               | Third-party data collection pipelines (future)      |
+
+> **Note**: Phases 1 and 6 do not have separate task list files. Phase 1 tasks are documented in [06-implementation-phases.md §6.2](./06-implementation-phases.md). Phase 6 tasks are documented in [06-implementation-phases.md §6.7](./06-implementation-phases.md).
+
+---
+
+## 8.6 Phase 9–11 Cross-Reference Guide
+
+This guide maps Phase 9–11 content to its location across the documentation.
+
+### Phase 9: Internal Metric Instrumentation Gap Fill
+
+| Content                         | Location                                                                 |
+| ------------------------------- | ------------------------------------------------------------------------ |
+| Plan & architecture             | [06-implementation-phases.md §6.8.2](./06-implementation-phases.md)      |
+| Task list (10 tasks)            | [Phase9_taskList.md](./Phase9_taskList.md)                               |
+| Future metric definitions (~50) | [09-data-collection-reference.md §5b](./09-data-collection-reference.md) |
+| New class: `MetricsRegistry`    | `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (planned)                   |
+| New dashboards                  | `fee-market`, `job-queue` (planned)                                      |
+
+**Metric categories**: NodeStore I/O, Cache Hit Rates, TxQ, PerfLog Per-RPC, PerfLog Per-Job, Counted Objects, Fee Escalation & Load Factors.
+
+### Phase 10: Synthetic Workload Generation & Telemetry Validation
+
+| Content              | Location                                                                 |
+| -------------------- | ------------------------------------------------------------------------ |
+| Plan & architecture  | [06-implementation-phases.md §6.8.3](./06-implementation-phases.md)      |
+| Task list (7 tasks)  | [Phase10_taskList.md](./Phase10_taskList.md)                             |
+| Validation inventory | [09-data-collection-reference.md §5c](./09-data-collection-reference.md) |
+| Test harness         | `docker/telemetry/docker-compose.workload.yaml` (planned)                |
+| CI workflow          | `.github/workflows/telemetry-validation.yml` (planned)                   |
+
+**Validates**: 16 spans, 22 attributes, 300+ metrics, 10 dashboards, log-trace correlation.
+
+### Phase 11: Third-Party Data Collection Pipelines
+
+| Content                           | Location                                                                 |
+| --------------------------------- | ------------------------------------------------------------------------ |
+| Plan & architecture               | [06-implementation-phases.md §6.8.4](./06-implementation-phases.md)      |
+| Task list (11 tasks)              | [Phase11_taskList.md](./Phase11_taskList.md)                             |
+| External metric definitions (~30) | [09-data-collection-reference.md §5d](./09-data-collection-reference.md) |
+| Custom OTel Collector receiver    | `docker/telemetry/otel-rippled-receiver/` (planned)                      |
+| Prometheus alerting rules (11)    | [09-data-collection-reference.md §5d](./09-data-collection-reference.md) |
+| New dashboards (4)                | Validator Health, Network Topology, Fee Market (External), DEX & AMM     |
+
+**Consumer categories**: Exchanges, Payment Processors, DeFi/AMM, NFT Marketplaces, Analytics Providers, Wallets, Compliance, Academic Researchers, Institutional Custody, CBDC Bridge Operators.
 
 ---
 

OpenTelemetryPlan/OpenTelemetryPlan.md

@@ -54,6 +54,7 @@ flowchart TB
         backends["07-observability-backends.md"]
         appendix["08-appendix.md"]
         secure["secure-OTel.md"]
+        dataref["09-data-collection-reference.md"]
     end
 
     overview --> fundamentals
@@ -69,6 +70,7 @@ flowchart TB
     phases --> backends
     backends --> appendix
     backends --> secure
+    appendix --> dataref
 
     style overview fill:#1b5e20,stroke:#0d3d14,color:#fff,stroke-width:2px
     style fundamentals fill:#00695c,stroke:#004d40,color:#fff
@@ -84,6 +86,7 @@ flowchart TB
     style backends fill:#4a148c,stroke:#2e0d57,color:#fff
     style appendix fill:#4a148c,stroke:#2e0d57,color:#fff
     style secure fill:#4a148c,stroke:#2e0d57,color:#fff
+    style dataref fill:#4a148c,stroke:#2e0d57,color:#fff
 ```
 
 </div>
@@ -92,17 +95,18 @@ flowchart TB
 
 ## Table of Contents
 
-| Section | Document                                                   | Description                                                            |
-| ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------- |
-| **0**   | [Tracing Fundamentals](./00-tracing-fundamentals.md)       | Distributed tracing concepts, span relationships, context propagation  |
-| **1**   | [Architecture Analysis](./01-architecture-analysis.md)     | xrpld component analysis, trace points, instrumentation priorities     |
-| **2**   | [Design Decisions](./02-design-decisions.md)               | SDK selection, exporters, span naming, attributes, context propagation |
-| **3**   | [Implementation Strategy](./03-implementation-strategy.md) | Directory structure, key principles, performance optimization          |
-| **5**   | [Configuration Reference](./05-configuration-reference.md) | xrpld config, CMake integration, Collector configurations              |
-| **6**   | [Implementation Phases](./06-implementation-phases.md)     | 5-phase timeline, tasks, risks, success metrics                        |
-| **7**   | [Observability Backends](./07-observability-backends.md)   | Backend selection guide and production architecture                    |
-| **8**   | [Appendix](./08-appendix.md)                               | Glossary, references, version history                                  |
-| **Sec** | [Securing the OTel Pipeline](./secure-OTel.md)             | Threat model and hardening (mTLS, peer trace-context validation)       |
+| Section | Document                                                       | Description                                                            |
+| ------- | -------------------------------------------------------------- | ---------------------------------------------------------------------- |
+| **0**   | [Tracing Fundamentals](./00-tracing-fundamentals.md)           | Distributed tracing concepts, span relationships, context propagation  |
+| **1**   | [Architecture Analysis](./01-architecture-analysis.md)         | xrpld component analysis, trace points, instrumentation priorities     |
+| **2**   | [Design Decisions](./02-design-decisions.md)                   | SDK selection, exporters, span naming, attributes, context propagation |
+| **3**   | [Implementation Strategy](./03-implementation-strategy.md)     | Directory structure, key principles, performance optimization          |
+| **5**   | [Configuration Reference](./05-configuration-reference.md)     | xrpld config, CMake integration, Collector configurations              |
+| **6**   | [Implementation Phases](./06-implementation-phases.md)         | 5-phase timeline, tasks, risks, success metrics                        |
+| **7**   | [Observability Backends](./07-observability-backends.md)       | Backend selection guide and production architecture                    |
+| **8**   | [Appendix](./08-appendix.md)                                   | Glossary, references, version history                                  |
+| **9**   | [Data Collection Reference](./09-data-collection-reference.md) | Complete inventory of spans, attributes, metrics, and dashboards       |
+| **Sec** | [Securing the OTel Pipeline](./secure-OTel.md)                 | Threat model and hardening (mTLS, peer trace-context validation)       |
 
 ---
 
@@ -164,17 +168,20 @@ OpenTelemetry Collector configurations are provided for development and producti
 
 ## 6. Implementation Phases
 
-The implementation spans 9 weeks across 5 phases:
+The implementation spans 13 weeks across 8 phases:
 
-| Phase | Duration  | Focus               | Key Deliverables                                    |
-| ----- | --------- | ------------------- | --------------------------------------------------- |
-| 1     | Weeks 1-2 | Core Infrastructure | SDK integration, Telemetry interface, Configuration |
-| 2     | Weeks 3-4 | RPC Tracing         | HTTP context extraction, Handler instrumentation    |
-| 3     | Weeks 5-6 | Transaction Tracing | Protocol Buffer context, Relay propagation          |
-| 4     | Weeks 7-8 | Consensus Tracing   | Round spans, Proposal/validation tracing            |
-| 5     | Week 9    | Documentation       | Runbook, Dashboards, Training                       |
+| Phase | Duration    | Focus                 | Key Deliverables                                            |
+| ----- | ----------- | --------------------- | ----------------------------------------------------------- |
+| 1     | Weeks 1-2   | Core Infrastructure   | SDK integration, Telemetry interface, Configuration         |
+| 2     | Weeks 3-4   | RPC Tracing           | HTTP context extraction, Handler instrumentation            |
+| 3     | Weeks 5-6   | Transaction Tracing   | Protocol Buffer context, Relay propagation                  |
+| 4     | Weeks 7-8   | Consensus Tracing     | Round spans, Proposal/validation tracing                    |
+| 5     | Week 9      | Documentation         | Runbook, Dashboards, Training                               |
+| 6     | Week 10     | StatsD Metrics Bridge | OTel Collector StatsD receiver, 3 Grafana dashboards        |
+| 7     | Weeks 11-12 | Native OTel Metrics   | OTelCollector impl, OTLP metrics export, StatsD deprecation |
+| 8     | Week 13     | Log-Trace Correlation | trace_id in logs, Loki ingestion, Tempo↔Loki linking        |
 
-**Total Effort**: 47 person-days (2 developers working in parallel)
+**Total Effort**: 65.1 developer-days with 2 developers
 
 ➡️ **[View full Implementation Phases](./06-implementation-phases.md)**
 
@@ -200,6 +207,14 @@ The appendix contains a glossary of OpenTelemetry and xrpld-specific terms, refe
 
 ---
 
+## 9. Data Collection Reference
+
+A single-source-of-truth reference documenting every piece of telemetry data collected by xrpld. Covers all 16 OpenTelemetry spans with their 22 attributes, all StatsD metrics (gauges, counters, histograms, overlay traffic), SpanMetrics-derived Prometheus metrics, and all 10 Grafana dashboards. Includes Tempo search guides and Prometheus query examples.
+
+➡️ **[View Data Collection Reference](./09-data-collection-reference.md)**
+
+---
+
 ## Securing the OTel Pipeline
 
 Threat model and hardening guidance for production deployments where xrpld nodes ship telemetry to a centrally-hosted collector across an untrusted network. Covers the two attack surfaces (collector ingress and peer trace-context spoofing) and the chosen defenses: mTLS as primary collector auth, NetworkPolicy as defense-in-depth, and source-side validation plus per-peer rate limiting for the `protocol::TraceContext` field on peer messages.

OpenTelemetryPlan/Phase10_taskList.md

@@ -0,0 +1,258 @@
+# Phase 10: Synthetic Workload Generation & Telemetry Validation — Task List
+
+> **Status**: Future Enhancement
+>
+> **Goal**: Build tools that generate realistic XRPL traffic to validate the full Phases 1-9 telemetry stack end-to-end — all spans, attributes, metrics, dashboards, and log-trace correlation — under controlled load.
+>
+> **Scope**: Python/shell test harness + multi-node docker-compose environment + automated validation scripts + performance benchmarks.
+>
+> **Branch**: `pratik/otel-phase10-workload-validation` (from `pratik/otel-phase9-metric-gap-fill`)
+>
+> **Depends on**: Phase 9 (internal metric gap fill) — validates the full metric surface
+
+### Related Plan Documents
+
+| Document                                                             | Relevance                                                       |
+| -------------------------------------------------------------------- | --------------------------------------------------------------- |
+| [06-implementation-phases.md](./06-implementation-phases.md)         | Phase 10 plan: motivation, architecture, exit criteria (§6.8.3) |
+| [09-data-collection-reference.md](./09-data-collection-reference.md) | Defines the full inventory of spans/metrics to validate         |
+| [Phase9_taskList.md](./Phase9_taskList.md)                           | Prerequisite — all internal metrics must be emitting            |
+
+### Why This Phase Exists
+
+Before Phases 1-9 can be considered production-ready, we need proof that:
+
+1. All required spans fire with correct attributes under real transaction workloads
+2. All 255+ StatsD metrics + ~50 Phase 9 metrics appear in Prometheus with non-zero values
+3. Log-trace correlation (Phase 8) produces clickable trace_id links in Loki
+4. All 10 Grafana dashboards render meaningful data (no empty panels)
+5. Performance overhead stays within bounds (< 3% CPU, < 5MB memory)
+6. The telemetry stack survives sustained load without data loss or queue backpressure
+
+---
+
+## Task 10.1: Multi-Node Test Harness
+
+**Objective**: Create a docker-compose environment with 3-5 validator nodes that produces real consensus rounds.
+
+**What to do**:
+
+- Create `docker/telemetry/docker-compose.workload.yaml`:
+  - 5 xrpld validator nodes with UNL configured for each other
+  - All telemetry enabled: `[telemetry] enabled=1`, `[insight] server=otel`
+  - Full OTel stack: Collector, Tempo, Prometheus, Loki, Grafana
+  - Shared network with service discovery
+
+- Each node should:
+  - Generate validator keys at startup
+  - Configure all 5 nodes in its UNL
+  - Enable all trace categories including `trace_peer=1`
+  - Write logs to a file tailed by the OTel Collector filelog receiver
+
+- Include a `Makefile` target: `make telemetry-workload-up` / `make telemetry-workload-down`
+
+**Key files**:
+
+- New: `docker/telemetry/docker-compose.workload.yaml`
+- New: `docker/telemetry/workload/generate-validator-keys.sh`
+- New: `docker/telemetry/workload/xrpld-validator.cfg.template`
+
+---
+
+## Task 10.2: RPC Load Generator
+
+**Objective**: Configurable tool that fires all traced RPC commands at controlled rates.
+
+**What to do**:
+
+- Create `docker/telemetry/workload/rpc_load_generator.py`:
+  - Connects to one or more xrpld WebSocket endpoints
+  - Fires all RPC commands that have trace spans: `server_info`, `ledger`, `tx`, `account_info`, `account_lines`, `fee`, `submit`, etc.
+  - Configurable parameters: rate (RPS), duration, command distribution weights
+  - Injects `traceparent` HTTP headers to test W3C context propagation
+  - Logs progress and errors to stdout
+
+- Command distribution should match realistic production ratios:
+  - 40% `server_info` / `fee` (health checks)
+  - 30% `account_info` / `account_lines` / `account_objects` (wallet queries)
+  - 15% `ledger` / `ledger_data` (explorer queries)
+  - 10% `tx` / `account_tx` (transaction lookups)
+  - 5% `book_offers` / `amm_info` (DEX queries)
+
+**Key files**:
+
+- New: `docker/telemetry/workload/rpc_load_generator.py`
+- New: `docker/telemetry/workload/requirements.txt`
+
+---
+
+## Task 10.3: Transaction Submitter
+
+**Objective**: Generate diverse transaction types to exercise `tx.*` and `ledger.*` spans.
+
+**What to do**:
+
+- Create `docker/telemetry/workload/tx_submitter.py`:
+  - Pre-funds test accounts from genesis account
+  - Submits a mix of transaction types:
+    - `Payment` (XRP and issued currencies) — exercises `tx.process`, `tx.apply`
+    - `OfferCreate` / `OfferCancel` — DEX activity
+    - `TrustSet` — trust line creation for issued currencies
+    - `NFTokenMint` / `NFTokenCreateOffer` / `NFTokenAcceptOffer` — NFT activity
+    - `EscrowCreate` / `EscrowFinish` — escrow lifecycle
+    - `AMMCreate` / `AMMDeposit` / `AMMWithdraw` — AMM pool operations (if amendment enabled)
+  - Configurable: TPS target, transaction mix weights, duration
+  - Monitors submission results and tracks success/failure rates
+
+- The transaction mix ensures the telemetry captures the full range of ledger activity that third parties care about.
+
+**Key files**:
+
+- New: `docker/telemetry/workload/tx_submitter.py`
+- New: `docker/telemetry/workload/test_accounts.json` (pre-generated keypairs)
+
+---
+
+## Task 10.4: Telemetry Validation Suite
+
+**Objective**: Automated scripts that verify all expected telemetry data exists after a workload run.
+
+**What to do**:
+
+- Create `docker/telemetry/workload/validate_telemetry.py`:
+
+  **Span validation** (queries Tempo API):
+  - Assert all required span names appear in traces (conditional spans — `grpc.*`,
+    `ledger.acquire`, `txq.*`, `consensus.mode_change` — are marked `optional` and
+    skipped when not exercised by the workload)
+  - Assert each span has its required attributes (bare/underscore keys per the
+    2026-05-13 span-attr naming redesign; dotted `xrpl.*` reserved for resource attrs)
+  - Assert parent-child relationships are correct (`rpc.ws_message` → `rpc.process` → `rpc.command.*`)
+  - Assert span durations are reasonable (> 0, < 60s)
+
+  **Metric validation** (queries Prometheus API):
+  - Assert all SpanMetrics-derived metrics are non-zero: `traces_span_metrics_calls_total`, `traces_span_metrics_duration_milliseconds_bucket`
+  - Assert all StatsD metrics are non-zero: `xrpld_LedgerMaster_Validated_Ledger_Age`, `xrpld_Peer_Finder_Active_*`, etc.
+  - Assert all Phase 9 metrics are non-zero: `xrpld_nodestore_*`, `xrpld_cache_*`, `xrpld_txq_*`, `xrpld_rpc_method_*`, `xrpld_object_count`, `xrpld_load_factor*`
+  - Assert metric label cardinality is within bounds
+
+  **Log-trace correlation validation** (queries Loki API):
+  - Assert logs contain `trace_id=` and `span_id=` fields
+  - Pick a random trace_id from Tempo → query Loki for matching logs → assert results exist
+  - Assert Grafana derived field links are functional
+
+  **Dashboard validation**:
+  - For each of the 10 Grafana dashboards, query the dashboard API and assert no panels show "No data"
+
+- Output: JSON report with pass/fail per check, suitable for CI.
+
+**Key files**:
+
+- New: `docker/telemetry/workload/validate_telemetry.py`
+- New: `docker/telemetry/workload/expected_spans.json` (span inventory for validation)
+- New: `docker/telemetry/workload/expected_metrics.json` (metric inventory for validation)
+
+---
+
+## Task 10.5: Performance Benchmark Suite
+
+**Objective**: Measure CPU/memory/latency overhead of the telemetry stack.
+
+**What to do**:
+
+- Create `docker/telemetry/workload/benchmark.sh`:
+  - **Baseline run**: Start cluster with `[telemetry] enabled=0`, run transaction workload for 5 minutes, record metrics
+  - **Telemetry run**: Start cluster with full telemetry enabled, run identical workload, record metrics
+  - **Comparison**: Calculate deltas for:
+    - CPU usage (per-node average)
+    - Memory RSS (per-node peak)
+    - RPC p99 latency
+    - Transaction throughput (TPS)
+    - Consensus round time p95
+    - Ledger close time p95
+
+- Output: Markdown table comparing baseline vs. telemetry, with pass/fail against targets:
+  - CPU overhead < 3%
+  - Memory overhead < 5MB
+  - RPC latency impact < 2ms p99
+  - Throughput impact < 5%
+  - Consensus impact < 1%
+
+- Store results in `docker/telemetry/workload/benchmark-results/` for historical tracking.
+
+**Key files**:
+
+- New: `docker/telemetry/workload/benchmark.sh`
+- New: `docker/telemetry/workload/collect_system_metrics.sh`
+
+---
+
+## Task 10.6: CI Integration
+
+**Objective**: Wire the validation suite into CI for regression detection.
+
+**What to do**:
+
+- Create a CI workflow (GitHub Actions or equivalent) that:
+  1. Builds xrpld with `-DXRPL_ENABLE_TELEMETRY=ON`
+  2. Starts the multi-node workload harness
+  3. Runs the RPC load generator + transaction submitter for 2 minutes
+  4. Runs the validation suite
+  5. Runs the benchmark suite
+  6. Fails the build if any validation check fails or benchmark exceeds thresholds
+  7. Archives the validation report and benchmark results as artifacts
+
+- This should be a separate workflow (not part of the main CI), triggered manually or on telemetry-related branch changes.
+
+**Key files**:
+
+- New: `.github/workflows/telemetry-validation.yml`
+- New: `docker/telemetry/workload/run-full-validation.sh` (orchestrator script)
+
+---
+
+## Task 10.7: Documentation
+
+**Objective**: Document the workload tools and validation process.
+
+**What to do**:
+
+- Create `docker/telemetry/workload/README.md`:
+  - Quick start guide for running workload harness
+  - Configuration options for load generator and tx submitter
+  - How to read validation reports
+  - How to run benchmarks and interpret results
+
+- Update `docs/telemetry-runbook.md`:
+  - Add "Validating Telemetry Stack" section
+  - Add "Performance Benchmarking" section
+
+- Update `OpenTelemetryPlan/09-data-collection-reference.md`:
+  - Add "Validation" section with expected metric/span counts
+
+---
+
+## Exit Criteria — Delivered in PR #6519
+
+- [x] Multi-node validator cluster starts and reaches consensus
+- [x] RPC load generator fires all traced RPC commands at configurable rates
+- [x] Transaction submitter generates 6+ transaction types at configurable TPS
+- [x] Validation suite confirms all required spans, attributes, and metrics
+- [x] Log-trace correlation validated end-to-end (Loki ↔ Tempo)
+- [x] Grafana dashboards render data (no empty panels)
+- [x] Overhead benchmark (`benchmark.sh`) measures telemetry-off vs telemetry-on deltas
+- [x] CI workflow runs validation on telemetry branch changes
+- [x] Validation report output is CI-parseable (JSON with exit codes)
+- [x] OTel-driven regression gate captures per-span/per-RPC/per-job timings from
+      Prometheus and compares against a committed baseline
+
+## Follow-up Work (tracked in separate PRs)
+
+- [ ] FU-2: Automate baseline persistence across CI runs (artifact uploaded
+      on merge to `develop`, downloaded on PR runs). Current mechanism
+      requires a manual baseline-refresh PR.
+- [ ] FU-4: Replace the proxy measurements in `benchmark.sh` (wall-clock curl
+      p99, ledger-cadence-as-TPS, ledger-cadence-as-consensus-p95) with
+      PromQL quantile queries from the same pipeline the regression gate uses.
+- [ ] FU-6: Grafana dashboard plotting historical baseline values keyed by
+      commit SHA, for triaging noisy regressions.

OpenTelemetryPlan/Phase11_taskList.md

@@ -0,0 +1,544 @@
+# Phase 11: Third-Party Data Collection Pipelines — Task List
+
+> **Status**: Future Enhancement
+>
+> **Goal**: Build a custom OTel Collector receiver that periodically polls xrpld's admin RPCs and exports structured metrics for external consumers — making all XRPL health, validator, peer, fee, and DEX data available as Prometheus/OTLP metrics without xrpld code changes.
+>
+> **Scope**: Go-based OTel Collector receiver plugin + Grafana dashboards + Prometheus alerting rules.
+>
+> **Branch**: `pratik/otel-phase11-third-party-collection` (from `pratik/otel-phase10-workload-validation`)
+>
+> **Depends on**: Phase 10 (validation harness for testing the new receiver)
+
+### Related Plan Documents
+
+| Document                                                             | Relevance                                                       |
+| -------------------------------------------------------------------- | --------------------------------------------------------------- |
+| [06-implementation-phases.md](./06-implementation-phases.md)         | Phase 11 plan: motivation, architecture, exit criteria (§6.8.4) |
+| [09-data-collection-reference.md](./09-data-collection-reference.md) | Defines full metric inventory including third-party metrics     |
+| [Phase10_taskList.md](./Phase10_taskList.md)                         | Prerequisite — validation harness for testing                   |
+
+### Third-Party Consumer Gap Analysis
+
+This phase addresses the cross-cutting gap identified during research: **xrpld has no native Prometheus/OTLP metrics export for data accessible only via RPC**. Every consumer (exchanges, payment processors, analytics providers, validators, researchers, compliance firms, custodians) must build custom JSON-RPC polling and conversion. This receiver centralizes that work.
+
+| Consumer Category          | Data Unlocked by This Phase                                        |
+| -------------------------- | ------------------------------------------------------------------ |
+| **Exchanges**              | Real-time fee estimates, TxQ capacity, server health scores        |
+| **Payment Processors**     | Settlement latency percentiles, corridor health, path availability |
+| **Analytics Providers**    | Validator metrics, network topology, amendment voting status       |
+| **DeFi / AMM**             | AMM pool TVL, DEX order book depth, trade volumes                  |
+| **Validators / Operators** | Per-peer latency, version distribution, UNL health, alerting       |
+| **Compliance**             | Transaction volume trends, network growth metrics                  |
+| **Academic Researchers**   | Consensus performance time-series, decentralization metrics        |
+| **CBDC / Tokenization**    | Token supply tracking, trust line adoption, freeze status          |
+| **Institutional Custody**  | Multi-sig status, escrow tracking, reserve calculations            |
+| **Wallet Providers**       | Server health for node selection, fee prediction data              |
+
+---
+
+## Task 11.1: OTel Collector Receiver Scaffold
+
+**Objective**: Create the Go project structure for a custom OTel Collector receiver that polls xrpld JSON-RPC.
+
+**What to do**:
+
+- Create `docker/telemetry/otel-rippled-receiver/`:
+  - `receiver.go` — implements `receiver.Metrics` interface
+  - `config.go` — configuration struct (endpoint, poll interval, enabled RPCs)
+  - `factory.go` — receiver factory registration
+  - `go.mod` / `go.sum` — Go module with OTel Collector SDK dependency
+
+- Configuration model:
+
+  ```yaml
+  xrpld_receiver:
+    endpoint: "http://localhost:5005" # xrpld admin RPC
+    poll_interval: 30s # how often to poll
+    enabled_collectors:
+      - server_info
+      - get_counts
+      - fee
+      - peers
+      - validators
+      - feature
+      - server_state
+    amm_pools: [] # optional: AMM pool IDs to track
+    book_offers_pairs: [] # optional: currency pairs for DEX depth
+  ```
+
+- Build a custom OTel Collector binary that includes this receiver alongside the standard receivers.
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/receiver.go`
+- New: `docker/telemetry/otel-rippled-receiver/config.go`
+- New: `docker/telemetry/otel-rippled-receiver/factory.go`
+- New: `docker/telemetry/otel-rippled-receiver/go.mod`
+- New: `docker/telemetry/otel-rippled-receiver/Dockerfile`
+
+---
+
+## Task 11.2: server_info / server_state Collector
+
+**Objective**: Poll `server_info` and `server_state` and export all fields as OTel metrics.
+
+**What to do**:
+
+- Implement `serverInfoCollector` that calls `server_info` (admin) and extracts:
+
+  **Node Health Gauges:**
+  - `xrpl_server_state` (enum → int: disconnected=0, connected=1, syncing=2, tracking=3, full=4, proposing=5)
+  - `xrpl_server_state_duration_seconds`
+  - `xrpl_uptime_seconds`
+  - `xrpl_io_latency_ms`
+  - `xrpl_amendment_blocked` (0 or 1)
+  - `xrpl_peers_count`
+  - `xrpl_peer_disconnects_total`
+  - `xrpl_peer_disconnects_resources_total`
+  - `xrpl_jq_trans_overflow_total`
+
+  **Consensus Gauges:**
+  - `xrpl_last_close_proposers`
+  - `xrpl_last_close_converge_time_seconds`
+  - `xrpl_validation_quorum`
+
+  **Ledger Gauges:**
+  - `xrpl_validated_ledger_seq`
+  - `xrpl_validated_ledger_age_seconds`
+  - `xrpl_validated_ledger_base_fee_drops`
+  - `xrpl_validated_ledger_reserve_base_drops`
+  - `xrpl_validated_ledger_reserve_inc_drops`
+  - `xrpl_close_time_offset_seconds` (0 when absent)
+
+  **Load Factor Gauges:**
+  - `xrpl_load_factor`
+  - `xrpl_load_factor_server`
+  - `xrpl_load_factor_fee_escalation`
+  - `xrpl_load_factor_fee_queue`
+  - `xrpl_load_factor_local`
+  - `xrpl_load_factor_net`
+  - `xrpl_load_factor_cluster`
+
+  **State Accounting Gauges** (per state: disconnected, connected, syncing, tracking, full):
+  - `xrpl_state_duration_seconds{state="<name>"}`
+  - `xrpl_state_transitions_total{state="<name>"}`
+
+  **Validator Info** (when node is a validator):
+  - `xrpl_validator_list_count`
+  - `xrpl_validator_list_expiration_seconds` (epoch)
+  - `xrpl_validator_list_active` (0 or 1)
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/collectors/server_info.go`
+
+---
+
+## Task 11.3: get_counts Collector
+
+**Objective**: Poll `get_counts` and export internal object counts and NodeStore stats.
+
+**What to do**:
+
+- Implement `getCountsCollector`:
+
+  **Database Gauges:**
+  - `xrpl_db_size_kb{db="total"}`, `xrpl_db_size_kb{db="ledger"}`, `xrpl_db_size_kb{db="transaction"}`
+
+  **NodeStore Gauges:**
+  - `xrpl_nodestore_reads_total`, `xrpl_nodestore_reads_hit`, `xrpl_nodestore_writes_total`
+  - `xrpl_nodestore_read_bytes`, `xrpl_nodestore_written_bytes`
+  - `xrpl_nodestore_read_duration_us`, `xrpl_nodestore_write_load`
+  - `xrpl_nodestore_read_queue`, `xrpl_nodestore_read_threads_running`
+
+  **Cache Gauges:**
+  - `xrpl_cache_hit_rate{cache="SLE"}`, `xrpl_cache_hit_rate{cache="ledger"}`, `xrpl_cache_hit_rate{cache="accepted_ledger"}`
+  - `xrpl_cache_size{cache="treenode"}`, `xrpl_cache_size{cache="fullbelow"}`, `xrpl_cache_size{cache="accepted_ledger"}`
+
+  **Object Count Gauges:**
+  - `xrpl_object_count{type="<name>"}` for each counted object type (Transaction, Ledger, NodeObject, STTx, STLedgerEntry, InboundLedger, Pathfinder, etc.)
+
+  **Rates:**
+  - `xrpl_historical_fetch_per_minute`
+  - `xrpl_local_txs`
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/collectors/get_counts.go`
+
+---
+
+## Task 11.4: Peer Topology Collector
+
+**Objective**: Poll `peers` and export per-peer and aggregate network metrics.
+
+**What to do**:
+
+- Implement `peersCollector`:
+
+  **Aggregate Gauges:**
+  - `xrpl_peers_inbound_count`
+  - `xrpl_peers_outbound_count`
+  - `xrpl_peers_cluster_count`
+
+  **Per-Peer Gauges** (with labels `peer_key` truncated to 8 chars for cardinality control):
+  - `xrpl_peer_latency_ms{peer="<key>", version="<ver>", inbound="<bool>"}`
+  - `xrpl_peer_uptime_seconds{peer="<key>"}`
+  - `xrpl_peer_load{peer="<key>"}`
+
+  **Distribution Gauges** (aggregated across all peers):
+  - `xrpl_peer_latency_p50_ms`, `xrpl_peer_latency_p95_ms`, `xrpl_peer_latency_p99_ms`
+  - `xrpl_peer_version_count{version="<semver>"}` — count of peers per software version
+
+  **Tracking Status:**
+  - `xrpl_peer_diverged_count` — peers with `track=diverged`
+  - `xrpl_peer_unknown_count` — peers with `track=unknown`
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/collectors/peers.go`
+
+**Cardinality note**: Per-peer metrics use truncated keys. For large peer sets (50+), the aggregate distribution gauges are preferred over per-peer labels.
+
+---
+
+## Task 11.5: Validator & Amendment Collector
+
+**Objective**: Poll `validators` and `feature` to export validator health and amendment voting status.
+
+**What to do**:
+
+- Implement `validatorCollector`:
+
+  **From `validators` RPC:**
+  - `xrpl_trusted_validators_count`
+  - `xrpl_validator_signing` (0 or 1 — whether local validator is signing)
+
+  **From `feature` RPC:**
+  - `xrpl_amendment_enabled_count` — total enabled amendments
+  - `xrpl_amendment_majority_count` — amendments with majority but not yet enabled
+  - `xrpl_amendment_vetoed_count` — locally vetoed amendments
+  - `xrpl_amendment_unsupported_majority` (0 or 1) — any unsupported amendment has majority (critical alert)
+
+  **Per-amendment with majority** (limited cardinality — only amendments with `majority` set):
+  - `xrpl_amendment_majority_time{name="<amendment>"}` — epoch time when majority was gained
+  - `xrpl_amendment_votes{name="<amendment>"}` — current vote count
+  - `xrpl_amendment_threshold{name="<amendment>"}` — votes needed
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/collectors/validators.go`
+
+---
+
+## Task 11.6: Fee & TxQ Collector
+
+**Objective**: Poll `fee` RPC and export real-time fee market data.
+
+**What to do**:
+
+- Implement `feeCollector` that calls the public `fee` RPC:
+
+  **Fee Level Gauges:**
+  - `xrpl_fee_current_ledger_size` — transactions in current open ledger
+  - `xrpl_fee_expected_ledger_size` — expected transactions at close
+  - `xrpl_fee_max_queue_size` — maximum transaction queue size
+  - `xrpl_fee_open_ledger_fee_drops` — minimum fee for open ledger inclusion
+  - `xrpl_fee_median_fee_drops` — median fee level
+  - `xrpl_fee_minimum_fee_drops` — base reference fee
+  - `xrpl_fee_queue_size` — current queue depth
+
+- This overlaps with Phase 9's internal TxQ metrics but provides an external-only collection path that doesn't require xrpld code changes.
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/collectors/fee.go`
+
+---
+
+## Task 11.7: DEX & AMM Collector (Optional)
+
+**Objective**: Periodically poll configured AMM pools and order book pairs for DeFi metrics.
+
+**What to do**:
+
+- Implement `dexCollector` (enabled only when `amm_pools` or `book_offers_pairs` are configured):
+
+  **AMM Pool Gauges** (per configured pool):
+  - `xrpl_amm_reserve{pool="<id>", asset="<currency>"}` — pool reserve amount
+  - `xrpl_amm_lp_token_supply{pool="<id>"}` — outstanding LP tokens
+  - `xrpl_amm_trading_fee{pool="<id>"}` — pool trading fee (basis points)
+  - `xrpl_amm_tvl_drops{pool="<id>"}` — total value locked (XRP-denominated)
+
+  **Order Book Gauges** (per configured pair):
+  - `xrpl_orderbook_bid_depth{pair="<base>/<quote>"}` — total bid volume
+  - `xrpl_orderbook_ask_depth{pair="<base>/<quote>"}` — total ask volume
+  - `xrpl_orderbook_spread{pair="<base>/<quote>"}` — best bid-ask spread
+  - `xrpl_orderbook_offer_count{pair="<base>/<quote>", side="bid|ask"}` — number of offers
+
+**Key files**:
+
+- New: `docker/telemetry/otel-rippled-receiver/collectors/dex.go`
+
+**Note**: This is optional because it requires explicit configuration of which pools/pairs to track. Default configuration tracks no DEX data.
+
+---
+
+## Task 11.8: Prometheus Alerting Rules
+
+**Objective**: Create production-ready alerting rules for the metrics exported by this receiver.
+
+**What to do**:
+
+- Create `docker/telemetry/prometheus/rippled-alerts.yml`:
+
+  **Tier 1 — Critical (page immediately):**
+
+  ```yaml
+  - alert: XRPLServerNotFull
+    expr: xrpl_server_state < 4
+    for: 15m
+
+  - alert: XRPLAmendmentBlocked
+    expr: xrpl_amendment_blocked == 1
+    for: 1m
+
+  - alert: XRPLNoPeers
+    expr: xrpl_peers_count == 0
+    for: 5m
+
+  - alert: XRPLLedgerStale
+    expr: xrpl_validated_ledger_age_seconds > 120
+    for: 2m
+
+  - alert: XRPLHighIOLatency
+    expr: xrpl_io_latency_ms > 100
+    for: 5m
+
+  - alert: XRPLUnsupportedAmendmentMajority
+    expr: xrpl_amendment_unsupported_majority == 1
+    for: 1m
+  ```
+
+  **Tier 2 — Warning (investigate within hours):**
+
+  ```yaml
+  - alert: XRPLLowPeerCount
+    expr: xrpl_peers_count < 10
+    for: 15m
+
+  - alert: XRPLHighLoadFactor
+    expr: xrpl_load_factor > 10
+    for: 10m
+
+  - alert: XRPLSlowConsensus
+    expr: xrpl_last_close_converge_time_seconds > 6
+    for: 5m
+
+  - alert: XRPLValidatorListExpiring
+    expr: (xrpl_validator_list_expiration_seconds - time()) < 86400
+    for: 1h
+
+  - alert: XRPLClockDrift
+    expr: xrpl_close_time_offset_seconds > 0
+    for: 5m
+
+  - alert: XRPLStateFlapping
+    expr: rate(xrpl_state_transitions_total{state="full"}[1h]) > 2
+    for: 30m
+  ```
+
+**Key files**:
+
+- New: `docker/telemetry/prometheus/rippled-alerts.yml`
+- Update: `docker/telemetry/prometheus/prometheus.yml` (add rule_files reference)
+
+---
+
+## Task 11.9: New Grafana Dashboards
+
+**Objective**: Create 4 new dashboards for the data exported by the receiver.
+
+**What to do**:
+
+- **Validator Health** (`validator-health`):
+  - Server state timeline, state duration breakdown
+  - Proposer count trend, converge time trend, validation quorum
+  - Validator list expiration countdown
+  - Amendment voting status (majority/enabled/vetoed)
+
+- **Network Topology** (`xrpld-network-topology`):
+  - Peer count (inbound/outbound/cluster), peer version distribution
+  - Peer latency distribution (p50/p95/p99), diverged peer count
+  - Geographic distribution (if enriched with GeoIP)
+  - Peer uptime distribution
+
+- **Fee Market** (`xrpld-fee-market-external`):
+  - Current fee levels (open ledger, median, minimum), fee escalation timeline
+  - Queue depth vs. capacity, transactions per ledger
+  - Load factor breakdown (server/network/cluster/escalation)
+
+- **DEX & AMM Overview** (`xrpld-dex-amm`) (only populated when DEX collectors are configured):
+  - AMM pool TVL, reserve ratios, LP token supply
+  - Order book depth per pair, spread trends
+  - Trading fee revenue estimates
+
+**Key files**:
+
+- New: `docker/telemetry/grafana/dashboards/rippled-validator-health.json`
+- New: `docker/telemetry/grafana/dashboards/rippled-network-topology.json`
+- New: `docker/telemetry/grafana/dashboards/rippled-fee-market-external.json`
+- New: `docker/telemetry/grafana/dashboards/rippled-dex-amm.json`
+
+---
+
+## Task 11.10: Integration with Phase 10 Validation
+
+**Objective**: Extend the Phase 10 validation suite to verify this receiver's metrics.
+
+**What to do**:
+
+- Update `docker/telemetry/workload/validate_telemetry.py`:
+  - Add assertions for all `xrpl_*` metrics produced by the receiver
+  - Verify metric labels have expected values
+  - Verify alerting rules fire correctly (inject a "bad" state and check alert)
+
+- Update `docker/telemetry/docker-compose.workload.yaml`:
+  - Add the custom OTel Collector build with the xrpld receiver
+  - Configure the receiver to poll one of the test nodes
+
+**Key files**:
+
+- Update: `docker/telemetry/workload/validate_telemetry.py`
+- Update: `docker/telemetry/docker-compose.workload.yaml`
+- Update: `docker/telemetry/workload/expected_metrics.json`
+
+---
+
+## Task 11.11: Documentation
+
+**Objective**: Document the receiver, its metrics, deployment, and alerting.
+
+**What to do**:
+
+- Create `docker/telemetry/otel-rippled-receiver/README.md`:
+  - Architecture overview (how the receiver fits into the OTel Collector)
+  - Configuration reference (all config options with defaults)
+  - Metric reference table (all exported metrics with types and labels)
+  - Deployment guide (building custom collector binary, docker-compose integration)
+
+- Update `OpenTelemetryPlan/09-data-collection-reference.md`:
+  - Add "Third-Party Metrics (OTel Collector Receiver)" section
+  - Add new Grafana dashboard reference (4 dashboards)
+  - Add alerting rules reference
+
+- Update `docs/telemetry-runbook.md`:
+  - Add "Third-Party Metrics Receiver" troubleshooting section
+  - Add alerting playbook (what to do for each Tier 1/Tier 2 alert)
+
+---
+
+## Task 11.12: Alert Rules for External Dashboard Parity Metrics
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — 18 alert rules ported from the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
+>
+> **Upstream**: Phase 7 Tasks 7.9-7.16 (metrics), Phase 9 Tasks 9.11-9.13 (dashboards).
+> **Downstream**: None — terminal task in the parity chain.
+
+**Objective**: Add Grafana alerting rules for the Phase 7+ parity metrics (validation agreement, validator health, peer quality, state tracking, ledger economy). These complement Task 11.8's `xrpl_*` alerts by covering the `xrpld_*` internal metrics.
+
+**Critical Group** (8 rules, eval interval 10s):
+
+| Rule                | Condition                                                     | For |
+| ------------------- | ------------------------------------------------------------- | --- |
+| Agreement Below 90% | `xrpld_validation_agreement{metric="agreement_pct_24h"} < 90` | 30s |
+| Not Proposing       | `xrpld_state_tracking{metric="state_value"} < 6`              | 10s |
+| Unhealthy State     | `xrpld_state_tracking{metric="state_value"} < 4`              | 10s |
+| Amendment Blocked   | `xrpld_validator_health{metric="amendment_blocked"} == 1`     | 1m  |
+| UNL Expiring        | `xrpld_validator_health{metric="unl_expiry_days"} < 14`       | 1h  |
+| High IO Latency     | `histogram_quantile(0.95, xrpld_ios_latency_bucket) > 50`     | 1m  |
+| High Load Factor    | `xrpld_load_factor_metrics{metric="load_factor"} > 1000`      | 1m  |
+| Peer Count Critical | `xrpld_server_info{metric="peers"} < 5`                       | 1m  |
+
+**Network Group** (3 rules, eval interval 10s):
+
+| Rule                      | Condition                                                         | For |
+| ------------------------- | ----------------------------------------------------------------- | --- |
+| Peer Drop >10%            | `delta(xrpld_server_info{metric="peers"}[30s]) / ... * 100 < -10` | 30s |
+| Peer Drop >30%            | Same formula, threshold -30                                       | 30s |
+| P90 Latency + Disconnects | `peer_latency_p90_ms > 500 AND rate(disconnects) > 0`             | 2m  |
+
+**Performance Group** (7 rules, eval interval 10s):
+
+| Rule                | Condition                                                    | For |
+| ------------------- | ------------------------------------------------------------ | --- |
+| CPU High            | Per-core CPU > 80% (requires node_exporter)                  | 2m  |
+| Memory Critical     | Memory usage > 90% (requires node_exporter)                  | 1m  |
+| Disk Warning        | Disk usage > 85% (requires node_exporter)                    | 2m  |
+| Job Queue Overflow  | `rate(xrpld_jq_trans_overflow_total[5m]) > 0`                | 1m  |
+| Upgrade Recommended | `xrpld_peer_quality{metric="peers_higher_version_pct"} > 60` | 1m  |
+| TX Rate Drop        | Transaction rate dropped > 50% in 5m window                  | 5m  |
+| Stale Ledger        | `xrpld_ledger_economy{metric="ledger_age_seconds"} > 30`     | 1m  |
+
+**Notification channel templates**: Email/SMTP, Discord, Slack, PagerDuty.
+
+**Key files**:
+
+- New/extend: `docker/telemetry/grafana/alerting/alert-rules-parity.yaml`
+- New: `docker/telemetry/grafana/alerting/contact-points.yaml` (template configs)
+- New: `docker/telemetry/grafana/alerting/notification-policies.yaml`
+
+**Exit Criteria**:
+
+- [ ] All 18 rules evaluate without errors in Grafana alerting UI
+- [ ] Critical rules fire within expected timeframe when conditions are met
+- [ ] Notification channel templates are documented (not hard-coded to any service)
+
+---
+
+## Task 11.13: Dual-Datasource Architecture Documentation
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
+
+**Objective**: Document the external dashboard's "fast path" pattern as a future optimization for real-time panels.
+
+**Pattern**: A lightweight Prometheus scrape endpoint (separate from OTLP pipeline) that polls critical metrics every 2-5s, bypassing the 10s OTLP metric reader interval and Prometheus scrape interval.
+
+**Use case**: Real-time state panels (server state, ledger age, peer count) where 10-15s latency is too slow for operational dashboards.
+
+**Decision**: Document as a future option, not implement now. The current 10s interval is acceptable for v1. The external dashboard achieves 2-5s freshness by polling RPC directly, which is what the Phase 11 receiver already does. Adding a separate scrape endpoint to xrpld would only be needed if sub-second metric freshness is required from the internal metrics pipeline.
+
+**What to document**:
+
+- Architecture comparison: OTLP pipeline (10-15s) vs. direct scrape (2-5s) vs. push gateway
+- When to consider: operator feedback indicating 10s is insufficient for alerting SLOs
+- How to implement if needed: add `/metrics` HTTP endpoint to xrpld with Prometheus client library
+- Trade-offs: additional port, additional dependency, duplication with OTLP metrics
+
+**Key files**:
+
+- Update: `OpenTelemetryPlan/09-data-collection-reference.md` (add "Future: Dual-Datasource Architecture" section)
+- Update: `docs/telemetry-runbook.md` (add brief note in performance tuning section)
+
+**Exit Criteria**:
+
+- [ ] Architecture comparison documented with clear trade-offs
+- [ ] Decision rationale recorded (why deferred, when to revisit)
+
+---
+
+## Exit Criteria
+
+- [ ] Custom OTel Collector receiver builds and starts without errors
+- [ ] All `xrpl_*` metrics from server_info, get_counts, peers, validators, fee appear in Prometheus
+- [ ] Metrics update at configured poll interval (default 30s)
+- [ ] 4 new Grafana dashboards operational with data
+- [ ] Prometheus alerting rules fire correctly for simulated failure conditions
+- [ ] DEX/AMM collector works when configured (optional — not required for base exit criteria)
+- [ ] Phase 10 validation suite passes with receiver metrics included
+- [ ] Receiver handles xrpld restart/unavailability gracefully (no crash, logs warning, retries)
+- [ ] Documentation complete: receiver README, metric reference, alerting playbook
+- [ ] Go receiver has unit tests with >80% coverage
+- [ ] 18 Grafana alert rules for Phase 7+ parity metrics evaluate correctly (Task 11.12)
+- [ ] Dual-datasource architecture documented with trade-offs (Task 11.13)

OpenTelemetryPlan/Phase2_taskList.md

@@ -205,3 +205,36 @@ _Note: `pathfind_dest_amount` was removed — the destination amount is a financ
 **Deferred with rationale**: Tasks 2.1 (→Phase 3), 2.5 (low priority).
 **Dropped**: Task 2.8 (node health not duplicated on traces).
 **Superseded**: Task 2.2 (Phase 1c SpanGuard factory covers this).
+
+---
+
+## Known Issues / Future Work
+
+### Thread safety of TelemetryImpl::stop() vs startSpan()
+
+`TelemetryImpl::stop()` resets `sdkProvider_` (a `std::shared_ptr`) without
+synchronization. `getTracer()` reads the same member from RPC handler threads.
+This is a data race if any thread calls `startSpan()` concurrently with `stop()`.
+
+**Current mitigation**: `Application::stop()` shuts down `serverHandler_`,
+`overlay_`, and `jobQueue_` before calling `telemetry_->stop()`, so no callers
+remain. See comments in `Telemetry.cpp:stop()` and `Application.cpp`.
+
+**TODO**: Add an `std::atomic<bool> stopped_` flag checked in `getTracer()` to
+make this robust against future shutdown order changes.
+
+### Macro incompatibility: XRPL_TRACE_SPAN vs XRPL_TRACE_SET_ATTR
+
+`XRPL_TRACE_SPAN` and `XRPL_TRACE_SPAN_KIND` declare `_xrpl_guard_` as a bare
+`SpanGuard`, but `XRPL_TRACE_SET_ATTR` and `XRPL_TRACE_EXCEPTION` call
+`_xrpl_guard_.has_value()` which requires `std::optional<SpanGuard>`. Using
+`XRPL_TRACE_SPAN` followed by `XRPL_TRACE_SET_ATTR` in the same scope would
+fail to compile.
+
+**Current mitigation**: No call site currently uses `XRPL_TRACE_SPAN` — all
+production code uses the conditional macros (`XRPL_TRACE_RPC`, `XRPL_TRACE_TX`,
+etc.) which correctly wrap the guard in `std::optional`.
+
+**TODO**: Either make `XRPL_TRACE_SPAN`/`XRPL_TRACE_SPAN_KIND` also wrap in
+`std::optional`, or document that `XRPL_TRACE_SET_ATTR` is only compatible with
+the conditional macros.

OpenTelemetryPlan/Phase3_taskList.md

@@ -252,7 +252,7 @@ design.
 
 ## Task 3.8: Transaction Span Peer Version Attribute
 
-> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md) — adds peer version context inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — adds peer version context inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
 >
 > **Upstream**: Phase 2 (RPC span infrastructure must exist).
 > **Downstream**: Phase 10 (validation checks for this attribute).
@@ -282,7 +282,7 @@ design.
 
 - [ ] `tx.receive` spans carry `peer_version` attribute with a non-empty version string
 - [ ] Attribute is omitted (not set to empty string) when `getVersion()` returns empty
-- [ ] Attribute visible in Jaeger span detail view
+- [ ] Attribute visible in Tempo trace detail view
 
 ---
 
@@ -535,3 +535,14 @@ This gives the best of both worlds: guaranteed cross-node correlation via determ
 - [ ] <5% overhead on transaction throughput
 - [x] Deterministic trace_id: same trace_id for same tx across all nodes
 - [x] Protobuf span_id propagation preserves parent-child ordering when available
+
+---
+
+## Known Issues / Future Work
+
+### Unused trace_state proto field
+
+The `TraceContext.trace_state` field (field 4) in `xrpl.proto` is reserved for
+W3C `tracestate` vendor-specific key-value pairs but is not read or written by
+`TraceContextPropagator`. Wire it when cross-vendor trace propagation is needed.
+No wire cost since proto `optional` fields are zero-cost when absent.

OpenTelemetryPlan/Phase4_taskList.md

@@ -218,7 +218,7 @@
 
 ## Task 4.8: Consensus Validation Span Enrichment — NOT DONE
 
-> **Source**: [External Dashboard Parity](../docs/superpowers/specs/2026-03-30-external-dashboard-parity-design.md) — adds validation agreement context inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — adds validation agreement context inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
 >
 > **Upstream**: Phase 4 tasks 4.1-4.4 (span creation must exist).
 > **Downstream**: Phase 7 (ValidationTracker reads these attributes), Phase 10 (validation checks).
@@ -379,7 +379,7 @@ Two strategies for cross-node trace correlation, switchable via config:
 Derive `trace_id = SHA256(previousLedger.id())[0:16]` so all nodes in the same
 consensus round share the same trace_id without P2P context propagation.
 
-- **Pros**: All nodes appear in the same trace in Tempo/Jaeger automatically.
+- **Pros**: All nodes appear in the same trace in Tempo automatically.
   No collector-side post-processing needed.
 - **Cons**: Overrides OTel's random trace_id generation; requires custom
   `IdGenerator` or manual span context construction.
@@ -926,7 +926,7 @@ Received messages use **span links** (follows-from), NOT parent-child:
 
 - The receiver's processing span links to the sender's context
 - This preserves each node's independent trace tree
-- Cross-node correlation visible via linked traces in Tempo/Jaeger
+- Cross-node correlation visible via linked traces in Tempo
 
 ## Interaction with Deterministic Trace ID (Strategy A)
 

OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md

@@ -0,0 +1,221 @@
+# Phase 5: Integration Test Task List
+
+> **Goal**: End-to-end verification of the complete telemetry pipeline using a
+> 6-node consensus network. Proves that RPC, transaction, and consensus spans
+> flow through the observability stack (otel-collector, Tempo, Prometheus,
+> Grafana) under realistic conditions.
+>
+> **Scope**: Integration test script, manual testing plan, 6-node local network
+> setup, Tempo/Prometheus/Grafana verification.
+>
+> **Branch**: `pratik/otel-phase5-docs-deployment`
+
+### Related Plan Documents
+
+| Document                                                         | Relevance                                  |
+| ---------------------------------------------------------------- | ------------------------------------------ |
+| [07-observability-backends.md](./07-observability-backends.md)   | Tempo, Grafana, Prometheus setup           |
+| [05-configuration-reference.md](./05-configuration-reference.md) | Collector config, Docker Compose           |
+| [06-implementation-phases.md](./06-implementation-phases.md)     | Phase 5 tasks, definition of done          |
+| [Phase5_taskList.md](./Phase5_taskList.md)                       | Phase 5 main task list (5.6 = integration) |
+
+---
+
+## Task IT.1: Create Integration Test Script
+
+**Objective**: Automated bash script that stands up a 6-node xrpld network
+with telemetry, exercises all span categories, and verifies data in
+Tempo/Prometheus.
+
+**What to do**:
+
+- Create `docker/telemetry/integration-test.sh`:
+  - Prerequisites check (docker, xrpld binary, curl, jq)
+  - Start observability stack via `docker compose`
+  - Generate 6 validator key pairs via temp standalone xrpld
+  - Generate 6 node configs + shared `validators.txt`
+  - Start 6 xrpld nodes in consensus mode (`--start`, no `-a`)
+  - Wait for all nodes to reach `"proposing"` state (120s timeout)
+
+**Key new file**: `docker/telemetry/integration-test.sh`
+
+**Verification**:
+
+- [ ] Script starts without errors
+- [ ] All 6 nodes reach "proposing" state
+- [ ] Observability stack is healthy (otel-collector, Tempo, Prometheus, Grafana)
+
+---
+
+## Task IT.2: RPC Span Verification (Phase 2)
+
+**Objective**: Verify RPC spans flow through the telemetry pipeline.
+
+**What to do**:
+
+- Send `server_info`, `server_state`, `ledger` RPCs to node1 (port 5005)
+- Wait for batch export (5s)
+- Query Tempo API for:
+  - `rpc.request` spans (ServerHandler::onRequest)
+  - `rpc.process` spans (ServerHandler::processRequest)
+  - `rpc.command.server_info` spans (callMethod)
+  - `rpc.command.server_state` spans (callMethod)
+  - `rpc.command.ledger` spans (callMethod)
+- Verify `command` attribute present on `rpc.command.*` spans
+
+**Verification**:
+
+- [ ] Tempo shows `rpc.request` traces
+- [ ] Tempo shows `rpc.process` traces
+- [ ] Tempo shows `rpc.command.*` traces with correct attributes
+
+---
+
+## Task IT.3: Transaction Span Verification (Phase 3)
+
+**Objective**: Verify transaction spans flow through the telemetry pipeline.
+
+**What to do**:
+
+- Get genesis account sequence via `account_info` RPC
+- Submit Payment transaction using genesis seed (`snoPBrXtMeMyMHUVTgbuqAfg1SUTb`)
+- Wait for consensus inclusion (10s)
+- Query Tempo API for:
+  - `tx.process` spans (NetworkOPsImp::processTransaction) on submitting node
+  - `tx.receive` spans (PeerImp::handleTransaction) on peer nodes
+- Verify `xrpl.tx.hash` attribute on `tx.process` spans
+- Verify `xrpl.peer.id` attribute on `tx.receive` spans
+
+**Verification**:
+
+- [ ] Tempo shows `tx.process` traces with `xrpl.tx.hash`
+- [ ] Tempo shows `tx.receive` traces with `xrpl.peer.id`
+
+---
+
+## Task IT.4: Consensus Span Verification (Phase 4)
+
+**Objective**: Verify consensus spans flow through the telemetry pipeline.
+
+**What to do**:
+
+- Consensus runs automatically in 6-node network
+- Query Tempo API for:
+  - `consensus.proposal.send` (Adaptor::propose)
+  - `consensus.ledger_close` (Adaptor::onClose)
+  - `consensus.accept` (Adaptor::onAccept)
+  - `consensus.validation.send` (Adaptor::validate)
+- Verify attributes:
+  - `xrpl.consensus.mode` on `consensus.ledger_close`
+  - `proposers` on `consensus.accept`
+  - `xrpl.consensus.ledger.seq` on `consensus.validation.send`
+
+**Verification**:
+
+- [ ] Tempo shows `consensus.ledger_close` traces with `xrpl.consensus.mode`
+- [ ] Tempo shows `consensus.accept` traces with `proposers`
+- [ ] Tempo shows `consensus.proposal.send` traces
+- [ ] Tempo shows `consensus.validation.send` traces
+
+---
+
+## Task IT.5: Spanmetrics Verification (Phase 5)
+
+**Objective**: Verify spanmetrics connector derives RED metrics from spans.
+
+**What to do**:
+
+- Query Prometheus for `traces_span_metrics_calls_total`
+- Query Prometheus for `traces_span_metrics_duration_milliseconds_count`
+- Verify Grafana loads at `http://localhost:3000`
+
+**Verification**:
+
+- [ ] Prometheus returns non-empty results for `traces_span_metrics_calls_total`
+- [ ] Prometheus returns non-empty results for duration histogram
+- [ ] Grafana UI accessible with dashboards visible
+
+---
+
+## Task IT.6: Manual Testing Plan
+
+**Objective**: Document how to run tests manually for future reference.
+
+**What to do**:
+
+- Create `docker/telemetry/TESTING.md` with:
+  - Prerequisites section
+  - Single-node standalone test (quick verification)
+  - 6-node consensus test (full verification)
+  - Expected span catalog (all 11 span names with attributes)
+  - Verification queries (Tempo API, Prometheus API)
+  - Troubleshooting guide
+
+**Key new file**: `docker/telemetry/TESTING.md`
+
+**Verification**:
+
+- [ ] Document covers both single-node and multi-node testing
+- [ ] All 11 span names documented with source file and attributes
+- [ ] Troubleshooting section covers common failure modes
+
+---
+
+## Task IT.7: Run and Verify
+
+**Objective**: Execute the integration test and validate results.
+
+**What to do**:
+
+- Run `docker/telemetry/integration-test.sh` locally
+- Debug any failures
+- Leave stack running for manual verification
+- Share URLs:
+  - Tempo: `http://localhost:3200`
+  - Grafana: `http://localhost:3000`
+  - Prometheus: `http://localhost:9090`
+
+**Verification**:
+
+- [ ] Script completes with all checks passing
+- [ ] Tempo UI shows xrpld service with all expected span names
+- [ ] Grafana dashboards load and show data
+
+---
+
+## Task IT.8: Commit
+
+**Objective**: Commit all new files to Phase 5 branch.
+
+**What to do**:
+
+- Run `pcc` (pre-commit checks)
+- Commit 3 new files to `pratik/otel-phase5-docs-deployment`
+
+**Verification**:
+
+- [ ] `pcc` passes
+- [ ] Commit created on Phase 5 branch
+
+---
+
+## Summary
+
+| Task | Description                   | New Files | Depends On |
+| ---- | ----------------------------- | --------- | ---------- |
+| IT.1 | Integration test script       | 1         | Phase 5    |
+| IT.2 | RPC span verification         | 0         | IT.1       |
+| IT.3 | Transaction span verification | 0         | IT.1       |
+| IT.4 | Consensus span verification   | 0         | IT.1       |
+| IT.5 | Spanmetrics verification      | 0         | IT.1       |
+| IT.6 | Manual testing plan           | 1         | --         |
+| IT.7 | Run and verify                | 0         | IT.1-IT.6  |
+| IT.8 | Commit                        | 0         | IT.7       |
+
+**Exit Criteria**:
+
+- [ ] All 6 xrpld nodes reach "proposing" state
+- [ ] All 11 expected span names visible in Tempo
+- [ ] Spanmetrics available in Prometheus
+- [ ] Grafana dashboards show data
+- [ ] Manual testing plan document complete

OpenTelemetryPlan/Phase7_taskList.md

@@ -0,0 +1,587 @@
+# Phase 7: Native OTel Metrics Migration — Task List
+
+> **Goal**: Replace `StatsDCollector` with a native OpenTelemetry Metrics SDK implementation behind the existing `beast::insight::Collector` interface, eliminating the StatsD UDP dependency.
+>
+> **Scope**: New `OTelCollectorImpl` class, `CollectorManager` config change, OTel Collector pipeline update, Grafana dashboard metric name migration, integration tests.
+>
+> **Branch**: `pratik/otel-phase7-native-metrics` (from `pratik/otel-phase6-statsd`)
+
+### Related Plan Documents
+
+| Document                                                             | Relevance                                                       |
+| -------------------------------------------------------------------- | --------------------------------------------------------------- |
+| [06-implementation-phases.md](./06-implementation-phases.md)         | Phase 7 plan: motivation, architecture, exit criteria (§6.8)    |
+| [02-design-decisions.md](./02-design-decisions.md)                   | Collector interface design, beast::insight coexistence strategy |
+| [05-configuration-reference.md](./05-configuration-reference.md)     | `[insight]` and `[telemetry]` config sections                   |
+| [09-data-collection-reference.md](./09-data-collection-reference.md) | Complete metric inventory that must be preserved                |
+
+---
+
+## Task 7.1: Add OTel Metrics SDK to Build Dependencies
+
+**Objective**: Enable the OTel C++ Metrics SDK components in the build system.
+
+**What to do**:
+
+- Edit `conanfile.py`:
+  - Add OTel metrics SDK components to the dependency list when `telemetry=True`
+  - Components needed: `opentelemetry-cpp::metrics`, `opentelemetry-cpp::otlp_http_metric_exporter`
+
+- Edit `CMakeLists.txt` (telemetry section):
+  - Link `opentelemetry::metrics` and `opentelemetry::otlp_http_metric_exporter` targets
+
+**Key modified files**:
+
+- `conanfile.py`
+- `CMakeLists.txt` (or the relevant telemetry cmake target)
+
+**Reference**: [05-configuration-reference.md §5.3](./05-configuration-reference.md) — CMake integration
+
+---
+
+## Task 7.2: Implement OTelCollector Class
+
+**Objective**: Create the core `OTelCollector` implementation that maps beast::insight instruments to OTel Metrics SDK instruments.
+
+**What to do**:
+
+- Create `include/xrpl/beast/insight/OTelCollector.h`:
+  - Public factory: `static std::shared_ptr<OTelCollector> New(std::string const& endpoint, std::string const& prefix, beast::Journal journal)`
+  - Derives from `StatsDCollector` (or directly from `Collector` — TBD based on shared code)
+
+- Create `src/libxrpl/beast/insight/OTelCollector.cpp` (~400-500 lines):
+  - **OTelCounterImpl**: Wraps `opentelemetry::metrics::Counter<int64_t>`. `increment(amount)` calls `counter->Add(amount)`.
+  - **OTelGaugeImpl**: Uses `opentelemetry::metrics::ObservableGauge<uint64_t>` with an async callback. `set(value)` stores value atomically; callback reads it during collection.
+  - **OTelMeterImpl**: Wraps `opentelemetry::metrics::Counter<uint64_t>`. `increment(amount)` calls `counter->Add(amount)`. Semantically identical to Counter but unsigned.
+  - **OTelEventImpl**: Wraps `opentelemetry::metrics::Histogram<double>`. `notify(duration)` calls `histogram->Record(duration.count())`. Uses explicit bucket boundaries matching SpanMetrics: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms.
+  - **OTelHookImpl**: Stores handler function. Called during periodic metric collection (same 1s pattern via PeriodicMetricReader).
+  - **OTelCollectorImp**: Main class.
+    - Creates `MeterProvider` with `PeriodicMetricReader` (1s export interval)
+    - Creates `OtlpHttpMetricExporter` pointing to `[telemetry]` endpoint
+    - Sets resource attributes (service.name, service.instance.id) matching trace exporter
+    - Implements all `make_*()` factory methods
+    - Prefixes metric names with `[insight] prefix=` value
+
+- Guard all OTel SDK includes with `#ifdef XRPL_ENABLE_TELEMETRY` to compile to `NullCollector` equivalents when telemetry disabled.
+
+**Key new files**:
+
+- `include/xrpl/beast/insight/OTelCollector.h`
+- `src/libxrpl/beast/insight/OTelCollector.cpp`
+
+**Key patterns to follow**:
+
+- Match `StatsDCollector.cpp` structure: private impl classes, intrusive list for metrics, strand-based thread safety
+- Match existing telemetry code style from `src/libxrpl/telemetry/Telemetry.cpp`
+- Use RAII for MeterProvider lifecycle (shutdown on destructor)
+
+**Reference**: [04-code-samples.md](./04-code-samples.md) — code style and patterns
+
+---
+
+## Task 7.3: Update CollectorManager
+
+**Objective**: Add `server=otel` config option to route metric creation to the new OTel backend.
+
+**What to do**:
+
+- Edit `src/xrpld/app/main/CollectorManager.cpp`:
+  - In the constructor, add a third branch after `server == "statsd"`:
+    ```cpp
+    else if (server == "otel")
+    {
+        // Read endpoint from [telemetry] section
+        auto const endpoint = get(telemetryParams, "endpoint",
+            "http://localhost:4318/v1/metrics");
+        std::string const& prefix(get(params, "prefix"));
+        collector_ = beast::insight::OTelCollector::New(
+            endpoint, prefix, journal);
+    }
+    ```
+  - This requires access to the `[telemetry]` config section — may need to pass it as a parameter or read from Application config.
+
+- Edit `src/xrpld/app/main/CollectorManager.h`:
+  - Add `#include <xrpl/beast/insight/OTelCollector.h>`
+
+**Key modified files**:
+
+- `src/xrpld/app/main/CollectorManager.cpp`
+- `src/xrpld/app/main/CollectorManager.h`
+
+---
+
+## Task 7.4: Update OTel Collector Configuration
+
+**Objective**: Add a metrics pipeline to the OTLP receiver and remove the StatsD receiver dependency.
+
+**What to do**:
+
+- Edit `docker/telemetry/otel-collector-config.yaml`:
+  - Remove `statsd` receiver (no longer needed when `server=otel`)
+  - Add metrics pipeline under `service.pipelines`:
+    ```yaml
+    metrics:
+      receivers: [otlp, spanmetrics]
+      processors: [batch]
+      exporters: [prometheus]
+    ```
+  - The OTLP receiver already listens on :4318 — it just needs to be added to the metrics pipeline receivers.
+  - Keep `spanmetrics` connector in the metrics pipeline so span-derived RED metrics continue working.
+
+- Edit `docker/telemetry/docker-compose.yml`:
+  - Remove UDP :8125 port mapping from otel-collector service
+  - Update xrpld service config: change `[insight] server=statsd` to `server=otel`
+
+**Key modified files**:
+
+- `docker/telemetry/otel-collector-config.yaml`
+- `docker/telemetry/docker-compose.yml`
+
+**Note**: Keep a commented-out `statsd` receiver block for operators who need backward compatibility.
+
+---
+
+## Task 7.5: Preserve Metric Names in Prometheus
+
+**Objective**: Ensure existing Grafana dashboards continue working with identical metric names.
+
+**What to do**:
+
+- In `OTelCollector.cpp`, construct OTel instrument names to match existing Prometheus metric names:
+  - beast::insight `make_gauge("LedgerMaster", "Validated_Ledger_Age")` → OTel instrument name: `xrpld_LedgerMaster_Validated_Ledger_Age`
+  - The prefix + group + name concatenation must produce the same string as `StatsDCollector`'s format
+  - Use underscores as separators (matching StatsD convention)
+
+- Verify in integration test that key Prometheus queries still return data:
+  - `xrpld_LedgerMaster_Validated_Ledger_Age`
+  - `xrpld_Peer_Finder_Active_Inbound_Peers`
+  - `xrpld_rpc_requests`
+
+**Key consideration**: OTel Prometheus exporter may normalize metric names differently than StatsD receiver. Test this early (Task 7.2) and adjust naming strategy if needed. The OTel SDK's Prometheus exporter adds `_total` suffix to counters and converts dots to underscores — match existing conventions.
+
+---
+
+## Task 7.6: Update Grafana Dashboards
+
+**Objective**: Update the 3 StatsD dashboards if any metric names change due to OTLP export format differences.
+
+**What to do**:
+
+- If Task 7.5 confirms metric names are preserved exactly, no dashboard changes needed.
+- If OTLP export produces different names (e.g., `_total` suffix on counters), update:
+  - `docker/telemetry/grafana/dashboards/statsd-node-health.json`
+  - `docker/telemetry/grafana/dashboards/statsd-network-traffic.json`
+  - `docker/telemetry/grafana/dashboards/statsd-rpc-pathfinding.json`
+- Rename dashboard titles from "StatsD" to "System Metrics" or similar (since they're no longer StatsD-sourced).
+
+**Key modified files**:
+
+- `docker/telemetry/grafana/dashboards/statsd-*.json` (3 files, conditionally)
+
+---
+
+## Task 7.7: Update Integration Tests
+
+**Objective**: Verify the full OTLP metrics pipeline end-to-end.
+
+**What to do**:
+
+- Edit `docker/telemetry/integration-test.sh`:
+  - Update test config to use `[insight] server=otel`
+  - Verify metrics arrive in Prometheus via OTLP (not StatsD)
+  - Add check that StatsD receiver is no longer required
+  - Preserve all existing metric presence checks
+
+**Key modified files**:
+
+- `docker/telemetry/integration-test.sh`
+
+---
+
+## Task 7.8: Update Documentation
+
+**Objective**: Update all plan docs, runbook, and reference docs to reflect the migration.
+
+**What to do**:
+
+- Edit `docs/telemetry-runbook.md`:
+  - Update `[insight]` config examples to show `server=otel`
+  - Update troubleshooting section (no more StatsD UDP debugging)
+
+- Edit `OpenTelemetryPlan/09-data-collection-reference.md`:
+  - Update Data Flow Overview diagram (remove StatsD receiver)
+  - Update Section 2 header from "StatsD Metrics" to "System Metrics (OTel native)"
+  - Update config examples
+
+- Edit `OpenTelemetryPlan/05-configuration-reference.md`:
+  - Add `server=otel` option to `[insight]` section docs
+
+- Edit `docker/telemetry/TESTING.md`:
+  - Update setup instructions to use `server=otel`
+
+**Key modified files**:
+
+- `docs/telemetry-runbook.md`
+- `OpenTelemetryPlan/09-data-collection-reference.md`
+- `OpenTelemetryPlan/05-configuration-reference.md`
+- `docker/telemetry/TESTING.md`
+
+---
+
+## Task 7.9: ValidationTracker — Validation Agreement Computation
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — the most valuable metric from the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
+>
+> **Upstream**: Phase 4 Task 4.8 (validation span attributes provide ledger hash context).
+> **Downstream**: Phase 9 (Validator Health dashboard), Phase 10 (validation checks), Phase 11 (agreement alert rules).
+
+**Objective**: Implement a stateful class that tracks whether our validator's validations agree with network consensus, maintaining rolling 1h and 24h windows with an 8-second grace period and 5-minute late repair window.
+
+**Architecture**:
+
+```
+consensus.validation.send ────> ValidationTracker ────> MetricsRegistry
+(records our validation         (reconciles after       (exports agreement
+ for ledger X)                   8s grace period)        gauges every 10s)
+
+ledger.validate ──────────────> ValidationTracker
+(records which ledger           (marks ledger X as
+ network validated)              agreed or missed)
+```
+
+**What to do**:
+
+- Create `src/xrpld/telemetry/ValidationTracker.h`:
+  - `recordOurValidation(ledgerHash, ledgerSeq)` — called when we send a validation
+  - `recordNetworkValidation(ledgerHash, seq)` — called when a ledger is fully validated
+  - `reconcile()` — called periodically; reconciles pending ledger events after 8s grace period
+  - Getters: `agreementPct1h()`, `agreementPct24h()`, `agreements1h()`, `missed1h()`, `agreements24h()`, `missed24h()`, `totalAgreements()`, `totalMissed()`, `totalValidationsSent()`, `totalValidationsChecked()`
+  - Thread-safety: atomics for counters, mutex for window deques
+
+- Create `src/xrpld/telemetry/detail/ValidationTracker.cpp`:
+  - Reconciliation logic: after 8s grace period, check if `weValidated && networkValidated && sameHash` → agreement; else missed
+  - Late repair: if a late validation arrives within 5 minutes, correct a false-positive miss
+  - Sliding window: `std::deque<WindowEvent>` evicts entries older than 1h/24h on each reconciliation pass
+  - Ring buffer of 1000 `LedgerEvent` structs for pending reconciliation
+
+- Add recording hooks (modifying Phase 4 code from Phase 7 branch):
+  - `RCLConsensus.cpp` `validate()`: call `tracker.recordOurValidation()`
+  - `LedgerMaster.cpp` fully-validated path: call `tracker.recordNetworkValidation()`
+
+**Key data structures**:
+
+```cpp
+struct LedgerEvent {
+    uint256 ledgerHash;
+    LedgerIndex seq;
+    TimePoint closeTime;
+    bool weValidated = false;
+    bool networkValidated = false;
+    bool reconciled = false;
+    bool agreed = false;
+};
+
+struct WindowEvent {
+    TimePoint time;
+    bool agreed;
+};
+```
+
+**Key new files**:
+
+- `src/xrpld/telemetry/ValidationTracker.h`
+- `src/xrpld/telemetry/detail/ValidationTracker.cpp`
+
+**Key modified files**:
+
+- `src/xrpld/telemetry/MetricsRegistry.h` (add ValidationTracker member)
+- `src/xrpld/telemetry/MetricsRegistry.cpp` (add gauge callback reading from tracker)
+- `src/xrpld/app/consensus/RCLConsensus.cpp` (add recording hooks)
+- `src/xrpld/app/ledger/detail/LedgerMaster.cpp` (add recording hook)
+
+**Exit Criteria**:
+
+- [ ] ValidationTracker correctly tracks agreement with 8s grace period
+- [ ] 5-minute late repair corrects false-positive misses
+- [ ] Thread-safe (atomics + mutex for window deques)
+- [ ] Rolling windows correctly evict stale entries
+- [ ] Unit tests: normal agreement, missed validation, late repair, window eviction
+
+---
+
+## Task 7.10: Validator Health Observable Gauges
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
+
+**Objective**: Export amendment blocked, UNL health, and quorum data as a native OTel observable gauge.
+
+**What to do**:
+
+- In `MetricsRegistry.cpp` `registerAsyncGauges()`, add:
+
+```cpp
+validatorHealthGauge_ = meter_->CreateDoubleObservableGauge(
+    "xrpld_validator_health", "Validator health indicators");
+```
+
+**Gauge label values**:
+
+| Label `metric=`     | Type   | Source                                            |
+| ------------------- | ------ | ------------------------------------------------- |
+| `amendment_blocked` | int64  | `app_.getOPs().isAmendmentBlocked()` → 0/1        |
+| `unl_blocked`       | int64  | `app_.getOPs().isUNLBlocked()` → 0/1              |
+| `unl_expiry_days`   | double | `app_.validators().expires()` → days until expiry |
+| `validation_quorum` | int64  | `app_.validators().quorum()`                      |
+
+### Sub-task 7.10a: Per-Validator Validation Count (Flag Ledger Window)
+
+**Objective**: Track how many ledgers each UNL validator has validated over
+the last 256 consecutive ledgers (one flag ledger window). This is the key
+UNL participation metric — validators consistently below threshold may be
+candidates for removal from the UNL.
+
+**What to do**:
+
+- Add a new observable gauge:
+
+```cpp
+validatorParticipationGauge_ = meter_->CreateInt64ObservableGauge(
+    "xrpld_validator_participation",
+    "Per-validator validation count over the last 256 ledgers");
+```
+
+- The callback queries `app_.getValidations()` to get the trusted
+  validation set for each of the last 256 ledger hashes (from
+  `LedgerMaster::getValidatedLedger()` walking backwards). For each
+  validator public key in the UNL, count how many of those 256 ledgers
+  have a matching validation.
+
+- **Label dimensions**:
+  - `validator` — base58-encoded validator master public key
+  - `service_instance_id` — this node's identity (standard)
+
+- **Emission**: every flag ledger (256 ledgers, ~15 minutes) or on a
+  10-second async gauge callback with cached results (recompute only
+  at flag ledger boundaries).
+
+- **Data source**: `RCLValidations::getTrustedForLedger(hash, seq)` returns
+  `std::vector<std::shared_ptr<STValidation>>` with `getSignerPublic()`
+  for each. The UNL list is from `app_.getValidators().getTrustedMasterKeys()`.
+
+- **Dashboard panel**: Add a table panel to the Validator Health dashboard
+  showing `xrpld_validator_participation` grouped by `validator` label,
+  with a threshold color (green >= 240, yellow >= 200, red < 200).
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
+
+**Exit Criteria**:
+
+- [ ] Gauge emits one time series per UNL validator
+- [ ] Values range 0-256 and update at flag ledger boundaries
+- [ ] Grafana table panel shows per-validator participation
+- [ ] Validators below 75% participation are highlighted in red
+
+---
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
+
+**Exit Criteria**:
+
+- [ ] All 4 base label values emitted every 10s
+- [ ] `unl_expiry_days` is negative when expired, positive when active
+- [ ] Per-validator participation gauge emits at flag ledger boundaries
+- [ ] Values visible in Prometheus
+
+---
+
+## Task 7.11: Peer Quality Observable Gauges
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
+
+**Objective**: Export peer health aggregates (latency P90, insane peers, version awareness) as a native OTel observable gauge.
+
+**What to do**:
+
+- In `MetricsRegistry.cpp` `registerAsyncGauges()`, add a callback that iterates `app_.overlay().foreach(...)` to:
+  - Collect per-peer latency values, sort, compute P90
+  - Count peers with `tracking_ == diverged` (insane)
+  - Compare peer `getVersion()` to own version for upgrade awareness
+
+**Gauge label values**:
+
+| Label `metric=`            | Type   | Source                                |
+| -------------------------- | ------ | ------------------------------------- |
+| `peer_latency_p90_ms`      | double | P90 from sorted peer latencies        |
+| `peers_insane_count`       | int64  | Peers with diverged tracking status   |
+| `peers_higher_version_pct` | double | % of peers on newer xrpld version     |
+| `upgrade_recommended`      | int64  | 1 if `peers_higher_version_pct > 60%` |
+
+**Implementation note**: The callback runs every 10s on the metrics reader thread. Iterating ~50-200 peers is acceptable overhead.
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
+
+**Exit Criteria**:
+
+- [ ] P90 latency computed correctly
+- [ ] Insane count matches `peers` RPC output
+- [ ] Version comparison handles format variations (e.g., "xrpld-2.4.0-rc1")
+
+---
+
+## Task 7.12: Ledger Economy Observable Gauges
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
+
+**Objective**: Export fee, reserve, ledger age, and transaction rate as a native OTel observable gauge.
+
+**Gauge label values**:
+
+| Label `metric=`      | Type   | Source                                              |
+| -------------------- | ------ | --------------------------------------------------- |
+| `base_fee_xrp`       | double | Base fee from validated ledger fee settings (drops) |
+| `reserve_base_xrp`   | double | Account reserve from validated ledger (drops)       |
+| `reserve_inc_xrp`    | double | Owner reserve increment (drops)                     |
+| `ledger_age_seconds` | double | `now - lastValidatedCloseTime`                      |
+| `transaction_rate`   | double | Derived: tx count delta / time delta (smoothed)     |
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
+
+**Exit Criteria**:
+
+- [ ] Fee values match `server_info` RPC output
+- [ ] `ledger_age_seconds` increases monotonically between ledger closes
+- [ ] `transaction_rate` is smoothed (rolling average)
+
+---
+
+## Task 7.13: State Tracking Observable Gauges
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
+
+**Objective**: Export extended state value (0-6 encoding combining OperatingMode + ConsensusMode) and time-in-current-state.
+
+**Gauge label values**:
+
+| Label `metric=`                 | Type   | Source                                          |
+| ------------------------------- | ------ | ----------------------------------------------- |
+| `state_value`                   | int64  | 0-6 encoding (see spec for mapping)             |
+| `time_in_current_state_seconds` | double | `now - lastModeChangeTime` from StateAccounting |
+
+**State value encoding**: 0=disconnected, 1=connected, 2=syncing, 3=tracking, 4=full, 5=validating (full + validating), 6=proposing (full + proposing).
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
+
+**Exit Criteria**:
+
+- [ ] `state_value` correctly combines OperatingMode and ConsensusMode
+- [ ] `time_in_current_state_seconds` resets on mode change
+
+---
+
+## Task 7.14: Storage Detail and Sync Info Gauges
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
+
+**Objective**: Export NuDB-specific storage size and initial sync duration.
+
+**Gauge label values**:
+
+| Gauge Name             | Label `metric=`                 | Type   | Source                        |
+| ---------------------- | ------------------------------- | ------ | ----------------------------- |
+| `xrpld_storage_detail` | `nudb_bytes`                    | int64  | NuDB backend file size        |
+| `xrpld_sync_info`      | `initial_sync_duration_seconds` | double | Time from start to first FULL |
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp`
+
+**Exit Criteria**:
+
+- [ ] NuDB file size reported in bytes (0 if NuDB not configured)
+- [ ] Sync duration captured once and remains stable after reaching FULL
+
+---
+
+## Task 7.15: New Synchronous Counters
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
+
+**Objective**: Add 7 new event counters incremented at their respective instrumentation sites.
+
+| Counter Name                        | Increment Site                   | Source File           |
+| ----------------------------------- | -------------------------------- | --------------------- |
+| `xrpld_ledgers_closed_total`        | `onAccept()` in consensus        | RCLConsensus.cpp      |
+| `xrpld_validations_sent_total`      | `validate()` in consensus        | RCLConsensus.cpp      |
+| `xrpld_validations_checked_total`   | Network validation received      | LedgerMaster.cpp      |
+| `xrpld_validation_agreements_total` | ValidationTracker reconciliation | ValidationTracker.cpp |
+| `xrpld_validation_missed_total`     | ValidationTracker reconciliation | ValidationTracker.cpp |
+| `xrpld_state_changes_total`         | `setMode()` in NetworkOPs        | NetworkOPs.cpp        |
+| `xrpld_jq_trans_overflow_total`     | Job queue overflow path          | JobQueue.cpp          |
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` (declarations), plus recording sites in RCLConsensus.cpp, LedgerMaster.cpp, NetworkOPs.cpp, JobQueue.cpp
+
+**Exit Criteria**:
+
+- [ ] All 7 counters monotonically increase during normal operation
+- [ ] Counter values match expected rates (e.g., ledgers_closed ≈ 1 per 3-5s)
+
+---
+
+## Task 7.16: Validation Agreement Observable Gauge
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
+
+**Objective**: Export rolling window agreement stats from `ValidationTracker` (Task 7.9).
+
+**Gauge label values**:
+
+| Gauge Name                   | Label `metric=`     | Type   | Source                      |
+| ---------------------------- | ------------------- | ------ | --------------------------- |
+| `xrpld_validation_agreement` | `agreement_pct_1h`  | double | `tracker.agreementPct1h()`  |
+|                              | `agreements_1h`     | int64  | `tracker.agreements1h()`    |
+|                              | `missed_1h`         | int64  | `tracker.missed1h()`        |
+|                              | `agreement_pct_24h` | double | `tracker.agreementPct24h()` |
+|                              | `agreements_24h`    | int64  | `tracker.agreements24h()`   |
+|                              | `missed_24h`        | int64  | `tracker.missed24h()`       |
+
+**Key modified files**: `src/xrpld/telemetry/MetricsRegistry.cpp`
+
+**Exit Criteria**:
+
+- [ ] Agreement percentages in range [0.0, 100.0]
+- [ ] Window stats stabilize after 1h/24h of operation
+
+---
+
+## Summary Table
+
+| Task | Description                            | New Files | Modified Files | Depends On |
+| ---- | -------------------------------------- | --------- | -------------- | ---------- |
+| 7.1  | Add OTel Metrics SDK to build deps     | 0         | 2              | —          |
+| 7.2  | Implement OTelCollector class          | 2         | 0              | 7.1        |
+| 7.3  | Update CollectorManager config routing | 0         | 2              | 7.2        |
+| 7.4  | Update OTel Collector YAML and Docker  | 0         | 2              | 7.3        |
+| 7.5  | Preserve metric names in Prometheus    | 0         | 1              | 7.2        |
+| 7.6  | Update Grafana dashboards (if needed)  | 0         | 3              | 7.5        |
+| 7.7  | Update integration tests               | 0         | 1              | 7.4        |
+| 7.8  | Update documentation                   | 0         | 4              | 7.6        |
+| 7.9  | ValidationTracker (agreement tracking) | 2         | 4              | 7.2, P4.8  |
+| 7.10 | Validator health observable gauges     | 0         | 2              | 7.2        |
+| 7.11 | Peer quality observable gauges         | 0         | 2              | 7.2        |
+| 7.12 | Ledger economy observable gauges       | 0         | 2              | 7.2        |
+| 7.13 | State tracking observable gauges       | 0         | 2              | 7.2        |
+| 7.14 | Storage detail and sync info gauges    | 0         | 2              | 7.2        |
+| 7.15 | New synchronous counters               | 0         | 6              | 7.2        |
+| 7.16 | Validation agreement observable gauge  | 0         | 1              | 7.9        |
+
+**Parallel work**: Tasks 7.4 and 7.5 can run in parallel after 7.2/7.3 complete. Task 7.6 depends on 7.5's findings. Tasks 7.7 and 7.8 can run in parallel after 7.6. Tasks 7.10-7.14 can all run in parallel after 7.2. Task 7.15 depends on 7.2. Task 7.16 depends on 7.9. Task 7.9 depends on 7.2 and Phase 4 Task 4.8.
+
+**Exit Criteria** (from [06-implementation-phases.md §6.8](./06-implementation-phases.md)):
+
+- [ ] All 255+ metrics visible in Prometheus via OTLP pipeline (no StatsD receiver)
+- [ ] `server=otel` is the default in development docker-compose
+- [ ] `server=statsd` still works as a fallback
+- [ ] Existing Grafana dashboards display data correctly
+- [ ] Integration test passes with OTLP-only metrics pipeline
+- [ ] No performance regression vs StatsD baseline (< 1% CPU overhead)
+- [ ] Deferred Task 6.1 (`|m` wire format) no longer relevant — Meter mapped to OTel Counter
+- [ ] ValidationTracker agreement % stabilizes after 1h under normal consensus
+- [ ] All new gauges and counters visible in Prometheus with non-zero values

OpenTelemetryPlan/Phase8_taskList.md

@@ -0,0 +1,239 @@
+# Phase 8: Log-Trace Correlation and Centralized Log Ingestion — Task List
+
+> **Goal**: Inject trace context (trace_id, span_id) into xrpld's Journal log output for log-trace correlation, and add OTel Collector filelog receiver to ingest logs into Grafana Loki for unified observability.
+>
+> **Scope**: Two independent sub-phases — 8a (code change: trace_id in logs) and 8b (infra only: filelog receiver to Loki). No changes to the `beast::Journal` public API.
+>
+> **Branch**: `pratik/otel-phase8-log-correlation` (from `pratik/otel-phase7-native-metrics`)
+
+### Related Plan Documents
+
+| Document                                                         | Relevance                                                      |
+| ---------------------------------------------------------------- | -------------------------------------------------------------- |
+| [06-implementation-phases.md](./06-implementation-phases.md)     | Phase 8 plan: motivation, architecture, exit criteria (§6.8.1) |
+| [07-observability-backends.md](./07-observability-backends.md)   | Loki backend recommendation, Grafana data source provisioning  |
+| [Phase7_taskList.md](./Phase7_taskList.md)                       | Prerequisite — native OTel metrics pipeline must be working    |
+| [05-configuration-reference.md](./05-configuration-reference.md) | `[telemetry]` config (trace_id injection toggle)               |
+
+---
+
+## Task 8.1: Inject trace_id into Logs::format()
+
+**Objective**: Add OTel trace context to every log line that is emitted within an active span.
+
+**What to do**:
+
+- Edit `src/libxrpl/basics/Log.cpp`:
+  - In `Logs::format()` (around line 346), after severity is appended, check for active OTel span. The implementation checks the context value directly to avoid the heap allocation that `GetSpan()` performs on the no-span path:
+    ```cpp
+    #ifdef XRPL_ENABLE_TELEMETRY
+    {
+        auto context = opentelemetry::context::RuntimeContext::GetCurrent();
+        auto spanValue = context.GetValue(opentelemetry::trace::kSpanKey);
+        if (opentelemetry::nostd::holds_alternative<
+                opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>>(spanValue))
+        {
+            auto span = opentelemetry::nostd::get<
+                opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>>(spanValue);
+            auto spanCtx = span->GetContext();
+            if (spanCtx.IsValid())
+            {
+                char traceId[32], spanId[16];
+                spanCtx.trace_id().ToLowerBase16(
+                    opentelemetry::nostd::span<char, 32>{traceId});
+                spanCtx.span_id().ToLowerBase16(
+                    opentelemetry::nostd::span<char, 16>{spanId});
+                output += "trace_id=";
+                output.append(traceId, 32);
+                output += " span_id=";
+                output.append(spanId, 16);
+                output += ' ';
+            }
+        }
+    }
+    #endif
+    ```
+  - Add `#include` for OTel context headers, guarded by `#ifdef XRPL_ENABLE_TELEMETRY`
+
+- Edit `include/xrpl/basics/Log.h`:
+  - No changes needed — format() signature unchanged
+
+**Key modified files**:
+
+- `src/libxrpl/basics/Log.cpp`
+
+**Performance note**: The implementation checks the thread-local context value directly (avoiding the heap allocation that `GetSpan()` performs on the no-span path). On threads without an active span (~99% of log lines), the cost is a thread-local read + variant type check (~15-20ns). On the active-span path, an additional shared_ptr copy + `GetContext()` + `IsValid()` adds ~50ns total. Overhead is negligible at typical logging rates.
+
+---
+
+## Task 8.2: Add Loki to Docker Compose Stack
+
+**Objective**: Add Grafana Loki as a log storage backend in the development observability stack.
+
+**What to do**:
+
+- Edit `docker/telemetry/docker-compose.yml`:
+  - Add Loki service:
+    ```yaml
+    loki:
+      image: grafana/loki:3.4.2
+      ports:
+        - "3100:3100"
+      command: -config.file=/etc/loki/local-config.yaml
+    ```
+  - Add Loki as a Grafana data source in provisioning
+
+- Create `docker/telemetry/grafana/provisioning/datasources/loki.yaml`:
+  - Configure Loki data source with derived fields linking `trace_id` to Tempo
+
+**Key new files**:
+
+- `docker/telemetry/grafana/provisioning/datasources/loki.yaml`
+
+**Key modified files**:
+
+- `docker/telemetry/docker-compose.yml`
+
+---
+
+## Task 8.3: Add Filelog Receiver to OTel Collector
+
+**Objective**: Configure the OTel Collector to tail xrpld's log file and export to Loki.
+
+**What to do**:
+
+- Edit `docker/telemetry/otel-collector-config.yaml`:
+  - Add `filelog` receiver:
+    ```yaml
+    receivers:
+      filelog:
+        include: [/var/log/xrpld/*/debug.log]
+        operators:
+          - type: regex_parser
+            regex: '^(?P<timestamp>\S+)\s+(?P<partition>\S+):(?P<severity>\S+)\s+(?:trace_id=(?P<trace_id>[a-f0-9]+)\s+span_id=(?P<span_id>[a-f0-9]+)\s+)?(?P<message>.*)$'
+            timestamp:
+              parse_from: attributes.timestamp
+              layout: "%Y-%m-%dT%H:%M:%S.%fZ"
+    ```
+  - Add logs pipeline:
+    ```yaml
+    service:
+      pipelines:
+        logs:
+          receivers: [filelog]
+          processors: [batch]
+          exporters: [otlp/loki]
+    ```
+  - Add Loki exporter:
+    ```yaml
+    exporters:
+      otlphttp/loki:
+        endpoint: http://loki:3100/otlp
+    ```
+
+- Mount xrpld's log directory into the collector container via docker-compose volume
+
+**Key modified files**:
+
+- `docker/telemetry/otel-collector-config.yaml`
+- `docker/telemetry/docker-compose.yml`
+
+---
+
+## Task 8.4: Configure Grafana Trace-to-Log Correlation
+
+**Objective**: Enable one-click navigation from Tempo traces to Loki logs in Grafana.
+
+**What to do**:
+
+- Edit Grafana Tempo data source provisioning to add `tracesToLogs` configuration:
+
+  ```yaml
+  tracesToLogs:
+    datasourceUid: loki
+    filterByTraceID: true
+    filterBySpanID: false
+    tags: ["partition", "severity"]
+  ```
+
+- Edit Grafana Loki data source provisioning to add `derivedFields` linking trace_id back to Tempo:
+  ```yaml
+  derivedFields:
+    - datasourceUid: tempo
+      matcherRegex: "trace_id=(\\w+)"
+      name: TraceID
+      url: "$${__value.raw}"
+  ```
+
+**Key modified files**:
+
+- `docker/telemetry/grafana/provisioning/datasources/loki.yaml`
+- `docker/telemetry/grafana/provisioning/datasources/` (Tempo data source file)
+
+---
+
+## Task 8.5: Update Integration Tests
+
+**Objective**: Verify trace_id appears in logs and Loki correlation works.
+
+**What to do**:
+
+- Edit `docker/telemetry/integration-test.sh`:
+  - After sending RPC requests (which create spans), grep xrpld's log output for `trace_id=`
+  - Verify trace_id matches a trace visible in Tempo
+  - Optionally: query Loki via API to confirm log ingestion
+
+**Key modified files**:
+
+- `docker/telemetry/integration-test.sh`
+
+---
+
+## Task 8.6: Update Documentation
+
+**Objective**: Document the log correlation feature in runbook and reference docs.
+
+**What to do**:
+
+- Edit `docs/telemetry-runbook.md`:
+  - Add "Log-Trace Correlation" section explaining how to use Grafana Tempo -> Loki linking
+  - Add LogQL query examples for filtering by trace_id
+
+- Edit `OpenTelemetryPlan/09-data-collection-reference.md`:
+  - Add new section "3. Log Correlation" between SpanMetrics and StatsD sections
+  - Document the log format with trace_id injection
+  - Document Loki as a new backend
+
+- Edit `docker/telemetry/TESTING.md`:
+  - Add log correlation verification steps
+
+**Key modified files**:
+
+- `docs/telemetry-runbook.md`
+- `OpenTelemetryPlan/09-data-collection-reference.md`
+- `docker/telemetry/TESTING.md`
+
+---
+
+## Summary Table
+
+| Task | Description                                | Sub-Phase | New Files | Modified Files | Depends On |
+| ---- | ------------------------------------------ | --------- | --------- | -------------- | ---------- |
+| 8.1  | Inject trace_id into Logs::format()        | 8a        | 0         | 1              | Phase 7    |
+| 8.2  | Add Loki to Docker Compose stack           | 8b        | 1         | 1              | --         |
+| 8.3  | Add filelog receiver to OTel Collector     | 8b        | 0         | 2              | 8.1, 8.2   |
+| 8.4  | Configure Grafana trace-to-log correlation | 8b        | 0         | 2              | 8.3        |
+| 8.5  | Update integration tests                   | 8a + 8b   | 0         | 1              | 8.4        |
+| 8.6  | Update documentation                       | 8a + 8b   | 0         | 3              | 8.5        |
+
+**Parallel work**: Task 8.2 (Loki infra) can run in parallel with Task 8.1 (code change). Tasks 8.3-8.6 are sequential.
+
+**Exit Criteria** (from [06-implementation-phases.md §6.8.1](./06-implementation-phases.md)):
+
+- [ ] Log lines within active spans contain `trace_id=<hex> span_id=<hex>`
+- [ ] Log lines outside spans have no trace context (no empty fields)
+- [ ] Loki ingests xrpld logs via OTel Collector filelog receiver
+- [ ] Grafana Tempo -> Loki one-click correlation works
+- [ ] Grafana Loki -> Tempo reverse lookup works via derived field
+- [ ] Integration test verifies trace_id presence in logs
+- [ ] No performance regression from trace_id injection (< 0.1% overhead)

OpenTelemetryPlan/Phase9_taskList.md

@@ -0,0 +1,549 @@
+# Phase 9: Internal Metric Instrumentation Gap Fill — Task List
+
+> **Status**: Future Enhancement
+>
+> **Goal**: Instrument xrpld to emit ~50+ metrics that exist in `get_counts`/`server_info`/TxQ/PerfLog but currently lack time-series export via the OTel or beast::insight pipelines.
+>
+> **Scope**: Hybrid approach — extend `beast::insight` for metrics near existing registrations, use OTel Metrics SDK `ObservableGauge` callbacks for new categories (TxQ, PerfLog, CountedObjects).
+>
+> **Branch**: `pratik/otel-phase9-metric-gap-fill` (from `pratik/otel-phase8-log-correlation`)
+>
+> **Depends on**: Phase 7 (native OTel metrics pipeline) and Phase 8 (log-trace correlation)
+
+### Related Plan Documents
+
+| Document                                                             | Relevance                                                      |
+| -------------------------------------------------------------------- | -------------------------------------------------------------- |
+| [06-implementation-phases.md](./06-implementation-phases.md)         | Phase 9 plan: motivation, architecture, exit criteria (§6.8.2) |
+| [09-data-collection-reference.md](./09-data-collection-reference.md) | Current metric inventory + future metrics section              |
+| [Phase7_taskList.md](./Phase7_taskList.md)                           | Prerequisite — OTel Metrics SDK and `OTelCollector` class      |
+| [Phase8_taskList.md](./Phase8_taskList.md)                           | Prerequisite — log-trace correlation                           |
+
+### Third-Party Consumer Context
+
+These metrics serve multiple external consumer categories identified during research:
+
+| Consumer Category         | Key Metrics They Need                                           |
+| ------------------------- | --------------------------------------------------------------- |
+| **Exchanges**             | Fee escalation levels, TxQ depth, settlement latency            |
+| **Payment Processors**    | Load factors, io_latency, transaction throughput                |
+| **Analytics Providers**   | NodeStore I/O, cache hit rates, counted objects                 |
+| **Validators/Operators**  | Per-job execution times, PerfLog RPC counters, consensus timing |
+| **Academic Researchers**  | Consensus performance time-series, fee market dynamics          |
+| **Institutional Custody** | Server health scores, reserve calculations, node availability   |
+
+---
+
+## Task 9.1: NodeStore I/O Metrics
+
+**Objective**: Export node store read/write performance as time-series metrics.
+
+**What to do**:
+
+- In `src/libxrpl/nodestore/Database.cpp`, extend existing `beast::insight` registrations to add:
+  - Gauge: `node_reads_total` (cumulative read operations)
+  - Gauge: `node_reads_hit` (cache-served reads)
+  - Gauge: `node_writes` (cumulative write operations)
+  - Gauge: `node_written_bytes` (cumulative bytes written)
+  - Gauge: `node_read_bytes` (cumulative bytes read)
+  - Gauge: `node_reads_duration_us` (cumulative read time in microseconds)
+  - Gauge: `write_load` (current write load score)
+  - Gauge: `read_queue` (items in read queue)
+
+- These values are already computed in `Database::getCountsJson()` (line ~236). Wire the same counters to `beast::insight` hooks.
+
+**Key modified files**:
+
+- `src/libxrpl/nodestore/Database.cpp`
+- `src/libxrpl/nodestore/Database.h` (add insight members)
+
+**Derived Prometheus metrics**: `xrpld_nodestore_reads_total`, `xrpld_nodestore_reads_hit`, `xrpld_nodestore_write_load`, etc.
+
+**Grafana dashboard**: Add "NodeStore I/O" panel group to _Node Health_ dashboard.
+
+---
+
+## Task 9.2: Cache Hit Rate Metrics
+
+**Objective**: Export SHAMap and ledger cache performance as time-series gauges.
+
+**What to do**:
+
+- Register OTel `ObservableGauge` callbacks (via Phase 7's `OTelCollector`) for:
+  - `SLE_hit_rate` — SLE cache hit rate (0.0–1.0)
+  - `ledger_hit_rate` — Ledger object cache hit rate
+  - `AL_hit_rate` — AcceptedLedger cache hit rate
+  - `treenode_cache_size` — SHAMap TreeNode cache size (entries)
+  - `treenode_track_size` — Tracked tree nodes
+  - `fullbelow_size` — FullBelow cache size
+
+- The callback should read from the same sources as `GetCounts.cpp` handler (line ~43).
+
+- Create a centralized `MetricsRegistry` class that holds all OTel async gauge registrations, polled at 10-second intervals by the `PeriodicMetricReader`.
+
+**Key modified files**:
+
+- New: `src/xrpld/telemetry/MetricsRegistry.h` / `.cpp`
+- `src/xrpld/rpc/handlers/GetCounts.cpp` (extract shared access methods)
+- `src/xrpld/app/main/Application.cpp` (register MetricsRegistry at startup)
+
+**Derived Prometheus metrics**: `xrpld_cache_SLE_hit_rate`, `xrpld_cache_ledger_hit_rate`, `xrpld_cache_treenode_size`, etc.
+
+---
+
+## Task 9.3: Transaction Queue (TxQ) Metrics
+
+**Objective**: Export TxQ depth, capacity, and fee escalation levels as time-series.
+
+**What to do**:
+
+- Register OTel `ObservableGauge` callbacks for TxQ state (from `TxQ.h` line ~143):
+  - `txq_count` — Current transactions in queue
+  - `txq_max_size` — Maximum queue capacity
+  - `txq_in_ledger` — Transactions in current open ledger
+  - `txq_per_ledger` — Expected transactions per ledger
+  - `txq_reference_fee_level` — Reference fee level
+  - `txq_min_processing_fee_level` — Minimum fee to get processed
+  - `txq_med_fee_level` — Median fee level in queue
+  - `txq_open_ledger_fee_level` — Open ledger fee escalation level
+
+- Add to the `MetricsRegistry` (Task 9.2).
+
+**Key modified files**:
+
+- `src/xrpld/telemetry/MetricsRegistry.cpp` (add TxQ callbacks)
+- `src/xrpld/app/tx/detail/TxQ.h` (expose metrics accessor if needed)
+
+**Derived Prometheus metrics**: `xrpld_txq_count`, `xrpld_txq_max_size`, `xrpld_txq_open_ledger_fee_level`, etc.
+
+**Grafana dashboard**: New _Fee Market & TxQ_ dashboard (`fee-market`).
+
+---
+
+## Task 9.4: PerfLog Per-RPC Method Metrics
+
+**Objective**: Export per-RPC-method call counts and latency as OTel metrics.
+
+**What to do**:
+
+- Register OTel instruments for PerfLog RPC counters (from `PerfLogImp.cpp` line ~63):
+  - Counter: `xrpld_rpc_method_started_total{method="<name>"}` — calls started
+  - Counter: `xrpld_rpc_method_finished_total{method="<name>"}` — calls completed
+  - Counter: `xrpld_rpc_method_errored_total{method="<name>"}` — calls errored
+  - Histogram: `xrpld_rpc_method_duration_us{method="<name>"}` — execution time distribution
+
+- Use OTel `Counter<int64_t>` and `Histogram<double>` instruments with `method` attribute label.
+
+- Hook into the existing PerfLog callback mechanism rather than adding new instrumentation points.
+
+**Key modified files**:
+
+- `src/xrpld/perflog/detail/PerfLogImp.cpp` (add OTel instrument updates alongside existing JSON counters)
+- `src/xrpld/telemetry/MetricsRegistry.cpp` (register instruments)
+
+**Derived Prometheus metrics**: `xrpld_rpc_method_started_total{method="server_info"}`, `xrpld_rpc_method_duration_us_bucket{method="ledger"}`, etc.
+
+**Grafana dashboard**: Add "Per-Method RPC Breakdown" panel group to _RPC Performance_ dashboard.
+
+---
+
+## Task 9.5: PerfLog Per-Job-Type Metrics
+
+**Objective**: Export per-job-type queue and execution metrics.
+
+**What to do**:
+
+- Register OTel instruments for PerfLog job counters:
+  - Counter: `xrpld_job_queued_total{job_type="<name>"}` — jobs queued
+  - Counter: `xrpld_job_started_total{job_type="<name>"}` — jobs started
+  - Counter: `xrpld_job_finished_total{job_type="<name>"}` — jobs completed
+  - Histogram: `xrpld_job_queued_duration_us{job_type="<name>"}` — time spent waiting in queue
+  - Histogram: `xrpld_job_running_duration_us{job_type="<name>"}` — execution time distribution
+
+- Hook into PerfLog's existing job tracking alongside Task 9.4.
+
+**Key modified files**:
+
+- `src/xrpld/perflog/detail/PerfLogImp.cpp`
+- `src/xrpld/telemetry/MetricsRegistry.cpp`
+
+**Derived Prometheus metrics**: `xrpld_job_queued_total{job_type="ledgerData"}`, `xrpld_job_running_duration_us_bucket{job_type="transaction"}`, etc.
+
+**Grafana dashboard**: New _Job Queue Analysis_ dashboard (`job-queue`).
+
+---
+
+## Task 9.6: Counted Object Instance Metrics
+
+**Objective**: Export live instance counts for key internal object types.
+
+**What to do**:
+
+- Register OTel `ObservableGauge` callbacks for `CountedObject<T>` instance counts:
+  - `xrpld_object_count{type="Transaction"}` — live Transaction objects
+  - `xrpld_object_count{type="Ledger"}` — live Ledger objects
+  - `xrpld_object_count{type="NodeObject"}` — live NodeObject instances
+  - `xrpld_object_count{type="STTx"}` — serialized transaction objects
+  - `xrpld_object_count{type="STLedgerEntry"}` — serialized ledger entries
+  - `xrpld_object_count{type="InboundLedger"}` — ledgers being fetched
+  - `xrpld_object_count{type="Pathfinder"}` — active pathfinding computations
+  - `xrpld_object_count{type="PathRequest"}` — active path requests
+  - `xrpld_object_count{type="HashRouterEntry"}` — hash router entries
+
+- The `CountedObject` template already tracks these via atomic counters. The callback just reads the current counts.
+
+**Key modified files**:
+
+- `src/xrpld/telemetry/MetricsRegistry.cpp` (add counted object callbacks)
+- `include/xrpl/basics/CountedObject.h` (may need static accessor for iteration)
+
+**Derived Prometheus metrics**: `xrpld_object_count{type="Transaction"}`, `xrpld_object_count{type="NodeObject"}`, etc.
+
+**Grafana dashboard**: Add "Object Instance Counts" panel to _Node Health_ dashboard.
+
+---
+
+## Task 9.7: Fee Escalation & Load Factor Metrics
+
+**Objective**: Export the full load factor breakdown as time-series.
+
+**What to do**:
+
+- Register OTel `ObservableGauge` callbacks for load factors (from `NetworkOPs.cpp` line ~2694):
+  - `load_factor` — combined transaction cost multiplier
+  - `load_factor_server` — server + cluster + network contribution
+  - `load_factor_local` — local server load only
+  - `load_factor_net` — network-wide load estimate
+  - `load_factor_cluster` — cluster peer load
+  - `load_factor_fee_escalation` — open ledger fee escalation
+  - `load_factor_fee_queue` — queue entry fee level
+
+- These overlap with some existing StatsD metrics but provide finer granularity (individual factor breakdown vs. combined value).
+
+**Key modified files**:
+
+- `src/xrpld/telemetry/MetricsRegistry.cpp`
+- `src/xrpld/app/misc/NetworkOPs.cpp` (expose load factor accessors if needed)
+
+**Derived Prometheus metrics**: `xrpld_load_factor`, `xrpld_load_factor_fee_escalation`, etc.
+
+**Grafana dashboard**: Add "Load Factor Breakdown" panel to _Fee Market & TxQ_ dashboard.
+
+---
+
+## Task 9.7a: push_metrics.py Parity — Missing Observable Gauges
+
+**Objective**: Fill the remaining metric gaps between the external `push_metrics.py` script (in `ripplex-ansible`) and the internal OTel `MetricsRegistry` observable gauges. After this task, all metrics collected by `push_metrics.py` that CAN be collected internally are covered.
+
+**What was done**:
+
+- Extended existing `cacheHitRateGauge_` callback with `AL_size` (AcceptedLedger cache size)
+- Extended existing `nodeStoreGauge_` callback with 4 new metrics from `getCountsJson()`:
+  - `node_reads_duration_us` (JSON string — uses `std::stoll(asString())`)
+  - `read_request_bundle` (native JSON int)
+  - `read_threads_running` (native JSON int)
+  - `read_threads_total` (native JSON int)
+- Added new `xrpld_server_info` Int64ObservableGauge with 8 metrics:
+  - `server_state` — operating mode as int (0=DISCONNECTED .. 4=FULL)
+  - `uptime` — seconds since server start
+  - `peers` — total peer count
+  - `validated_ledger_seq` — validated ledger sequence (atomic read)
+  - `ledger_current_index` — current open ledger sequence
+  - `peer_disconnects_resources` — cumulative resource-related disconnects
+  - `last_close_proposers` — from `getConsensusInfo()["previous_proposers"]`
+  - `last_close_converge_time_ms` — from `getConsensusInfo()["previous_mseconds"]`
+- Added new `xrpld_build_info` Int64ObservableGauge (info-style, value=1 with `version` label)
+- Added new `xrpld_complete_ledgers` Int64ObservableGauge parsing comma-separated ranges into `{bound, index}` pairs
+- Added new `xrpld_db_metrics` Int64ObservableGauge with 4 metrics:
+  - `db_kb_total`, `db_kb_ledger`, `db_kb_transaction` (SQLite stat queries)
+  - `historical_perminute` (historical ledger fetch rate)
+
+**Key modified files**:
+
+- `src/xrpld/telemetry/MetricsRegistry.h` (4 new gauge members, updated ASCII diagram)
+- `src/xrpld/telemetry/MetricsRegistry.cpp` (4 new callback registrations, 2 callback extensions)
+
+**Not implementable inside xrpld**:
+
+- `connection_count_51233/51234` — OS-level port connection counts from external shell script (`get_connection.sh`)
+
+**Derived Prometheus metrics**: `xrpld_server_info{metric="server_state"}`, `xrpld_build_info{version="2.4.0"}`, `xrpld_complete_ledgers{bound="start",index="0"}`, `xrpld_db_metrics{metric="db_kb_total"}`, etc.
+
+**Grafana dashboard**: New panels added to _Node Health_ dashboard (`node-health.json`).
+
+---
+
+## Task 9.8: New Grafana Dashboards
+
+**Objective**: Create Grafana dashboards for the new metric categories.
+
+**What to do**:
+
+- Create 2 new dashboards:
+  1. **Fee Market & TxQ** (`fee-market`) — TxQ depth/capacity, fee levels, load factor breakdown, fee escalation timeline
+  2. **Job Queue Analysis** (`job-queue`) — Per-job-type rates, queue wait times, execution times, job queue depth
+
+- Update 2 existing dashboards:
+  1. **Node Health** (`xrpld-statsd-node-health`) — Add NodeStore I/O panels, cache hit rate panels, object instance counts
+  2. **RPC Performance** (`rpc-performance`) — Add per-method RPC breakdown panels
+
+**Key modified files**:
+
+- New: `docker/telemetry/grafana/dashboards/rippled-fee-market.json`
+- New: `docker/telemetry/grafana/dashboards/rippled-job-queue.json`
+- `docker/telemetry/grafana/dashboards/rippled-statsd-node-health.json`
+- `docker/telemetry/grafana/dashboards/rippled-rpc-perf.json`
+
+---
+
+## Task 9.9: Update Documentation
+
+**Objective**: Update telemetry reference docs with all new metrics.
+
+**What to do**:
+
+- Update `OpenTelemetryPlan/09-data-collection-reference.md`:
+  - Add new section for OTel SDK-exported metrics (NodeStore, cache, TxQ, PerfLog, CountedObjects, load factors)
+  - Update Grafana dashboard reference table (add 2 new dashboards)
+  - Add Prometheus query examples for new metrics
+
+- Update `docs/telemetry-runbook.md`:
+  - Add an Alerting section covering the provisioned rules and how to wire a receiver
+  - Add troubleshooting entries for new metric categories
+
+- Provision Grafana alert rules (`docker/telemetry/grafana/provisioning/alerting/`):
+  - 6 rules in 3 groups — consensus/ledger (`LedgerHistoryMismatch`, `LedgerCloseStalled`), validator (`ValidationsMissed`, `ValidationsNotChecked`), job queue (`JobQueueTxOverflow`, `JobQueueLatencyHigh`)
+  - `xrpld-default` webhook contact point + flat notification policy; auto-loaded via the existing `provisioning/` mount (no docker-compose change)
+  - Alerting operator docs (per-alert meaning, tuning, receiver wiring) now live in the Alerting section of `docs/telemetry-runbook.md`
+
+**Key modified files**:
+
+- `OpenTelemetryPlan/09-data-collection-reference.md`
+- `docs/telemetry-runbook.md`
+- `docker/telemetry/grafana/provisioning/alerting/{rules,contactpoints,policies}.yaml` (new)
+- `docs/telemetry-runbook.md` (Alerting section added)
+
+---
+
+## Task 9.10: Integration Tests
+
+**Objective**: Verify all new metrics appear in Prometheus after a test workload.
+
+**What to do**:
+
+- Extend the existing telemetry integration test:
+  - Start xrpld with `[telemetry] enabled=1` and `[insight] server=otel`
+  - Submit a batch of RPC calls and transactions
+  - Query Prometheus for each new metric family
+  - Assert non-zero values for: NodeStore reads, cache hit rates, TxQ count, PerfLog RPC counters, object counts, load factors
+
+- Add unit tests for the `MetricsRegistry` class:
+  - Verify callback registration and deregistration
+  - Verify metric values match `get_counts` JSON output
+  - Verify graceful behavior when telemetry is disabled
+
+**Key modified files**:
+
+- `src/test/telemetry/MetricsRegistry_test.cpp` (new)
+- Existing integration test script (extend assertions)
+
+---
+
+## Task 9.11: Validator Health Dashboard (External Dashboard Parity)
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) — dashboards for Phase 7 metrics inspired by the community [xrpl-validator-dashboard](https://github.com/realgrapedrop/xrpl-validator-dashboard).
+>
+> **Upstream**: Phase 7 Tasks 7.9-7.16 (metrics must be emitting).
+> **Downstream**: Phase 10 (dashboard load checks), Phase 11 (alert rules reference these panels).
+
+**Objective**: Create a Grafana dashboard for validation agreement, amendment/UNL health, and state tracking.
+
+**Dashboard**: `validator-health.json`
+
+| Panel                      | Type       | PromQL                                                         |
+| -------------------------- | ---------- | -------------------------------------------------------------- |
+| Agreement % (1h)           | stat       | `xrpld_validation_agreement{metric="agreement_pct_1h"}`        |
+| Agreement % (24h)          | stat       | `xrpld_validation_agreement{metric="agreement_pct_24h"}`       |
+| Agreements vs Missed (1h)  | bargauge   | `agreements_1h` and `missed_1h` side by side                   |
+| Agreements vs Missed (24h) | bargauge   | `agreements_24h` and `missed_24h` side by side                 |
+| Validation Rate            | stat       | `rate(xrpld_validations_sent_total[5m]) * 60`                  |
+| Validations Checked Rate   | stat       | `rate(xrpld_validations_checked_total[5m]) * 60`               |
+| Amendment Blocked          | stat       | `xrpld_validator_health{metric="amendment_blocked"}`           |
+| UNL Expiry (days)          | stat       | `xrpld_validator_health{metric="unl_expiry_days"}`             |
+| Validation Quorum          | stat       | `xrpld_validator_health{metric="validation_quorum"}`           |
+| State Value Timeline       | timeseries | `xrpld_state_tracking{metric="state_value"}`                   |
+| Time in Current State      | stat       | `xrpld_state_tracking{metric="time_in_current_state_seconds"}` |
+| State Changes Rate         | stat       | `rate(xrpld_state_changes_total[1h])`                          |
+| Ledgers Closed Rate        | stat       | `rate(xrpld_ledgers_closed_total[5m]) * 60`                    |
+
+**Dashboard conventions**: `$node` template variable for `service_instance_id` filtering, dark theme, matching existing panel sizes and color schemes.
+
+**Key new files**: `docker/telemetry/grafana/dashboards/rippled-validator-health.json`
+
+**Exit Criteria**:
+
+- [ ] All 13 panels render with non-zero data during normal operation
+- [ ] `$node` filter works correctly for multi-node deployments
+- [ ] Amendment blocked and UNL expiry panels use color thresholds (red=blocked/expiring)
+
+---
+
+## Task 9.12: Peer Quality Dashboard (External Dashboard Parity)
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
+
+**Objective**: Create a Grafana dashboard for peer health aggregates.
+
+**Dashboard**: `peer-quality.json`
+
+| Panel                  | Type       | PromQL                                                         |
+| ---------------------- | ---------- | -------------------------------------------------------------- |
+| P90 Peer Latency       | timeseries | `xrpld_peer_quality{metric="peer_latency_p90_ms"}`             |
+| Insane/Diverged Peers  | stat       | `xrpld_peer_quality{metric="peers_insane_count"}`              |
+| Higher Version Peers % | stat       | `xrpld_peer_quality{metric="peers_higher_version_pct"}`        |
+| Upgrade Recommended    | stat       | `xrpld_peer_quality{metric="upgrade_recommended"}`             |
+| Resource Disconnects   | timeseries | `xrpld_Overlay_Peer_Disconnects_Charges`                       |
+| Inbound vs Outbound    | bargauge   | `xrpld_Peer_Finder_Active_Inbound_Peers`, `..._Outbound_Peers` |
+
+**Key new files**: `docker/telemetry/grafana/dashboards/rippled-peer-quality.json`
+
+**Exit Criteria**:
+
+- [ ] All 6 panels render correctly
+- [ ] P90 latency panel shows trend over time
+- [ ] Upgrade recommended panel uses color threshold (red=1, green=0)
+
+---
+
+## Task 9.13: Ledger Economy Dashboard Panels (External Dashboard Parity)
+
+> **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity)
+
+**Objective**: Add "Ledger Economy" row to the existing `node-health.json` dashboard.
+
+| Panel                | Type       | PromQL                                              |
+| -------------------- | ---------- | --------------------------------------------------- |
+| Base Fee (drops)     | stat       | `xrpld_ledger_economy{metric="base_fee_xrp"}`       |
+| Reserve Base (drops) | stat       | `xrpld_ledger_economy{metric="reserve_base_xrp"}`   |
+| Reserve Inc (drops)  | stat       | `xrpld_ledger_economy{metric="reserve_inc_xrp"}`    |
+| Ledger Age           | stat       | `xrpld_ledger_economy{metric="ledger_age_seconds"}` |
+| Transaction Rate     | timeseries | `xrpld_ledger_economy{metric="transaction_rate"}`   |
+
+**Key modified files**: `docker/telemetry/grafana/dashboards/node-health.json`
+
+**Exit Criteria**:
+
+- [ ] 5 new panels render correctly in existing dashboard
+- [ ] Fee values match `server_info` RPC output
+- [ ] Transaction rate shows smooth trend (not spiky)
+
+---
+
+## Exit Criteria
+
+- [ ] All ~50 new metrics visible in Prometheus via OTLP pipeline
+- [ ] `MetricsRegistry` class registers/deregisters cleanly with OTel SDK
+- [ ] Async gauge callbacks execute at 10s intervals without performance impact
+- [ ] 2 new Grafana dashboards operational (Fee Market, Job Queue)
+- [ ] 2 existing dashboards updated with new panel groups
+- [ ] Integration test validates all new metric families are non-zero
+- [ ] No performance regression (< 0.5% CPU overhead from new callbacks)
+- [ ] Documentation updated with full new metric inventory
+- [ ] Validator Health dashboard renders all 13 panels
+- [ ] Peer Quality dashboard renders all 6 panels
+- [ ] Ledger Economy panels added to node-health dashboard
+
+---
+
+## Appendix: Alerting Design
+
+> Design for the provisioned Grafana alert rules (Task 9.9a). Previously a standalone spec; merged here so the phase plan is self-contained.
+
+**Date:** 2026-07-06
+**Branch:** `pratik/otel-phase9-metric-gap-fill` (PR #6513, Jira RIPD-5187)
+**Status:** Approved
+
+### Purpose
+
+Phase 9 exports ~68 internal xrpld metrics and ships Grafana dashboards for
+them. This adds the missing operator-facing piece: **provisioned Grafana alert
+rules** that fire on the health-critical metrics phase 9 introduces. The
+phase-9 task list (line 311) and Jira story RIPD-5187 both already list
+"alerting rules" as a phase-9 deliverable, so this closes that gap.
+
+Scope is deliberately narrow — the three subsystems whose failure is
+node-fatal: **consensus/ledger health, validator health, job queue**. RPC/API
+health is explicitly out of scope.
+
+### Why phase 9 (not phase 11)
+
+Every metric these alerts fire on is _born_ in phase 9
+(`xrpld_ledger_history_mismatch_total`, `xrpld_ledgers_closed_total`,
+`xrpld_validation_missed_total`, `xrpld_validations_checked_total`,
+`xrpld_jq_trans_overflow_total`, `xrpld_job_queued_duration_us_bucket`). Alerts
+belong with the metrics they watch, and this is where the dependency lives.
+
+### Delivery
+
+Provisioned YAML, version-controlled — matching the existing datasource /
+dashboard provisioning pattern. No docker-compose change: the Grafana service
+already mounts `./grafana/provisioning:/etc/grafana/provisioning:ro`, and
+Grafana auto-loads `provisioning/alerting/*.yaml`.
+
+New files under `docker/telemetry/grafana/provisioning/alerting/`:
+
+| File                 | Purpose                                                                                                             |
+| -------------------- | ------------------------------------------------------------------------------------------------------------------- |
+| `contactpoints.yaml` | One contact point `xrpld-default` (webhook to a documented placeholder; comments show how to swap for Slack/email). |
+| `policies.yaml`      | Default notification policy: route all alerts → `xrpld-default`, grouped by `alertname` + `service_instance_id`.    |
+| `rules.yaml`         | 6 alert rules across 3 groups (below).                                                                              |
+
+Plus the Alerting section of `docs/telemetry-runbook.md` — operator runbook:
+what each alert means, likely causes, and how to point the contact point at a
+real receiver.
+
+### Alert rules
+
+All rules target Prometheus datasource `uid: prometheus`. Each rule uses the
+Grafana rule shape: query (A) → reduce (B, last value) → threshold (C). All
+`rate()`/`histogram_quantile()` expressions aggregate with
+`sum by (service_instance_id)` (or `+ le`) so **each node alerts independently**.
+Alert rules run headless, so they cannot use the dashboards' `$node` template
+variables — they match all series and group by `service_instance_id` instead.
+
+| Group     | Alert                 | Expression (5m window)                                                                                      | Fires                 | `for` | severity |
+| --------- | --------------------- | ----------------------------------------------------------------------------------------------------------- | --------------------- | ----- | -------- |
+| Consensus | LedgerHistoryMismatch | `sum by (service_instance_id)(rate(xrpld_ledger_history_mismatch_total[5m]))`                               | `> 0`                 | 5m    | critical |
+| Consensus | LedgerCloseStalled    | `sum by (service_instance_id)(rate(xrpld_ledgers_closed_total[5m]))`                                        | `< 0.001` (≈0)        | 3m    | critical |
+| Validator | ValidationsMissed     | `sum by (service_instance_id)(rate(xrpld_validation_missed_total[5m]))`                                     | `> 0`                 | 5m    | warning  |
+| Validator | ValidationsNotChecked | `sum by (service_instance_id)(rate(xrpld_validations_checked_total[5m]))`                                   | `< 0.001` (≈0)        | 5m    | warning  |
+| Job queue | JobQueueTxOverflow    | `sum by (service_instance_id)(rate(xrpld_jq_trans_overflow_total[5m]))`                                     | `> 0`                 | 5m    | warning  |
+| Job queue | JobQueueLatencyHigh   | `histogram_quantile(0.99, sum by (le, service_instance_id)(rate(xrpld_job_queued_duration_us_bucket[5m])))` | `> 1000000` (µs = 1s) | 5m    | warning  |
+
+Each rule carries labels `severity` and `category` (consensus/validator/jobqueue)
+and annotations `summary` + `description` (with `{{ $labels.service_instance_id }}`
+and `{{ $values.B.Value }}` interpolation).
+
+#### Threshold rationale
+
+- **LedgerCloseStalled `< 0.001` for 3m**: healthy nodes close a ledger every
+  ~3-5s; a 5m rate decaying to ~0 means the node is stuck. The epsilon (not
+  exact `0`) avoids float rate-noise suppressing the alert.
+- **JobQueueLatencyHigh 1s p99**: a default starting point, easy to tune — jobs
+  queued >1s at p99 indicate the node is saturated.
+- Others are `> 0` on error/miss counters: any sustained nonzero rate is
+  actionable.
+
+### Non-goals / YAGNI
+
+- No per-alert silencing schedules, no mute timings.
+- No RPC/API, overlay, or fee-market alerts (dashboards cover those visually).
+- Single contact point — multi-receiver routing is left to the operator.
+
+### Verification
+
+1. `yamllint` (or `python -c yaml.safe_load`) on all three YAML files.
+2. `docker compose -f docker/telemetry/docker-compose.yml config -q` still parses.
+3. Optional live check: start stack, `GET /api/v1/provisioning/alert-rules`
+   returns the 6 rules; Grafana logs show no provisioning errors.
+4. Code-review pass (subagent) against phase conventions before commit.

cmake/XrplCompiler.cmake

@@ -242,7 +242,7 @@ elseif(use_gold AND is_gcc)
        disabling would be to figure out all the settings
        required to make gold play nicely with jemalloc.
     #]=========================================================]
-    if(("${LD_VERSION}" MATCHES "GNU gold") AND (NOT jemalloc))
+    if(("${LD_VERSION}" MATCHES "GNU gold") AND (NOT jemalloc) AND (NOT tcmalloc))
         target_link_libraries(
             common
             INTERFACE

cmake/XrplCore.cmake

@@ -78,6 +78,19 @@ include(target_link_modules)
 # Level 01
 add_module(xrpl beast)
 target_link_libraries(xrpl.libxrpl.beast PUBLIC xrpl.imports.main)
+# OTelCollector in beast/insight uses the OTel Metrics SDK when telemetry is
+# enabled. Link the Conan-provided umbrella target rather than individual
+# component targets: the OTel package's per-component dependency graph is
+# under-declared (e.g. the OTLP client references sdk::common symbols without
+# declaring the edge), so naming components directly reorders the static-link
+# line into an unresolvable state. The umbrella carries the full, internally
+# consistent graph the package authors validated.
+if(telemetry)
+    target_link_libraries(
+        xrpl.libxrpl.beast
+        PUBLIC opentelemetry-cpp::opentelemetry-cpp
+    )
+endif()
 
 include(GitInfo)
 add_module(xrpl git)
@@ -212,6 +225,11 @@ target_link_libraries(
     PRIVATE xrpl.libxrpl.protocol
 )
 if(telemetry)
+    # Telemetry owns both the trace and (as of the direct-metrics API) the
+    # metrics pipeline. Link the umbrella target: it supplies the trace and
+    # metrics SDK components with the correct static-link ordering, which
+    # naming components individually does not (the package under-declares
+    # inter-component dependencies).
     target_link_libraries(
         xrpl.libxrpl.telemetry
         PUBLIC opentelemetry-cpp::opentelemetry-cpp

cmake/XrplDocs.cmake

@@ -27,8 +27,12 @@ file(
     src/*.cpp
     src/*.md
     Builds/*.md
-    *.md
 )
+# Add only top-level .md files (README, CONTRIBUTING, etc.) without
+# recursing into dot-directories like .claude/ whose files are not
+# valid Doxygen/CMake sources.
+file(GLOB doxygen_top_md CONFIGURE_DEPENDS "*.md")
+list(APPEND doxygen_input ${doxygen_top_md})
 list(APPEND doxygen_input external/README.md)
 set(dependencies "${doxygen_input}" "${doxyfile}")
 

cmake/XrplInterface.cmake

@@ -54,6 +54,11 @@ if(jemalloc)
     target_link_libraries(opts INTERFACE jemalloc::jemalloc)
 endif()
 
+if(tcmalloc)
+    find_package(gperftools REQUIRED)
+    target_link_libraries(opts INTERFACE gperftools::gperftools)
+endif()
+
 #[===================================================================[
    xrpld transitive library deps via an interface library
 #]===================================================================]

cmake/XrplSettings.cmake

@@ -125,6 +125,10 @@ else()
 endif()
 
 option(jemalloc "Enables jemalloc for heap profiling" OFF)
+option(tcmalloc "Link xrpld against tcmalloc (gperftools) as the global allocator" OFF)
+if(jemalloc AND tcmalloc)
+    message(FATAL_ERROR "jemalloc and tcmalloc are mutually exclusive")
+endif()
 option(werr "treat warnings as errors" OFF)
 option(
     local_protobuf

conanfile.py

@@ -3,6 +3,7 @@ import re
 from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
 
 from conan import ConanFile
+from conan.errors import ConanInvalidConfiguration
 
 
 class Xrpl(ConanFile):
@@ -21,6 +22,7 @@ class Xrpl(ConanFile):
         "rocksdb": [True, False],
         "shared": [True, False],
         "static": [True, False],
+        "tcmalloc": [True, False],
         "telemetry": [True, False],
         "tests": [True, False],
         "unity": [True, False],
@@ -53,6 +55,7 @@ class Xrpl(ConanFile):
         "rocksdb": True,
         "shared": False,
         "static": True,
+        "tcmalloc": False,
         "telemetry": True,
         "tests": False,
         "unity": False,
@@ -125,6 +128,10 @@ class Xrpl(ConanFile):
                 self.version = match.group(1)
 
     def configure(self):
+        if self.options.tcmalloc and self.options.jemalloc:
+            raise ConanInvalidConfiguration(
+                "jemalloc and tcmalloc cannot both be enabled"
+            )
         if self.settings.compiler == "apple-clang":
             self.options["boost"].visibility = "global"
         if self.settings.compiler in ["clang", "gcc"]:
@@ -135,6 +142,8 @@ class Xrpl(ConanFile):
         self.requires("date/3.0.4", transitive_headers=True)
         if self.options.jemalloc:
             self.requires("jemalloc/5.3.1")
+        if self.options.tcmalloc:
+            self.requires("gperftools/2.16")
         self.requires("lz4/1.10.0", force=True)
         self.requires("mpt-crypto/0.4.0-rc2", transitive_headers=True)
         self.requires("protobuf/6.33.5", force=True)
@@ -171,6 +180,7 @@ class Xrpl(ConanFile):
         tc.variables["assert"] = self.options.assertions
         tc.variables["coverage"] = self.options.coverage
         tc.variables["jemalloc"] = self.options.jemalloc
+        tc.variables["tcmalloc"] = self.options.tcmalloc
         tc.variables["rocksdb"] = self.options.rocksdb
         tc.variables["BUILD_SHARED_LIBS"] = self.options.shared
         tc.variables["static"] = self.options.static

docker/telemetry/.env.alerting.example

@@ -0,0 +1,32 @@
+# rippled OTel alerting delivery — copy to `.env.alerting` and fill in.
+#
+# `.env.alerting` is gitignored; never commit a real Slack webhook or address.
+# Grafana reads these when the stack starts and expands the ${VARS} referenced
+# in grafana/provisioning/alerting/contactpoints.yaml. See the Alerting
+# section of docs/telemetry-runbook.md.
+#
+# Any var left blank simply disables that delivery path — the stack still runs.
+
+# --- Slack ---
+# Incoming-webhook URL from the Slack app (Incoming Webhooks feature).
+# Used by both the warning (xrpld-default) and critical (xrpld-critical) tiers.
+SLACK_WEBHOOK_URL=
+
+# Channel label. With an incoming webhook the target channel is fixed by the
+# webhook itself; this only satisfies Grafana's Slack validator. Defaults to
+# #xrpld-alerts if unset.
+SLACK_CHANNEL=#xrpld-alerts
+
+# --- Email (critical tier only) ---
+# Comma-separated recipient list for critical alerts.
+ALERT_EMAIL_TO=
+
+# SMTP relay Grafana sends through. Email only delivers when SMTP is enabled
+# and these point at a real relay (e.g. an internal smart host or an
+# authenticated provider).
+GF_SMTP_ENABLED=false
+GF_SMTP_HOST=smtp.example.com:587
+GF_SMTP_USER=
+GF_SMTP_PASSWORD=
+GF_SMTP_FROM_ADDRESS=alerts@xrpld.local
+GF_SMTP_FROM_NAME=xrpld Alerts

docker/telemetry/.env.grafanacloud.example

@@ -0,0 +1,16 @@
+# Grafana Cloud OTLP credentials — copy to `.env.grafanacloud` and fill in.
+# Find all three under Grafana Cloud -> Connections -> "OpenTelemetry (OTLP)".
+#
+# `.env.grafanacloud` is gitignored; never commit real tokens.
+
+# OTLP/HTTP gateway URL for your stack, including the /otlp path.
+# Example: https://otlp-gateway-prod-us-east-0.grafana.net/otlp
+GRAFANA_CLOUD_OTLP_ENDPOINT=
+
+# Numeric instance/stack id shown on the OTLP connection page
+# (used as the Basic-auth username).
+GRAFANA_CLOUD_INSTANCE_ID=
+
+# A Cloud Access Policy token with metrics:write, traces:write, logs:write
+# (used as the Basic-auth password).
+GRAFANA_CLOUD_API_TOKEN=

docker/telemetry/.gitignore

@@ -0,0 +1,9 @@
+# Runtime data generated by xrpld and telemetry stack
+data/
+
+# Env. file carrying environmental setup data for local or cloud runs.
+.env.*
+
+# Keep examples
+!.env.alerting.example
+!.env.grafanacloud.example

docker/telemetry/TESTING.md

@@ -0,0 +1,708 @@
+# OpenTelemetry Integration Testing Guide
+
+This document describes how to verify the xrpld OpenTelemetry telemetry
+pipeline end-to-end, from span generation through the observability stack
+(otel-collector, Tempo, Prometheus, Grafana).
+
+---
+
+## Prerequisites
+
+### Build xrpld with telemetry
+
+```bash
+conan install . --build=missing -o telemetry=True
+cmake --preset default -Dtelemetry=ON
+cmake --build --preset default --target xrpld
+```
+
+The binary is at `.build/xrpld`.
+
+### Required tools
+
+- **Docker** with `docker compose` (v2)
+- **curl**
+- **jq** (JSON processor)
+
+### Verify binary
+
+```bash
+.build/xrpld --version
+```
+
+---
+
+## Test 1: Single-Node Standalone (Quick Verification)
+
+This test verifies RPC and transaction spans in standalone mode. Consensus
+spans will not fire because standalone mode does not run consensus.
+
+### Step 1: Start the observability stack
+
+```bash
+docker compose -f docker/telemetry/docker-compose.yml up -d
+```
+
+Wait for services to be ready:
+
+```bash
+# otel-collector health
+curl -sf http://localhost:13133/ && echo "collector ready"
+
+# Tempo readiness
+curl -sf http://localhost:3200/ready >/dev/null && echo "tempo ready"
+```
+
+### Step 2: Start xrpld in standalone mode
+
+```bash
+.build/xrpld --conf docker/telemetry/xrpld-telemetry.cfg -a --start
+```
+
+Wait a few seconds for the node to initialize.
+
+### Step 3: Exercise RPC spans
+
+```bash
+# server_info
+curl -s http://localhost:5005 \
+    -d '{"method":"server_info"}' | jq .result.info.server_state
+
+# server_state
+curl -s http://localhost:5005 \
+    -d '{"method":"server_state"}' | jq .result.state.server_state
+
+# ledger
+curl -s http://localhost:5005 \
+    -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' |
+    jq .result.ledger_current_index
+```
+
+### Step 4: Submit a transaction
+
+Close the ledger first (required in standalone mode):
+
+```bash
+curl -s http://localhost:5005 -d '{"method":"ledger_accept"}'
+```
+
+Submit a Payment from the genesis account:
+
+```bash
+curl -s http://localhost:5005 -d '{
+  "method": "submit",
+  "params": [{
+    "secret": "snoPBrXtMeMyMHUVTgbuqAfg1SUTb",
+    "tx_json": {
+      "TransactionType": "Payment",
+      "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh",
+      "Destination": "rPMh7Pi9ct699iZUTWzJaUMR1o42VEfGqF",
+      "Amount": "10000000"
+    }
+  }]
+}' | jq .result.engine_result
+```
+
+Expected result: `"tesSUCCESS"`.
+
+Close the ledger again to finalize:
+
+```bash
+curl -s http://localhost:5005 -d '{"method":"ledger_accept"}'
+```
+
+### Step 5: Verify traces in Tempo
+
+Wait 5 seconds for the batch export, then:
+
+```bash
+TEMPO="http://localhost:3200"
+
+# Check xrpld service is registered
+curl -s "$TEMPO/api/v2/search/tag/resource.service.name/values" | jq '.tagValues[].value'
+
+# Check RPC spans
+curl -s "$TEMPO/api/search" \
+    --data-urlencode 'q={resource.service.name="xrpld" && name="rpc.http_request"}' \
+    --data-urlencode 'limit=5' | jq '.traces | length'
+
+curl -s "$TEMPO/api/search" \
+    --data-urlencode 'q={resource.service.name="xrpld" && name="rpc.process"}' \
+    --data-urlencode 'limit=5' | jq '.traces | length'
+
+curl -s "$TEMPO/api/search" \
+    --data-urlencode 'q={resource.service.name="xrpld" && name="rpc.command.server_info"}' \
+    --data-urlencode 'limit=5' | jq '.traces | length'
+
+# Check transaction spans
+curl -s "$TEMPO/api/search" \
+    --data-urlencode 'q={resource.service.name="xrpld" && name="tx.process"}' \
+    --data-urlencode 'limit=5' | jq '.traces | length'
+```
+
+Or open Grafana Explore with Tempo datasource: http://localhost:3000
+
+### Step 6: Teardown
+
+```bash
+# Kill xrpld (Ctrl+C or)
+kill $(pgrep -f 'xrpld.*xrpld-telemetry')
+
+# Stop observability stack
+docker compose -f docker/telemetry/docker-compose.yml down
+
+# Clean xrpld data
+rm -rf data/
+```
+
+### Expected spans (standalone mode)
+
+| Span Name                   | Expected | Notes                         |
+| --------------------------- | -------- | ----------------------------- |
+| `rpc.http_request`          | Yes      | Every HTTP RPC call           |
+| `rpc.process`               | Yes      | Every RPC processing          |
+| `rpc.command.server_info`   | Yes      | server_info RPC               |
+| `rpc.command.server_state`  | Yes      | server_state RPC              |
+| `rpc.command.ledger`        | Yes      | ledger RPC                    |
+| `rpc.command.submit`        | Yes      | submit RPC                    |
+| `rpc.command.ledger_accept` | Yes      | ledger_accept RPC             |
+| `tx.process`                | Yes      | Transaction submission        |
+| `tx.receive`                | No       | No peers in standalone        |
+| `consensus.*`               | No       | Consensus disabled standalone |
+
+---
+
+## Test 2: 6-Node Consensus Network (Full Verification)
+
+This test verifies ALL span categories including consensus and peer
+transaction relay, using a 6-node validator network.
+
+### Automated
+
+Run the integration test script:
+
+```bash
+bash docker/telemetry/integration-test.sh
+```
+
+The script will:
+
+1. Start the observability stack
+2. Generate 6 validator key pairs
+3. Create config files for each node
+4. Start all 6 nodes
+5. Wait for consensus ("proposing" state)
+6. Exercise RPC, submit transactions
+7. Verify all span categories in Tempo
+8. Verify spanmetrics in Prometheus
+9. Print results and leave the stack running
+
+### Manual
+
+If you prefer to run the steps manually:
+
+#### Step 1: Start observability stack
+
+```bash
+docker compose -f docker/telemetry/docker-compose.yml up -d
+```
+
+#### Step 2: Generate validator keys
+
+Start a temporary standalone xrpld:
+
+```bash
+.build/xrpld --conf docker/telemetry/xrpld-telemetry.cfg -a --start &
+TEMP_PID=$!
+sleep 5
+```
+
+Generate 6 key pairs:
+
+```bash
+for i in $(seq 1 6); do
+    curl -s http://localhost:5005 \
+        -d '{"method":"validation_create"}' | jq '.result'
+done
+```
+
+Record the `validation_seed` and `validation_public_key` for each.
+Kill the temporary node:
+
+```bash
+kill $TEMP_PID
+rm -rf data/
+```
+
+#### Step 3: Create node configs
+
+For each node (1-6), create a config file. Template:
+
+```ini
+[server]
+port_rpc
+port_peer
+
+[port_rpc]
+port = {5004 + node_number}
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[port_peer]
+port = {51234 + node_number}
+ip = 0.0.0.0
+protocol = peer
+
+[node_db]
+type=NuDB
+path=/tmp/xrpld-integration/node{N}/nudb
+online_delete=256
+
+[database_path]
+/tmp/xrpld-integration/node{N}/db
+
+[debug_logfile]
+/tmp/xrpld-integration/node{N}/debug.log
+
+[validation_seed]
+{seed from step 2}
+
+[validators_file]
+/tmp/xrpld-integration/validators.txt
+
+[ips_fixed]
+127.0.0.1 51235
+127.0.0.1 51236
+127.0.0.1 51237
+127.0.0.1 51238
+127.0.0.1 51239
+127.0.0.1 51240
+
+[peer_private]
+1
+
+[telemetry]
+enabled=1
+endpoint=http://localhost:4318/v1/traces
+batch_size=512
+batch_delay_ms=2000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1
+
+[rpc_startup]
+{ "command": "log_level", "severity": "warning" }
+
+[ssl_verify]
+0
+```
+
+#### Step 4: Create validators.txt
+
+```ini
+[validators]
+{public_key_1}
+{public_key_2}
+{public_key_3}
+{public_key_4}
+{public_key_5}
+{public_key_6}
+```
+
+#### Step 5: Start all 6 nodes
+
+```bash
+for i in $(seq 1 6); do
+    .build/xrpld --conf /tmp/xrpld-integration/node$i/xrpld.cfg --start &
+    echo $! >/tmp/xrpld-integration/node$i/xrpld.pid
+done
+```
+
+#### Step 6: Wait for consensus
+
+Poll each node until `server_state` = `"proposing"`:
+
+```bash
+for port in 5005 5006 5007 5008 5009 5010; do
+    while true; do
+        state=$(curl -s http://localhost:$port \
+            -d '{"method":"server_info"}' |
+            jq -r '.result.info.server_state')
+        echo "Port $port: $state"
+        [ "$state" = "proposing" ] && break
+        sleep 5
+    done
+done
+```
+
+#### Step 7: Exercise RPC and submit transaction
+
+```bash
+# RPC calls
+curl -s http://localhost:5005 -d '{"method":"server_info"}'
+curl -s http://localhost:5005 -d '{"method":"server_state"}'
+curl -s http://localhost:5005 -d '{"method":"ledger","params":[{"ledger_index":"current"}]}'
+
+# Submit transaction
+curl -s http://localhost:5005 -d '{
+  "method": "submit",
+  "params": [{
+    "secret": "snoPBrXtMeMyMHUVTgbuqAfg1SUTb",
+    "tx_json": {
+      "TransactionType": "Payment",
+      "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh",
+      "Destination": "rPMh7Pi9ct699iZUTWzJaUMR1o42VEfGqF",
+      "Amount": "10000000"
+    }
+  }]
+}'
+```
+
+Wait 15 seconds for consensus and batch export.
+
+#### Step 8: Verify in Tempo
+
+See the "Verification Queries" section below.
+
+---
+
+## Expected Span Catalog
+
+All 16 production span names instrumented across Phases 2-5:
+
+| Span Name                   | Source File       | Phase | Key Attributes                                                                           | How to Trigger            |
+| --------------------------- | ----------------- | ----- | ---------------------------------------------------------------------------------------- | ------------------------- |
+| `rpc.http_request`          | ServerHandler.cpp | 2     | --                                                                                       | Any HTTP RPC call         |
+| `rpc.ws_upgrade`            | ServerHandler.cpp | 2     | --                                                                                       | WebSocket upgrade         |
+| `rpc.ws_message`            | ServerHandler.cpp | 2     | --                                                                                       | WebSocket RPC message     |
+| `rpc.process`               | ServerHandler.cpp | 2     | --                                                                                       | RPC processing            |
+| `rpc.command.<name>`        | RPCHandler.cpp    | 2     | `command`, `version`, `rpc_role`                                                         | Any RPC command           |
+| `tx.process`                | NetworkOPs.cpp    | 3     | `xrpl.tx.hash`, `local`, `path`                                                          | Submit transaction        |
+| `tx.receive`                | PeerImp.cpp       | 3     | `xrpl.peer.id`                                                                           | Peer relays transaction   |
+| `consensus.proposal.send`   | RCLConsensus.cpp  | 4     | `xrpl.consensus.round`                                                                   | Consensus proposing phase |
+| `consensus.ledger_close`    | RCLConsensus.cpp  | 4     | `xrpl.consensus.ledger.seq`, `xrpl.consensus.mode`                                       | Ledger close event        |
+| `consensus.accept`          | RCLConsensus.cpp  | 4     | `xrpl.consensus.proposers`, `xrpl.consensus.round_time_ms`                               | Ledger accepted           |
+| `consensus.validation.send` | RCLConsensus.cpp  | 4     | `xrpl.consensus.ledger.seq`, `xrpl.consensus.proposing`                                  | Validation sent           |
+| `consensus.accept.apply`    | RCLConsensus.cpp  | 4     | `xrpl.consensus.close_time`, `close_time_correct`, `close_resolution_ms`, `state`        | Ledger apply + close time |
+| `tx.apply`                  | BuildLedger.cpp   | 5     | `xrpl.ledger.tx_count`, `xrpl.ledger.tx_failed`                                          | Ledger close (tx set)     |
+| `ledger.build`              | BuildLedger.cpp   | 5     | `xrpl.ledger.seq`, `xrpl.ledger.close_time`, `close_time_correct`, `close_resolution_ms` | Ledger build              |
+| `ledger.validate`           | LedgerMaster.cpp  | 5     | `xrpl.ledger.seq`, `xrpl.ledger.validations`                                             | Ledger validated          |
+| `ledger.store`              | LedgerMaster.cpp  | 5     | `xrpl.ledger.seq`                                                                        | Ledger stored             |
+| `peer.proposal.receive`     | PeerImp.cpp       | 5     | `xrpl.peer.id`, `proposal_trusted`                                                       | Peer sends proposal       |
+| `peer.validation.receive`   | PeerImp.cpp       | 5     | `xrpl.peer.id`, `validation_trusted`                                                     | Peer sends validation     |
+
+---
+
+## Verification Queries
+
+### Tempo API
+
+Base URL: `http://localhost:3200`
+
+```bash
+TEMPO="http://localhost:3200"
+
+# List all services
+curl -s "$TEMPO/api/v2/search/tag/resource.service.name/values" | jq '.tagValues[].value'
+
+# Query traces by operation
+for op in "rpc.http_request" "rpc.ws_upgrade" "rpc.ws_message" "rpc.process" \
+    "rpc.command.server_info" "rpc.command.server_state" "rpc.command.ledger" \
+    "tx.process" "tx.receive" "tx.apply" \
+    "consensus.proposal.send" "consensus.ledger_close" \
+    "consensus.accept" "consensus.accept.apply" \
+    "consensus.validation.send" \
+    "ledger.build" "ledger.validate" "ledger.store" \
+    "peer.proposal.receive" "peer.validation.receive"; do
+    count=$(curl -s "$TEMPO/api/search" \
+        --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \
+        --data-urlencode "limit=5" |
+        jq '.traces | length')
+    printf "%-35s %s traces\n" "$op" "$count"
+done
+```
+
+### Prometheus API
+
+Base URL: `http://localhost:9090`
+
+```bash
+PROM="http://localhost:9090"
+
+# Span call counts (from spanmetrics connector)
+curl -s "$PROM/api/v1/query?query=span_calls_total" |
+    jq '.data.result[] | {span: .metric.span_name, count: .value[1]}'
+
+# Latency histogram
+curl -s "$PROM/api/v1/query?query=span_duration_milliseconds_count" |
+    jq '.data.result[] | {span: .metric.span_name, count: .value[1]}'
+
+# RPC calls by command
+curl -s "$PROM/api/v1/query?query=span_calls_total{span_name=~\"rpc.command.*\"}" |
+    jq '.data.result[] | {command: .metric["command"], count: .value[1]}'
+
+# Deployment-tier labels present on metrics (set by the collector's
+# resource/tier processor and promoted via resource_to_telemetry_conversion).
+# Expect deployment_environment and xrpl_network_type on each series.
+curl -s "$PROM/api/v1/query?query=span_calls_total" |
+    jq '.data.result[0].metric | {deployment_environment, xrpl_network_type, service_name}'
+```
+
+### Grafana
+
+Open http://localhost:3000 (anonymous admin access enabled).
+
+Pre-configured dashboards:
+
+- **RPC Performance**: Request rates, latency percentiles by command, top commands, WebSocket rate
+- **Transaction Overview**: Transaction processing rates, apply duration, peer relay, failed tx rate
+- **Consensus Health**: Consensus round duration, proposer counts, mode tracking, accept heatmap
+- **Ledger Operations**: Build/validate/store rates and durations, TX apply metrics
+- **Peer Network**: Proposal/validation receive rates, trusted vs untrusted breakdown (requires `trace_peer=1`)
+
+Pre-configured datasources:
+
+- **Tempo**: Trace data at `http://tempo:3200`
+- **Prometheus**: Metrics at `http://prometheus:9090`
+- **Loki**: Log data at `http://loki:3100` (via Grafana Explore)
+
+---
+
+## Exporting to Grafana Cloud
+
+Instead of (or alongside) the local backends, the collector can forward
+traces, metrics, and logs to a hosted **Grafana Cloud** stack. This is a
+runtime choice layered on top of the base stack — xrpld and the base
+`docker-compose.yml` are unchanged.
+
+### Step 1: Get Grafana Cloud OTLP credentials
+
+From **Grafana Cloud → Connections → OpenTelemetry (OTLP)**, note the OTLP
+gateway endpoint (ends in `/otlp`), the numeric instance id, and an
+access-policy token with `metrics:write`, `traces:write`, and `logs:write`.
+
+### Step 2: Fill in the env file
+
+```bash
+cp docker/telemetry/.env.grafanacloud.example docker/telemetry/.env.grafanacloud
+# edit .env.grafanacloud:
+#   GRAFANA_CLOUD_OTLP_ENDPOINT=https://otlp-gateway-<zone>.grafana.net/otlp
+#   GRAFANA_CLOUD_INSTANCE_ID=<instance id>
+#   GRAFANA_CLOUD_API_TOKEN=<token>
+```
+
+`.env.grafanacloud` is gitignored — never commit real tokens.
+
+### Step 3: Start the stack with cloud export enabled
+
+```bash
+docker compose -f docker/telemetry/docker-compose.yml \
+    -f docker/telemetry/docker-compose.grafanacloud.yaml up -d
+```
+
+The override swaps the collector onto `otel-collector-config.grafanacloud.yaml`,
+which keeps the local Tempo/Prometheus/Loki exporters and adds one
+OTLP/HTTP exporter to Grafana Cloud on all three pipelines. Bring the stack
+up with just the base file to return to local-only.
+
+### Step 4: Verify data reaches Grafana Cloud
+
+After exercising RPC/transaction workflows (Tests 1 or 2), open your Grafana
+Cloud instance and confirm:
+
+- **Traces**: Explore → hosted Tempo datasource → search `{resource.service.name="xrpld"}`
+- **Metrics**: Explore → hosted Prometheus/Mimir → query `span_calls_total`
+- **Logs**: Explore → hosted Loki → query `{job="xrpld"}` (requires `warning`+ file logging)
+
+If nothing appears, check the collector logs for auth/export errors:
+
+```bash
+docker compose -f docker/telemetry/docker-compose.yml \
+    -f docker/telemetry/docker-compose.grafanacloud.yaml \
+    logs otel-collector | grep -iE 'grafanacloud|401|403|export'
+```
+
+A `401`/`403` means the instance id or token is wrong; a connection error
+means the endpoint URL is wrong or missing the `/otlp` path.
+
+---
+
+## Test 3: Log-Trace Correlation (Phase 8)
+
+Phase 8 injects `trace_id` and `span_id` into xrpld's log output when
+a log line is emitted within an active OTel span. This test verifies the
+end-to-end log-trace correlation pipeline.
+
+### Step 1: Verify trace_id in log output
+
+After running Test 1 or Test 2 (which generate RPC spans), check the
+xrpld debug.log for trace context:
+
+```bash
+grep 'trace_id=[a-f0-9]\{32\} span_id=[a-f0-9]\{16\}' /path/to/debug.log
+```
+
+Expected: log lines with `trace_id=<32hex> span_id=<16hex>` between the
+severity code and the message. Example:
+
+```
+2024-Jan-15 10:30:45.123456 UTC RPCHandler:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Calling server_info
+```
+
+Lines emitted outside of an active span (background tasks, startup) will
+NOT have trace context — this is expected.
+
+### Step 2: Cross-check trace_id in Tempo
+
+Extract a `trace_id` from the log and verify it exists in Tempo:
+
+```bash
+TRACE_ID=$(grep -o 'trace_id=[a-f0-9]\{32\}' /path/to/debug.log | head -1 | cut -d= -f2)
+echo "Checking trace: $TRACE_ID"
+curl -s "http://localhost:3200/api/traces/$TRACE_ID" | jq '.batches | length'
+```
+
+Expected result: `> 0` (the trace exists in Tempo).
+
+### Step 3: Verify Loki log ingestion
+
+The OTel Collector's filelog receiver tails xrpld's debug.log and
+exports parsed entries to Loki. Verify Loki has received entries:
+
+```bash
+# Query Loki for any xrpld logs
+curl -sG "http://localhost:3100/loki/api/v1/query" \
+    --data-urlencode 'query={job="xrpld"}' \
+    --data-urlencode 'limit=5' | jq '.data.result | length'
+```
+
+Expected: > 0 results.
+
+### Step 4: Verify Grafana Tempo-to-Loki correlation
+
+1. Open Grafana at http://localhost:3000
+2. Navigate to **Explore** -> select **Tempo** datasource
+3. Search for a trace (e.g., operation `rpc.command.server_info`)
+4. Click **"Logs for this trace"** in the trace detail view
+5. Verify that Loki log lines appear, filtered by the trace's `trace_id`
+
+### Step 5: Verify Grafana Loki-to-Tempo correlation
+
+1. In Grafana **Explore**, select **Loki** datasource
+2. Query: `{job="xrpld"} |= "trace_id="`
+3. In the log results, click the **TraceID** derived field link
+4. Verify it navigates to the full trace in Tempo
+
+### Expected results
+
+| Check                          | Expected                                 |
+| ------------------------------ | ---------------------------------------- |
+| `trace_id=` in debug.log       | Present in log lines within active spans |
+| `span_id=` in debug.log        | Present alongside trace_id               |
+| Logs without active span       | No trace_id/span_id fields               |
+| trace_id in Tempo              | Matches a valid trace                    |
+| Loki log ingestion             | Logs visible via LogQL                   |
+| Tempo -> Loki "Logs for trace" | Shows correlated log lines               |
+| Loki -> Tempo TraceID link     | Navigates to correct trace               |
+
+---
+
+## Troubleshooting
+
+### No traces in Tempo
+
+1. Check otel-collector logs:
+   ```bash
+   docker compose -f docker/telemetry/docker-compose.yml logs otel-collector
+   ```
+2. Verify xrpld telemetry config has `enabled=1` and correct endpoint
+3. Check that otel-collector port 4318 is accessible:
+   ```bash
+   curl -sf http://localhost:4318 && echo "reachable"
+   ```
+4. Increase `batch_delay_ms` or decrease `batch_size` in xrpld config
+
+### Nodes not reaching "proposing" state
+
+1. Check that all peer ports (51235-51240) are not in use:
+   ```bash
+   for p in 51235 51236 51237 51238 51239 51240; do
+       ss -tlnp | grep ":$p " && echo "port $p in use"
+   done
+   ```
+2. Verify `[ips_fixed]` lists all 6 peer ports
+3. Verify `validators.txt` has all 6 public keys
+4. Check node debug logs: `tail -50 /tmp/xrpld-integration/node1/debug.log`
+5. Ensure `[peer_private]` is set to `1` (prevents reaching out to public network)
+
+### Transaction not processing
+
+1. Verify genesis account exists:
+   ```bash
+   curl -s http://localhost:5005 \
+       -d '{"method":"account_info","params":[{"account":"rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"}]}' |
+       jq .result.account_data.Balance
+   ```
+2. Check submit response for error codes
+3. In standalone mode, remember to call `ledger_accept` after submitting
+
+### No trace_id in log output (Phase 8)
+
+1. Verify xrpld was built with `telemetry=ON` (`-Dtelemetry=ON` in CMake)
+2. Verify `enabled=1` in the `[telemetry]` config section
+3. Log lines only contain trace context when emitted inside an active span.
+   Background logs (startup, periodic tasks outside spans) will not have
+   `trace_id`/`span_id`.
+4. Ensure the trace category is enabled (e.g., `trace_rpc=1` for RPC logs)
+
+### No logs in Loki (Phase 8)
+
+1. Verify the log file mount in docker-compose.yml:
+   ```yaml
+   volumes:
+     - ${XRPLD_LOG_DIR:-./data/logs}:/var/log/xrpld:ro
+   ```
+   The mount source defaults to the repo-relative `docker/telemetry/data/logs`
+   (where the telemetry configs write). Override `XRPLD_LOG_DIR` to tail logs
+   from another root.
+2. Check OTel Collector logs for filelog receiver errors:
+   ```bash
+   docker compose -f docker/telemetry/docker-compose.yml logs otel-collector | grep -i "filelog\|loki\|error"
+   ```
+3. Verify Loki is running:
+   ```bash
+   curl -s http://localhost:3100/ready
+   ```
+4. Verify the filelog receiver glob pattern matches your log files:
+   The default pattern is `/var/log/xrpld/*/debug.log`
+
+### Grafana trace-log links not working (Phase 8)
+
+1. Verify `tracesToLogs` is configured in the Tempo datasource provisioning
+   (`docker/telemetry/grafana/provisioning/datasources/tempo.yaml`)
+2. Verify `derivedFields` is configured in the Loki datasource provisioning
+   (`docker/telemetry/grafana/provisioning/datasources/loki.yaml`)
+3. Restart Grafana after changing provisioning files:
+   ```bash
+   docker compose -f docker/telemetry/docker-compose.yml restart grafana
+   ```
+
+### Spanmetrics not appearing in Prometheus
+
+1. Verify otel-collector config has `spanmetrics` connector
+2. Check that the metrics pipeline is configured:
+   ```yaml
+   service:
+     pipelines:
+       metrics:
+         receivers: [spanmetrics]
+         exporters: [prometheus]
+   ```
+3. Verify Prometheus can reach collector:
+   ```bash
+   curl -s http://localhost:9090/api/v1/targets | jq '.data.activeTargets'
+   ```

docker/telemetry/docker-compose.grafanacloud.yaml

@@ -0,0 +1,28 @@
+# Compose override — enable Grafana Cloud export at runtime.
+#
+# Layer this on top of the base stack to swap the collector onto the
+# Grafana Cloud dual-export config and inject the OTLP credentials. The
+# base docker-compose.yml is unchanged, so plain `up` stays local-only.
+#
+# Usage:
+#   1. Put your three Grafana Cloud values in docker/telemetry/.env.grafanacloud
+#      (template: .env.grafanacloud.example). That file is gitignored.
+#   2. Bring the stack up with BOTH compose files:
+#        docker compose -f docker/telemetry/docker-compose.yml \
+#                       -f docker/telemetry/docker-compose.grafanacloud.yaml up -d
+#   3. To go back to local-only, bring the stack up with just the base file.
+#
+# Log mount: inherited from the base compose (./data/logs -> /var/log/xrpld).
+# Both nodes' xrpld cfgs write to docker/telemetry/data/logs/<net>/debug.log,
+# so no per-box remap is needed here.
+
+services:
+  otel-collector:
+    volumes:
+      # Mount the Grafana Cloud collector config over the default path
+      # (replaces local-only config for this run; file on disk untouched).
+      - ./otel-collector-config.grafanacloud.yaml:/etc/otel-collector-config.yaml:ro
+    # Secrets from the env file, injected as container env and
+    # resolved by the collector via ${env:...}.
+    env_file:
+      - .env.grafanacloud

docker/telemetry/docker-compose.workload.yaml

@@ -0,0 +1,103 @@
+# Docker Compose workload harness for Phase 10 telemetry validation.
+#
+# Runs a 5-node validator cluster with full OTel telemetry stack:
+#   - 5 rippled validator nodes (consensus network)
+#   - OTel Collector (traces + native OTLP metrics)
+#   - Tempo (trace backend + search API)
+#   - Prometheus (metrics)
+#   - Loki (log aggregation for log-trace correlation)
+#   - Grafana (dashboards + trace/log exploration)
+#
+# Usage:
+#   # Start the harness (requires pre-built xrpld image or mount binary):
+#   docker compose -f docker/telemetry/docker-compose.workload.yaml up -d
+#
+#   # Or use the orchestrator:
+#   docker/telemetry/workload/run-full-validation.sh
+#
+# Prerequisites:
+#   - xrpld binary built with -DXRPL_ENABLE_TELEMETRY=ON
+#   - Validator keys generated via generate-validator-keys.sh
+#   - Node configs generated by run-full-validation.sh
+#
+# Note: No Docker healthchecks are defined here. The orchestrator script
+# (run-full-validation.sh) polls each service endpoint directly from the
+# host, which avoids issues with missing curl/wget in container images.
+
+services:
+  # ---------------------------------------------------------------------------
+  # Telemetry Backend Stack
+  # ---------------------------------------------------------------------------
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    ports:
+      - "4317:4317" # OTLP gRPC
+      - "4318:4318" # OTLP HTTP (traces + beast::insight metrics)
+      - "8889:8889" # Prometheus metrics endpoint
+      - "13133:13133" # Health check
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
+      # Mount the validation workdir so the filelog receiver can tail node
+      # logs. run-full-validation.sh sets XRPLD_LOG_DIR to its workdir; the
+      # default matches that workdir so a bare `docker compose up` also works.
+      - ${XRPLD_LOG_DIR:-/tmp/xrpld-validation}:/var/log/xrpld:ro
+    depends_on:
+      - tempo
+    networks:
+      - workload-net
+
+  tempo:
+    image: grafana/tempo:2.7.2
+    command: ["-config.file=/etc/tempo.yaml"]
+    ports:
+      - "3200:3200" # Tempo HTTP API
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml:ro
+      - tempo-data:/var/tempo
+    networks:
+      - workload-net
+
+  prometheus:
+    image: prom/prometheus:latest
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+    depends_on:
+      - otel-collector
+    networks:
+      - workload-net
+
+  loki:
+    image: grafana/loki:3.4.2
+    ports:
+      - "3100:3100" # Loki HTTP API
+    command: ["-config.file=/etc/loki/local-config.yaml"]
+    networks:
+      - workload-net
+
+  grafana:
+    image: grafana/grafana:latest
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    depends_on:
+      - tempo
+      - prometheus
+      - loki
+    networks:
+      - workload-net
+
+volumes:
+  tempo-data:
+
+networks:
+  workload-net:
+    driver: bridge

docker/telemetry/docker-compose.yml

@@ -2,12 +2,15 @@
 #
 # Provides services for local development:
 #   - otel-collector: receives OTLP traces from xrpld, batches and
-#     forwards them to Tempo. Listens on ports 4317 (gRPC)
-#     and 4318 (HTTP).
+#     forwards them to Tempo. Also tails xrpld log files
+#     via filelog receiver and exports to Loki. Listens on ports
+#     4317 (gRPC) and 4318 (HTTP).
 #   - tempo: Grafana Tempo tracing backend, queryable via Grafana Explore
 #     on port 3000. Recommended for production (S3/GCS storage, TraceQL).
-#   - grafana: dashboards on port 3000, pre-configured with Tempo
-#     datasource.
+#   - loki: Grafana Loki log aggregation backend for centralized log
+#     ingestion and log-trace correlation (Phase 8).
+#   - grafana: dashboards on port 3000, pre-configured with Tempo,
+#     Prometheus, and Loki datasources.
 #
 # Usage:
 #   docker compose -f docker/telemetry/docker-compose.yml up -d
@@ -24,14 +27,25 @@ services:
     image: otel/opentelemetry-collector-contrib:0.121.0
     command: ["--config=/etc/otel-collector-config.yaml"]
     ports:
-      - "4317:4317" # OTLP gRPC receiver
-      - "4318:4318" # OTLP HTTP receiver (xrpld sends traces here)
-      - "13133:13133" # Health check endpoint
+      - "4317:4317" # OTLP gRPC
+      - "4318:4318" # OTLP HTTP (traces + native OTel metrics)
+      - "8889:8889" # Prometheus metrics (spanmetrics + OTLP)
+      # StatsD UDP port removed — beast::insight now uses native OTLP.
+      # Uncomment if using server=statsd fallback:
+      # - "8125:8125/udp"
     volumes:
       # Mount collector pipeline config (receivers → processors → exporters)
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
+      # Mount the xrpld log root for the filelog receiver. The telemetry
+      # configs write to docker/telemetry/data/logs/<network>/debug.log, so
+      # the default source is the repo-relative ./data/logs — user-owned and
+      # needing no root, so `docker compose up` works with no setup. Override
+      # XRPLD_LOG_DIR to point at another root (e.g. the integration test sets
+      # it to its own workdir). Mounted read-only so the collector only tails.
+      - ${XRPLD_LOG_DIR:-./data/logs}:/var/log/xrpld:ro
     depends_on:
       - tempo
+      - loki
     networks:
       - xrpld-telemetry
 
@@ -50,28 +64,87 @@ services:
     networks:
       - xrpld-telemetry
 
+  # Grafana Loki for centralized log ingestion and log-trace
+  # correlation. Loki 3.x supports native OTLP ingestion, so the OTel
+  # Collector exports via otlphttp to Loki's /otlp endpoint.
+  # Query logs via Grafana Explore -> Loki at http://localhost:3000.
+  loki:
+    image: grafana/loki:3.4.2
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+    volumes:
+      - loki-data:/loki
+    networks:
+      - xrpld-telemetry
+
+  prometheus:
+    # Pinned to the v2.53 LTS line for reproducible, config-stable runs.
+    image: prom/prometheus:v2.53.3
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    depends_on:
+      - otel-collector
+    networks:
+      - xrpld-telemetry
+
   # Grafana: visualization UI with Tempo pre-configured as a datasource.
   # Anonymous admin access enabled for local development convenience.
   grafana:
     image: grafana/grafana:11.5.2
+    # Alerting secrets/addresses (Slack webhook, alert email) come from this
+    # gitignored file; Grafana expands the ${VARS} referenced in the alerting
+    # provisioning YAML. Absent file = unset vars = receivers have no live
+    # destination, which is fine for a local stack. See .env.alerting.example.
+    env_file:
+      - path: .env.alerting
+        required: false
     environment:
       - GF_AUTH_ANONYMOUS_ENABLED=true # No login required for local dev
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin # Full access without auth
+      # Remote image rendering: point Grafana at the renderer container.
+      # These belong on the grafana service (the server delegates renders);
+      # the callback URL is how the renderer fetches the panel from grafana.
+      - GF_RENDERING_SERVER_URL=http://renderer:8081/render
+      - GF_RENDERING_CALLBACK_URL=http://grafana:3000/
+    # SMTP for the critical-tier email receiver and the Slack webhook / email
+    # address for the contact points all come from the env_file above, which
+    # injects them straight into the container environment for Grafana to
+    # expand. They are deliberately NOT duplicated here: a compose-level
+    # `environment:` entry is interpolated from the host shell and would
+    # override (and blank out) the env_file values.
     ports:
       - "3000:3000" # Grafana web UI
     volumes:
       # Auto-provision Tempo datasource and search filters on startup
       - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
     depends_on:
       - tempo
+      - prometheus
+      - loki
+      - renderer
     networks:
       - xrpld-telemetry
 
+  # Grafana image renderer: a sidecar that renders panels/dashboards to PNG
+  # for image export and alerting. Grafana calls it at http://renderer:8081.
+  renderer:
+    image: grafana/grafana-image-renderer:latest
+    ports:
+      - "8081:8081" # Renderer HTTP endpoint (called by grafana)
+    networks:
+      - xrpld-telemetry
 # Named volume for Tempo trace storage (WAL and compacted blocks).
 # Data persists across container restarts. Remove with:
 #   docker compose -f docker/telemetry/docker-compose.yml down -v
 volumes:
   tempo-data:
+  prometheus-data:
+  loki-data:
 
 # Isolated bridge network so services communicate by container name
 # (e.g., the collector reaches Tempo at http://tempo:4317).

docker/telemetry/grafana/dashboards/fee-market.json

@@ -0,0 +1,589 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "What this shows: Transaction queue depth and capacity, fee-escalation levels, and load-factor breakdown that together set the minimum fee to enter the open ledger.\nUse it to: Understand why the minimum transaction fee is rising and how full the open ledger and queue are.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Transaction Queue Depth",
+      "description": "###### What this is:\n*Transactions currently waiting in the transaction queue versus the queue's maximum capacity.*\n\n###### How it's computed:\n*Instantaneous gauge readings of current queue count and configured max size.*\n\n###### Reading it:\n*Queue depth well below capacity is normal; depth approaching capacity means the node is saturating.*\n\n###### Healthy range:\n*Depth near 0 in quiet periods; workload-dependent under load.*\n\n###### Watch for:\n*Depth pinned at capacity for sustained periods, which signals demand exceeding throughput or a fee-spam burst.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerTxqGauge`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"txq_count\"}",
+          "legendFormat": "Queue Depth [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"txq_max_size\"}",
+          "legendFormat": "Max Capacity [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Transactions",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Transactions Per Ledger",
+      "description": "###### What this is:\n*Transactions already placed in the current open ledger versus the expected per-ledger target.*\n\n###### How it's computed:\n*Instantaneous gauge readings of in-ledger count and the target count that governs fee escalation.*\n\n###### Reading it:\n*Staying at or below the expected target is normal; exceeding it triggers open-ledger fee escalation.*\n\n###### Healthy range:\n*At or under the expected per-ledger target.*\n\n###### Watch for:\n*In-ledger count persistently above target, indicating sustained congestion pushing fees up.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerTxqGauge`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"txq_in_ledger\"}",
+          "legendFormat": "In Ledger [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"txq_per_ledger\"}",
+          "legendFormat": "Expected Per Ledger [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Transactions",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Fee Escalation Levels",
+      "description": "###### What this is:\n*The fee levels that govern queue admission: reference (baseline), minimum processing, median, and open-ledger levels.*\n\n###### How it's computed:\n*Instantaneous gauge readings of each fee level, shown on a log scale.*\n\n###### Reading it:\n*Open-ledger level near the reference level means cheap entry; a large gap above reference means escalation is active.*\n\n###### Healthy range:\n*Open-ledger level at or near reference during normal traffic.*\n\n###### Watch for:\n*Open-ledger level spiking far above reference, the hallmark of congestion or a fee-bidding war.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerTxqGauge`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"txq_reference_fee_level\"}",
+          "legendFormat": "Reference Fee Level [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"txq_min_processing_fee_level\"}",
+          "legendFormat": "Min Processing Fee Level [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"txq_med_fee_level\"}",
+          "legendFormat": "Median Fee Level [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "txq_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"txq_open_ledger_fee_level\"}",
+          "legendFormat": "Open Ledger Fee Level [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Fee Level",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 5,
+            "scaleDistribution": {
+              "type": "log",
+              "log": 2
+            }
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Load Factor Breakdown",
+      "description": "###### What this is:\n*The combined load factor and its server, fee-escalation, and fee-queue contributors as unitless fee multipliers (1.0 = no load).*\n\n###### How it's computed:\n*Instantaneous gauge readings of each load-factor component.*\n\n###### Reading it:\n*Values at 1.0 mean base fees; higher values raise the fee to transact.*\n\n###### Healthy range:\n*Around 1.0 under normal conditions.*\n\n###### Watch for:\n*Combined factor climbing well above 1.0, showing the node is charging premium fees due to congestion or overload.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLoadFactorGauge`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"load_factor\"}",
+          "legendFormat": "Combined Load Factor [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"load_factor_server\"}",
+          "legendFormat": "Server [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"load_factor_fee_escalation\"}",
+          "legendFormat": "Fee Escalation [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"load_factor_fee_queue\"}",
+          "legendFormat": "Fee Queue [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Multiplier",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 5
+          },
+          "color": {
+            "mode": "palette-classic"
+          },
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 2
+              },
+              {
+                "color": "red",
+                "value": 10
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Load Factor Components",
+      "description": "###### What this is:\n*The individual load-factor inputs, local server load, network load, and cluster load, as unitless multipliers.*\n\n###### How it's computed:\n*Instantaneous gauge readings of each component.*\n\n###### Reading it:\n*All at 1.0 means no load pressure from any source; a raised component identifies where load originates.*\n\n###### Healthy range:\n*Around 1.0 for each component.*\n\n###### Watch for:\n*A single component rising sharply, which pinpoints whether the pressure is local, network-wide, or cluster-driven.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLoadFactorGauge`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"load_factor_local\"}",
+          "legendFormat": "Local [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"load_factor_net\"}",
+          "legendFormat": "Network [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "load_factor_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", metric=\"load_factor_cluster\"}",
+          "legendFormat": "Cluster [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Multiplier",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 5
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Load Factor Attribution (Stacked Components)",
+      "description": "###### What this is:\n*The stacked contribution of each load-factor component (fee escalation, queue, local, net, cluster) to the effective transaction cost.*\n\n###### How it's computed:\n*Instantaneous gauge readings of each component stacked to show relative share.*\n\n###### Reading it:\n*The tallest band identifies which component is driving the fee, detail the aggregate load factor hides.*\n\n###### Healthy range:\n*Thin bands near 1.0 total under normal traffic.*\n\n###### Watch for:\n*One band dominating during a fee spike, revealing the specific driver of elevated cost.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLoadFactorGauge`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "load_factor_metrics{metric=\"load_factor_fee_escalation\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Fee Escalation [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "load_factor_metrics{metric=\"load_factor_fee_queue\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Fee Queue [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "load_factor_metrics{metric=\"load_factor_local\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Local [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "load_factor_metrics{metric=\"load_factor_net\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Net [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "load_factor_metrics{metric=\"load_factor_cluster\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Cluster [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Load Factor Multiplier",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3,
+            "stacking": {
+              "mode": "normal",
+              "group": "A"
+            },
+            "fillOpacity": 30
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Queue Abandonment Rate (Expired)",
+      "description": "###### What this is:\n*Transactions dropped from the queue because their last-ledger deadline passed before they could be included.*\n\n###### How it's computed:\n*Per-second rate of the cumulative expired-transaction counter over a 5-minute window.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means submitters under-bid the escalating fee and their transactions timed out.*\n\n###### Healthy range:\n*Near 0 expirations per second.*\n\n###### Watch for:\n*Sustained expiry rate, a demand-frustration signal often coinciding with fee spikes or spam that crowds out honest traffic.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementTxqExpired (caller TxQ.cpp)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (service_instance_id) (rate(txq_expired_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m]))",
+          "legendFormat": "Expired / Sec [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Expired / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Queue Admission Rejections (Dropped)",
+      "description": "###### What this is:\n*Transactions refused entry to the queue, broken down by reason such as queue_full.*\n\n###### How it's computed:\n*Per-second rate of the cumulative dropped-transaction counter over a 5-minute window, split by reason.*\n\n###### Reading it:\n*Near zero is healthy; queue_full rejections mean the queue is at capacity and applying backpressure.*\n\n###### Healthy range:\n*Near 0 rejections per second.*\n\n###### Watch for:\n*A burst of queue_full drops, distinct from expiry, indicating the node is being flooded faster than it can drain.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementTxqDropped (caller TxQ.cpp)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (reason, service_instance_id) (rate(txq_dropped_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m]))",
+          "legendFormat": "{{reason}} [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Dropped / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["transactions", "fees"],
+  "templating": {
+    "list": [
+      {
+        "name": "DS_PROMETHEUS",
+        "type": "datasource",
+        "label": "Prometheus",
+        "query": "prometheus",
+        "regex": "",
+        "current": {},
+        "hide": 0,
+        "refresh": 1,
+        "includeAll": false,
+        "multi": false,
+        "options": []
+      },
+      {
+        "name": "service_name",
+        "label": "Service Name",
+        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
+        "type": "query",
+        "query": "label_values(service_name)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "xrpld",
+          "value": "xrpld"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "deployment_environment",
+        "label": "Environment",
+        "description": "Filter by deployment tier (local/ci/test/prod)",
+        "type": "query",
+        "query": "label_values(deployment_environment)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "xrpl_network_type",
+        "label": "Network",
+        "description": "Filter by XRPL network (mainnet/testnet/devnet)",
+        "type": "query",
+        "query": "label_values(xrpl_network_type)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(service_instance_id)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Fee Market & TxQ",
+  "uid": "fee-market",
+  "version": 1,
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/job-queue.json

@@ -0,0 +1,598 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "What this shows: Per-job-type throughput, queue wait times, and execution times for the node's internal worker job queue.\nUse it to: Find job types that are backing up or running slowly and causing internal processing delays.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Current Job Latency (p99 Gauge)",
+      "description": "###### What this is:\n*At-a-glance p99 of how long jobs wait in the queue and how long they run once started.*\n\n###### How it's computed:\n*99th percentile derived from the job wait-time and run-time histograms over the last 5 minutes.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Wait and exec p99 under 100ms.*\n\n###### Watch for:\n*p99 wait climbing into the red, meaning worker threads are saturated and jobs are backing up.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted / recordJobFinished`",
+      "type": "gauge",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "orientation": "auto",
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", job_type=~\"$job_type\"}[5m])))",
+          "legendFormat": "p99 Wait"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", job_type=~\"$job_type\"}[5m])))",
+          "legendFormat": "p99 Exec"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "µs",
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 100000
+              },
+              {
+                "color": "red",
+                "value": 1000000
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "id": 1
+    },
+    {
+      "title": "Job Throughput Rate (Per Second)",
+      "description": "###### What this is:\n*Rate of jobs queued, started, and finished across all job types.*\n\n###### How it's computed:\n*Per-second rate of each cumulative job counter over a 5-minute window.*\n\n###### Reading it:\n*Queued, started, and finished tracking together means the queue keeps up.*\n\n###### Healthy range:\n*Workload-dependent; the three rates should stay roughly equal.*\n\n###### Watch for:\n*Queued rate persistently above finished rate, which indicates a growing backlog.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobQueued / recordJobStarted / recordJobFinished`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (service_instance_id) (rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", job_type=~\"$job_type\"}[5m]))",
+          "legendFormat": "Queued/s [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (service_instance_id) (rate(job_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", job_type=~\"$job_type\"}[5m]))",
+          "legendFormat": "Started/s [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (service_instance_id) (rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", job_type=~\"$job_type\"}[5m]))",
+          "legendFormat": "Finished/s [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10,
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 2
+    },
+    {
+      "title": "Per-Job-Type Queued Rate",
+      "description": "###### What this is:\n*Rate of jobs entering the queue, broken down by job type (top 10).*\n\n###### How it's computed:\n*Per-second rate of the queued-job counter per job_type over a 5-minute window.*\n\n###### Reading it:\n*Identifies which job types generate the most queue activity.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A single job type surging unexpectedly, which can point to a flood of a particular request or peer message.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobQueued`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "topk(10, rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", job_type=~\"$job_type\"}[5m]))",
+          "legendFormat": "{{job_type}} [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 5,
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 3
+    },
+    {
+      "title": "Per-Job-Type Finish Rate",
+      "description": "###### What this is:\n*Rate of jobs completing, broken down by job type (top 10).*\n\n###### How it's computed:\n*Per-second rate of the finished-job counter per job_type over a 5-minute window.*\n\n###### Reading it:\n*Compare against the queued rate per type to spot which types are falling behind.*\n\n###### Healthy range:\n*Workload-dependent; should match the per-type queued rate.*\n\n###### Watch for:\n*A type whose finish rate lags its queued rate, revealing where the backlog concentrates.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "topk(10, rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", job_type=~\"$job_type\"}[5m]))",
+          "legendFormat": "{{job_type}} [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 5,
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 4
+    },
+    {
+      "title": "Job Queue Wait Time",
+      "description": "###### What this is:\n*Distribution of how long jobs sit in the queue before a worker picks them up (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job wait-time histogram over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; a widening gap between p75 and p99 signals occasional stalls.*\n\n###### Healthy range:\n*Sub-millisecond to low-millisecond waits on an unloaded node.*\n\n###### Watch for:\n*Rising p99 wait, an early sign of worker-thread saturation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.75, sum by (le, service_instance_id) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", job_type=~\"$job_type\"}[5m])))",
+          "legendFormat": "p75 Wait [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le, service_instance_id) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", job_type=~\"$job_type\"}[5m])))",
+          "legendFormat": "p99 Wait [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "µs",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 5,
+            "axisLabel": "Duration (μs)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 5
+    },
+    {
+      "title": "Job Execution Time",
+      "description": "###### What this is:\n*Distribution of how long jobs run once started (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job run-time histogram over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; stable p75 with a controlled p99 is healthy.*\n\n###### Healthy range:\n*Workload-dependent, but stable over time.*\n\n###### Watch for:\n*Growing execution times, which point to CPU pressure or expensive individual jobs.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.75, sum by (le, service_instance_id) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", job_type=~\"$job_type\"}[5m])))",
+          "legendFormat": "p75 Exec [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le, service_instance_id) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", job_type=~\"$job_type\"}[5m])))",
+          "legendFormat": "p99 Exec [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "µs",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 5,
+            "axisLabel": "Duration (μs)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 6
+    },
+    {
+      "title": "Per-Job-Type Execution Time (p99)",
+      "description": "###### What this is:\n*The 10 slowest job types ranked by p99 execution time.*\n\n###### How it's computed:\n*p99 derived from the run-time histogram per job_type over a 5-minute window, top 10 selected.*\n\n###### Reading it:\n*Highlights which job types cost the most CPU time at the tail.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A job type whose p99 grows over time, indicating a slow or degrading operation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max"]
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", job_type=~\"$job_type\"}[5m]))))",
+          "legendFormat": "{{job_type}} [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "µs",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 5,
+            "axisLabel": "Duration (μs)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 7
+    },
+    {
+      "title": "Transaction Overflow Rate",
+      "description": "###### What this is:\n*Rate at which transaction jobs would be shed when the queue's transaction limit is exceeded.*\n\n###### How it's computed:\n*Per-second rate of the overflow counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*This counter is not currently wired up in the code, so the panel reads empty or zero regardless of load.*\n\n###### Healthy range:\n*Not applicable until wired; currently always empty or zero.*\n\n###### Watch for:\n*n/a until wired — this counter does not rise under load today.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`(job-queue transaction overflow counter)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 40
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "rate(jq_trans_overflow_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m]) * 60",
+          "legendFormat": "Overflows/min [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 10
+              }
+            ]
+          },
+          "custom": {
+            "axisLabel": "Overflows / Min",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10,
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "id": 8
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["node", "jobqueue"],
+  "templating": {
+    "list": [
+      {
+        "name": "DS_PROMETHEUS",
+        "type": "datasource",
+        "label": "Prometheus",
+        "query": "prometheus",
+        "regex": "",
+        "current": {},
+        "hide": 0,
+        "refresh": 1,
+        "includeAll": false,
+        "multi": false,
+        "options": []
+      },
+      {
+        "name": "service_name",
+        "label": "Service Name",
+        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
+        "type": "query",
+        "query": "label_values(service_name)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "xrpld",
+          "value": "xrpld"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "deployment_environment",
+        "label": "Environment",
+        "description": "Filter by deployment tier (local/ci/test/prod)",
+        "type": "query",
+        "query": "label_values(deployment_environment)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "xrpl_network_type",
+        "label": "Network",
+        "description": "Filter by XRPL network (mainnet/testnet/devnet)",
+        "type": "query",
+        "query": "label_values(xrpl_network_type)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(service_instance_id)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "job_type",
+        "label": "Job Type",
+        "description": "Filter by job type",
+        "type": "query",
+        "query": "label_values(job_queued_total, job_type)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Job Queue Analysis",
+  "uid": "job-queue",
+  "version": 1,
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/ledger-data-sync.json

@@ -0,0 +1,639 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "What this shows: Ledger data exchange and object-fetch traffic between this node and its peers: ledger sync, tree-node retrieval, and transaction-set exchange.\nUse it to: See how much ledger data the node is pulling or serving and spot catch-up activity.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Ledger Data Exchange (Bytes In)",
+      "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte counters, shown at their latest cumulative value per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_data_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Ledger Data Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_data_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Ledger Data Share [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_data_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Set Candidate Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_data_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Set Candidate Share [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_data_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Node Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_data_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Node Share [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_data_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Account State Node Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_data_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Account State Node Share [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes In",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Ledger Share/Get Traffic (Bytes)",
+      "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte counters at their latest value per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Ledger Share In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Ledger Get In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Set Candidate Share [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Set Candidate Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Node Share [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Node Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Account State Share [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "ledger_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Account State Get [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes In",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "GetObject Traffic by Type (Bytes In)",
+      "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte counters at their latest value per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Ledger Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Ledger Share [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_transaction_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Transaction Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_transaction_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Transaction Share [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Node Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Node Share [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Account State Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Account State Share [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes In",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "GetObject Aggregate & Special Types (Bytes In)",
+      "description": "###### What this is:\n*Aggregate object-fetch inbound bytes plus special buckets: content-addressed storage fetches, bulk fetch-pack downloads used during catch-up, and bulk transaction fetches.*\n\n###### How it's computed:\n*Per-category inbound byte counters at their latest value per node.*\n\n###### Reading it:\n*Fetch-pack rises sharply while catching up a range of ledgers; near zero when fully synced.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*Continuous fetch-pack traffic (node never fully catches up) or unexpectedly high content-store volume.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_cas_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "CAS Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_cas_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "CAS Share [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_fetch_pack_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Fetch Pack Share [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_fetch_pack_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Fetch Pack Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_transactions_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Transactions Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Aggregate Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Aggregate Share [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes In",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "GetObject Messages by Type",
+      "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message counters at their latest value per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_ledger_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Ledger Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_transaction_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Transaction Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_transaction_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Node Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_account_state_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Account State Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_cas_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "CAS Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_fetch_pack_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Fetch Pack Get [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "getobject_transactions_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Transactions Get [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages In",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Overlay Traffic Heatmap (All Categories, Bytes In)",
+      "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "bargauge",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"})",
+          "legendFormat": "{{__name__}} [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1048576
+              },
+              {
+                "color": "red",
+                "value": 104857600
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["ledger", "sync"],
+  "templating": {
+    "list": [
+      {
+        "name": "DS_PROMETHEUS",
+        "type": "datasource",
+        "label": "Prometheus",
+        "query": "prometheus",
+        "regex": "",
+        "current": {},
+        "hide": 0,
+        "refresh": 1,
+        "includeAll": false,
+        "multi": false,
+        "options": []
+      },
+      {
+        "name": "service_name",
+        "label": "Service Name",
+        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
+        "type": "query",
+        "query": "label_values(service_name)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "xrpld",
+          "value": "xrpld"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "deployment_environment",
+        "label": "Environment",
+        "description": "Filter by deployment tier (local/ci/test/prod)",
+        "type": "query",
+        "query": "label_values(deployment_environment)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "xrpl_network_type",
+        "label": "Network",
+        "description": "Filter by XRPL network (mainnet/testnet/devnet)",
+        "type": "query",
+        "query": "label_values(xrpl_network_type)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(ledger_data_get_bytes_in, service_instance_id)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Ledger Data & Sync",
+  "uid": "ledger-data-sync"
+}

docker/telemetry/grafana/dashboards/ledger-operations.json

@@ -0,0 +1,423 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Ledger Build Rate",
+      "description": "###### What this is:\n*How many new ledgers this node finishes building per second.*\n\n###### How it's computed:\n*Per-second rate of completed ledger-build operations, averaged over 5 minutes and split by node.*\n\n###### Reading it:\n*A steady flat line; the value should track the network close cadence.*\n\n###### Healthy range:\n*About 0.2-0.3 ledgers/sec on mainnet (roughly one every 3-5s); workload-dependent on test networks.*\n\n###### Watch for:\n*A drop toward zero (node fell out of sync or stalled) or a value well above the network rate (rebuilding history).*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"ledger.build\"}[5m]))",
+          "legendFormat": "Builds / Sec [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Ledger Build Duration",
+      "description": "###### What this is:\n*The time taken to build a single ledger, at the 95th percentile.*\n\n###### How it's computed:\n*95th-percentile of ledger-build durations over a 5-minute window, per node.*\n\n###### Reading it:\n*Lower is better; the line should stay well under the ledger interval.*\n\n###### Healthy range:\n*Typically tens to low hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained rises approaching the close interval, which indicate heavy transaction sets or disk/I/O pressure.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"ledger.build\"}[5m])))",
+          "legendFormat": "P95 Build Duration [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Ledger Validation Rate",
+      "description": "###### What this is:\n*How often ledgers reach full validation (accepted by the trusted validator quorum) per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-validation events over 5 minutes, per node.*\n\n###### Reading it:\n*Should closely match the build rate under normal, in-sync operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; workload-dependent elsewhere.*\n\n###### Watch for:\n*A validation rate that lags the build rate, signalling the node is building ahead of the network consensus it trusts.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"ledger.validate\"}[5m]))",
+          "legendFormat": "Validations / Sec [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Ledger Build Duration Heatmap",
+      "description": "###### What this is:\n*The full distribution of ledger-build times over the window, not just a single percentile.*\n\n###### How it's computed:\n*Counts of ledger builds falling in each duration band per 5-minute window, shown as color density.*\n\n###### Reading it:\n*A tight low band is healthy; scattered high cells mean occasional slow builds.*\n\n###### Healthy range:\n*Most mass concentrated in the low-millisecond bands; workload-dependent.*\n\n###### Watch for:\n*A second cluster of hot cells at high durations (bimodal build times) hidden by percentile charts.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`",
+      "type": "heatmap",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        },
+        "yAxis": {
+          "axisLabel": "Duration (ms)"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum(increase(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"ledger.build\"}[5m])) by (le)",
+          "legendFormat": "{{le}}",
+          "format": "heatmap"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Transaction Apply Duration",
+      "description": "###### What this is:\n*The time spent applying the agreed transaction set into the new ledger, at the 95th percentile.*\n\n###### How it's computed:\n*95th-percentile of transaction-apply durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; this is a large share of total build time.*\n\n###### Healthy range:\n*A few to tens of milliseconds; scales with transaction volume per ledger.*\n\n###### Watch for:\n*Spikes during large or expensive transaction sets, which push out overall ledger build time.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`applyTransactions`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"tx.apply\"}[5m])))",
+          "legendFormat": "P95 tx.apply [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Transaction Apply Rate",
+      "description": "###### What this is:\n*How often the transaction-apply phase runs per second (once per ledger build).*\n\n###### How it's computed:\n*Per-second rate of transaction-apply operations over 5 minutes, per node.*\n\n###### Reading it:\n*Should track the ledger build rate almost exactly.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; workload-dependent.*\n\n###### Watch for:\n*Divergence from the build rate, which would indicate a metric or pipeline anomaly.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`applyTransactions`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"tx.apply\"}[5m]))",
+          "legendFormat": "tx.apply / Sec [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Ledger Store Rate",
+      "description": "###### What this is:\n*How often completed ledgers are written into ledger history per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-store operations over 5 minutes, per node.*\n\n###### Reading it:\n*Should match the build rate during normal operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; can burst higher while backfilling history.*\n\n###### Watch for:\n*A store rate below the build rate (storage falling behind) or a stall at zero.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::storeLedger`",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"ledger.store\"}[5m]))",
+          "legendFormat": "Stores / Sec [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Build vs Close Duration",
+      "description": "###### What this is:\n*Ledger build time compared with the total consensus ledger-close time, both at the 95th percentile.*\n\n###### How it's computed:\n*Two 95th-percentile duration series over 5 minutes: ledger construction and the full consensus close, per node.*\n\n###### Reading it:\n*Build should sit below close; the gap is consensus overhead outside construction.*\n\n###### Healthy range:\n*Close a bit above build; both workload-dependent and under the round interval.*\n\n###### Watch for:\n*A widening gap (consensus-pipeline overhead growing) or build time approaching close time.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp) · [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`buildLedgerImpl ; RCLConsensus::Adaptor::onClose`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"ledger.build\"}[5m])))",
+          "legendFormat": "P95 ledger.build [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"consensus.ledger_close\"}[5m])))",
+          "legendFormat": "P95 consensus.ledger_close [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["ledger"],
+  "templating": {
+    "list": [
+      {
+        "name": "DS_PROMETHEUS",
+        "type": "datasource",
+        "label": "Prometheus",
+        "query": "prometheus",
+        "regex": "",
+        "current": {},
+        "hide": 0,
+        "refresh": 1,
+        "includeAll": false,
+        "multi": false,
+        "options": []
+      },
+      {
+        "name": "service_name",
+        "label": "Service Name",
+        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
+        "type": "query",
+        "query": "label_values(service_name)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "xrpld",
+          "value": "xrpld"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "deployment_environment",
+        "label": "Environment",
+        "description": "Filter by deployment tier (local/ci/test/prod)",
+        "type": "query",
+        "query": "label_values(deployment_environment)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "xrpl_network_type",
+        "label": "Network",
+        "description": "Filter by XRPL network (mainnet/testnet/devnet)",
+        "type": "query",
+        "query": "label_values(xrpl_network_type)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id — e.g. Node-1)",
+        "type": "query",
+        "query": "label_values(span_calls_total, service_instance_id)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Ledger Operations",
+  "uid": "ledger-operations",
+  "refresh": "5s",
+  "description": "What this shows: Ledger construction, validation, and storage activity and timing for this node.\nUse it to: Confirm ledgers are being built, validated, and stored on schedule and find the slow stage when they are not."
+}

docker/telemetry/grafana/dashboards/network-traffic.json

@@ -0,0 +1,904 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "What this shows: Peer connectivity and overlay bandwidth for this node: peer counts, disconnects, total bytes and messages exchanged, and the transaction, proposal, and validation traffic that flows across the peer-to-peer network.\nUse it to: Gauge peer health and overall overlay bandwidth and see which message types dominate network traffic.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Active Peers",
+      "description": "###### What this is:\n*Number of active inbound and outbound peer connections the node currently holds.*\n\n###### How it's computed:\n*Current value of the inbound and outbound active-peer counts per node.*\n\n###### Reading it:\n*Outbound is what the node dials out; inbound is what others open to it. Both should be stable.*\n\n###### Healthy range:\n*roughly 10-21 outbound and 0-85 inbound on mainnet, depending on config.*\n\n###### Watch for:\n*Outbound dropping toward zero (isolation) or inbound pinned at the limit with churn (connection pressure).*\n\n###### Source:\n[PeerfinderManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/PeerfinderManager.cpp)\n\n###### Function:\n`Logic Stats ctor`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "peer_finder_active_inbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Inbound Peers [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "peer_finder_active_outbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Outbound Peers [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Peers",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Peer Disconnects",
+      "description": "###### What this is:\n*Cumulative count of peer connections that have dropped.*\n\n###### How it's computed:\n*Running total of disconnect events per node.*\n\n###### Reading it:\n*A flat or slowly rising line is normal; the slope matters more than the absolute value.*\n\n###### Healthy range:\n*workload-dependent; slow, steady growth.*\n\n###### Watch for:\n*Sharp step-ups in the slope (network instability, resource exhaustion, or many peers dropping the node at once).*\n\n###### Source:\n[OverlayImpl.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.h)\n\n###### Function:\n`OverlayImpl::Stats ctor`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "overlay_peer_disconnects{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Disconnects [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Disconnects",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Total Network Bytes",
+      "description": "###### What this is:\n*Total bytes received and sent across all peer connections.*\n\n###### How it's computed:\n*Current cumulative in/out byte totals per node.*\n\n###### Reading it:\n*Overall bandwidth footprint; in and out usually track network activity together.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*Sudden sustained jumps not matched by ledger or transaction activity (relay storms or a noisy peer).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "total_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Bytes In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "total_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Bytes Out [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Total Network Messages",
+      "description": "###### What this is:\n*Total messages received and sent across all peer connections.*\n\n###### How it's computed:\n*Current cumulative in/out message totals per node.*\n\n###### Reading it:\n*Overall message throughput of the overlay; complements the byte totals.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*Message count climbing far faster than bytes (many tiny messages, possible flooding).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "total_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Messages In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "total_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Messages Out [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Transaction Traffic",
+      "description": "###### What this is:\n*Transaction relay messages in and out, plus duplicate transaction messages received.*\n\n###### How it's computed:\n*Current message counts for the transaction and transaction-duplicate categories.*\n\n###### Reading it:\n*In/out rise with network transaction volume; duplicates are transactions the node already had.*\n\n###### Healthy range:\n*workload-dependent; duplicates a modest fraction of inbound.*\n\n###### Watch for:\n*Duplicate inbound approaching or exceeding unique inbound (redundant relay), or a sharp spike suggesting transaction flooding.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "transactions_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Messages In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "transactions_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Messages Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "transactions_duplicate_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "TX Duplicate In [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Proposal Traffic",
+      "description": "###### What this is:\n*Consensus proposal messages in/out, plus untrusted and duplicate proposal messages received.*\n\n###### How it's computed:\n*Current message counts for the proposal, proposal-untrusted, and proposal-duplicate categories.*\n\n###### Reading it:\n*Trusted in/out track consensus rounds; untrusted come from validators not on this node's trusted list.*\n\n###### Healthy range:\n*workload-dependent; untrusted and duplicates low relative to trusted.*\n\n###### Watch for:\n*High untrusted (trusted-list misconfiguration) or high duplicates (inefficient relay or proposal spam).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "proposals_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Proposals In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "proposals_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Proposals Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "proposals_untrusted_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Untrusted In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "proposals_duplicate_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Duplicate In [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Validation Traffic",
+      "description": "###### What this is:\n*Validation messages in/out, plus untrusted and duplicate validation messages received.*\n\n###### How it's computed:\n*Current message counts for the validation, validation-untrusted, and validation-duplicate categories.*\n\n###### Reading it:\n*Trusted validations should arrive steadily each ledger; untrusted come from non-trusted validators.*\n\n###### Healthy range:\n*workload-dependent; untrusted and duplicates low relative to trusted.*\n\n###### Watch for:\n*Rising untrusted or duplicate validations (trusted-list health issues or validation spam).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "validations_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Validations In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "validations_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Validations Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "validations_untrusted_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Untrusted In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "validations_duplicate_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Duplicate In [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Overlay Traffic by Category (Bytes In)",
+      "description": "###### What this is:\n*Top overlay traffic categories ranked by inbound bytes, excluding the all-traffic total.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value per node.*\n\n###### Reading it:\n*Shows which message types dominate receive bandwidth right now.*\n\n###### Healthy range:\n*workload-dependent; transactions, proposals, and validations typically lead on a synced node.*\n\n###### Watch for:\n*A fetch or ledger-data category topping the list (sync activity) or an unexpected category dominating.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "bargauge",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "topk(10, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"})",
+          "legendFormat": "{{__name__}} [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "transactions_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Transactions"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "proposals_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Proposals"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "validations_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Validations"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "overhead_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Overhead"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "overhead_overlay_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Overhead Overlay"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ping_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Ping"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "status_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Status"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "getobject_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Get Object"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "havetxset_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Have Tx Set"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ledgerdata_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Ledger Data"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ledger_share_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Ledger Share"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ledger_data_get_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Ledger Data Get"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ledger_data_share_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Ledger Data Share"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ledger_data_account_state_node_get_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Account State Node Get"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ledger_data_account_state_node_share_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Account State Node Share"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ledger_data_transaction_node_get_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Transaction Node Get"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ledger_data_transaction_node_share_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Transaction Node Share"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ledger_data_transaction_set_candidate_get_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Tx Set Candidate Get"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ledger_account_state_node_share_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Account State Node Share (Legacy)"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ledger_transaction_set_candidate_share_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Tx Set Candidate Share"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ledger_transaction_node_share_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Transaction Node Share (Legacy)"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "set_get_bytes_in"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "Set Get"
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "title": "Duplicate Traffic (Wasted Bandwidth)",
+      "description": "###### What this is:\n*Throughput of duplicate transaction, proposal, and validation traffic: messages the node had already seen and discarded.*\n\n###### How it's computed:\n*Per-second rate of the duplicate byte counters for each category, in and out.*\n\n###### Reading it:\n*Lower is better; this is bandwidth spent on redundant relays.*\n\n###### Healthy range:\n*workload-dependent; a small fraction of total traffic.*\n\n###### Watch for:\n*Duplicate rate climbing toward the same order as useful traffic (poor relay topology or redundant flooding).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "rate(transactions_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])",
+          "legendFormat": "TX Duplicate In"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "rate(transactions_duplicate_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])",
+          "legendFormat": "TX Duplicate Out"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "rate(proposals_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])",
+          "legendFormat": "Proposals Duplicate In"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "rate(proposals_duplicate_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])",
+          "legendFormat": "Proposals Duplicate Out"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "rate(validations_duplicate_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])",
+          "legendFormat": "Validations Duplicate In"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "rate(validations_duplicate_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])",
+          "legendFormat": "Validations Duplicate Out"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps",
+          "custom": {
+            "axisLabel": "Throughput",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "All Traffic Categories (Detail)",
+      "description": "###### What this is:\n*The busiest overlay categories by inbound byte rate over time, excluding the all-traffic total.*\n\n###### How it's computed:\n*Per-second inbound byte rate for the busiest categories, ranked, excluding the all-traffic total.*\n\n###### Reading it:\n*Time-series companion to the category bar view; shows how the traffic mix shifts over the window.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A category ramping up and staying high, or the mix suddenly changing (sync, spam, or a misbehaving peer).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "topk(15, rate({__name__=~\".*_bytes_in\", __name__!~\"total_.*\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m]))",
+          "legendFormat": "{{__name__}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps",
+          "custom": {
+            "axisLabel": "Throughput",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["network"],
+  "templating": {
+    "list": [
+      {
+        "name": "DS_PROMETHEUS",
+        "type": "datasource",
+        "label": "Prometheus",
+        "query": "prometheus",
+        "regex": "",
+        "current": {},
+        "hide": 0,
+        "refresh": 1,
+        "includeAll": false,
+        "multi": false,
+        "options": []
+      },
+      {
+        "name": "service_name",
+        "label": "Service Name",
+        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
+        "type": "query",
+        "query": "label_values(service_name)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "xrpld",
+          "value": "xrpld"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "deployment_environment",
+        "label": "Environment",
+        "description": "Filter by deployment tier (local/ci/test/prod)",
+        "type": "query",
+        "query": "label_values(deployment_environment)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "xrpl_network_type",
+        "label": "Network",
+        "description": "Filter by XRPL network (mainnet/testnet/devnet)",
+        "type": "query",
+        "query": "label_values(xrpl_network_type)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(peer_finder_active_inbound_peers, service_instance_id)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Network Traffic",
+  "uid": "network-traffic"
+}

docker/telemetry/grafana/dashboards/overlay-traffic-detail.json

@@ -0,0 +1,696 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "What this shows: Fine-grained breakdown of peer-to-peer overlay traffic beyond the main network view: squelch relay control, protocol overhead, validator-list distribution, transaction-set exchange, transaction availability, ledger-proof and replay traffic, and unclassified messages.\nUse it to: Drill into individual overlay message categories to diagnose relay efficiency, overhead, and catch-up traffic.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Squelch Traffic (Messages)",
+      "description": "###### What this is:\n*Squelch relay-control messages in/out, plus messages suppressed by squelch and squelch directives that were ignored. Squelch reduces redundant message forwarding between peers.*\n\n###### How it's computed:\n*Current message counts for the squelch, squelch-suppressed, and squelch-ignored categories, in and out.*\n\n###### Reading it:\n*High suppressed counts mean squelch is saving bandwidth; ignored should stay low.*\n\n###### Healthy range:\n*workload-dependent; suppressed far above ignored.*\n\n###### Watch for:\n*High ignored counts (peers not honoring squelch) or squelch traffic itself dominating.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "squelch_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Squelch In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "squelch_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Squelch Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "squelch_suppressed_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Suppressed In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "squelch_suppressed_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Suppressed Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "squelch_ignored_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Ignored In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "squelch_ignored_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Ignored Out [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Overhead Traffic Breakdown (Bytes)",
+      "description": "###### What this is:\n*Overlay protocol overhead bytes split into base overhead, intra-cluster overhead, and validator-manifest distribution overhead.*\n\n###### How it's computed:\n*Current in/out byte counts for the overhead, overhead-cluster, and overhead-manifest categories.*\n\n###### Reading it:\n*Base overhead is routine; cluster and manifest rise around cluster syncs and manifest changes.*\n\n###### Healthy range:\n*workload-dependent; low and stable.*\n\n###### Watch for:\n*Sustained high cluster or manifest overhead (frequent cluster state churn or manifest reissue).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "overhead_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Base Overhead In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "overhead_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Base Overhead Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "overhead_cluster_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Cluster In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "overhead_cluster_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Cluster Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "overhead_manifest_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Manifest In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "overhead_manifest_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Manifest Out [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Validator List Traffic",
+      "description": "###### What this is:\n*Bytes and messages exchanged distributing validator lists (trusted-list configuration) between peers.*\n\n###### How it's computed:\n*Current in/out byte and message counts for the validator-lists category.*\n\n###### Reading it:\n*Bursts when lists update or new peers connect; quiet otherwise.*\n\n###### Healthy range:\n*workload-dependent; occasional bursts.*\n\n###### Watch for:\n*Continuous high volume (repeated list re-fetching or churn).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "validator_lists_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Bytes In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "validator_lists_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Bytes Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "validator_lists_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Messages In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "validator_lists_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Messages Out [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Count",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/Bytes/"
+            },
+            "properties": [
+              {
+                "id": "custom.axisPlacement",
+                "value": "right"
+              },
+              {
+                "id": "unit",
+                "value": "decbytes"
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "title": "Set Get/Share Traffic (Bytes)",
+      "description": "###### What this is:\n*Transaction-set fetch (get) and share bytes exchanged during ledger close.*\n\n###### How it's computed:\n*Current in/out byte counts for the set-get and set-share categories.*\n\n###### Reading it:\n*Some exchange each ledger is normal as peers reconcile transaction sets.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High set-get (peers frequently missing transaction sets: possible sync delays).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "set_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Set Get In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "set_get_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Set Get Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "set_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Set Share In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "set_share_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Set Share Out [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Have/Requested Transactions (Messages)",
+      "description": "###### What this is:\n*Transaction-availability messages: advertisements that a peer has certain transactions, and explicit requests for transaction data.*\n\n###### How it's computed:\n*Current in/out message counts for the have-transactions and requested-transactions categories.*\n\n###### Reading it:\n*Compare requested versus have to gauge how well transactions are propagating.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*Requested far exceeding have (peers behind on transaction propagation).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "have_transactions_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Have TX In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "have_transactions_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Have TX Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "requested_transactions_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Requested TX In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "requested_transactions_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Requested TX Out [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Messages",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Unknown / Unclassified Traffic",
+      "description": "###### What this is:\n*Overlay traffic that matches no known message category, in bytes and messages.*\n\n###### How it's computed:\n*Current in/out byte and message counts for the unknown category.*\n\n###### Reading it:\n*Should be at or near zero.*\n\n###### Healthy range:\n*zero.*\n\n###### Watch for:\n*Any sustained non-zero value (protocol version mismatch, corrupted messages, or an unclassified new message type).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "unknown_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Unknown Bytes In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "unknown_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Unknown Bytes Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "unknown_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Unknown Messages In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "unknown_messages_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Unknown Messages Out [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "axisLabel": "Count",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/Bytes/"
+            },
+            "properties": [
+              {
+                "id": "custom.axisPlacement",
+                "value": "right"
+              },
+              {
+                "id": "unit",
+                "value": "decbytes"
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "title": "Proof Path Traffic",
+      "description": "###### What this is:\n*Proof-path request/response bytes used to verify individual ledger entries without downloading the whole ledger.*\n\n###### How it's computed:\n*Current in/out byte counts for the proof-path request and response categories.*\n\n###### Reading it:\n*Rises when peers verify specific state, often during catch-up.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High sustained request volume (heavy state-verification load).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "proof_path_request_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Request Bytes In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "proof_path_request_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Request Bytes Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "proof_path_response_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Response Bytes In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "proof_path_response_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Response Bytes Out [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Replay Delta Traffic",
+      "description": "###### What this is:\n*Replay-delta request/response bytes used to efficiently replay ledger state changes during catch-up.*\n\n###### How it's computed:\n*Current in/out byte counts for the replay-delta request and response categories.*\n\n###### Reading it:\n*Active during catch-up and replay; quiet when synced.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*Continuous replay traffic (node repeatedly replaying rather than staying current).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "replay_delta_request_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Request Bytes In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "replay_delta_request_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Request Bytes Out [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "replay_delta_response_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Response Bytes In [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "replay_delta_response_bytes_out{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Response Bytes Out [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Bytes",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["network", "peer"],
+  "templating": {
+    "list": [
+      {
+        "name": "DS_PROMETHEUS",
+        "type": "datasource",
+        "label": "Prometheus",
+        "query": "prometheus",
+        "regex": "",
+        "current": {},
+        "hide": 0,
+        "refresh": 1,
+        "includeAll": false,
+        "multi": false,
+        "options": []
+      },
+      {
+        "name": "service_name",
+        "label": "Service Name",
+        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
+        "type": "query",
+        "query": "label_values(service_name)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "xrpld",
+          "value": "xrpld"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "deployment_environment",
+        "label": "Environment",
+        "description": "Filter by deployment tier (local/ci/test/prod)",
+        "type": "query",
+        "query": "label_values(deployment_environment)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "xrpl_network_type",
+        "label": "Network",
+        "description": "Filter by XRPL network (mainnet/testnet/devnet)",
+        "type": "query",
+        "query": "label_values(xrpl_network_type)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(squelch_messages_in, service_instance_id)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Overlay Traffic Detail",
+  "uid": "overlay-traffic-detail"
+}

docker/telemetry/grafana/dashboards/peer-network.json

@@ -0,0 +1,401 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "What this shows: Consensus proposals and validations received from peers, and transaction relay efficiency for this node.\nUse it to: Check that peers are feeding this node proposals, validations, and transactions, and gauge relay overhead.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Peer Proposal Receive Rate",
+      "description": "###### What this is:\n*How many consensus proposals this node receives from peers per second.*\n\n###### How it's computed:\n*Per-second rate of received proposals over 5 minutes, per node.*\n\n###### Reading it:\n*A steady rate roughly proportional to the number of proposing validators.*\n\n###### Healthy range:\n*Workload-dependent; scales with validator count and connectivity.*\n\n###### Watch for:\n*A drop toward zero (isolation from the network) or a sudden flood far above baseline (proposal spam).*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMProposeSet)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"peer.proposal.receive\"}[5m]))",
+          "legendFormat": "Proposals Received / Sec [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Proposals / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Peer Validation Receive Rate",
+      "description": "###### What this is:\n*How many ledger validations this node receives from peers per second.*\n\n###### How it's computed:\n*Per-second rate of received validations over 5 minutes, per node.*\n\n###### Reading it:\n*Steady and proportional to the number of validators the node hears from.*\n\n###### Healthy range:\n*Workload-dependent; roughly one burst per validator per closed ledger.*\n\n###### Watch for:\n*A fall toward zero (loss of validator connectivity) or an abnormal surge from untrusted sources.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMValidation)`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"peer.validation.receive\"}[5m]))",
+          "legendFormat": "Validations Received / Sec [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Validations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Proposals Trusted vs Untrusted",
+      "description": "###### What this is:\n*The share of received proposals that come from trusted (UNL) validators versus everyone else.*\n\n###### How it's computed:\n*Received-proposal rate split by trust status over 5 minutes, shown as proportions.*\n\n###### Reading it:\n*A healthy node with a good UNL shows a solid trusted slice.*\n\n###### Healthy range:\n*Workload-dependent; a meaningful trusted fraction is expected.*\n\n###### Watch for:\n*A collapsing trusted share or a large untrusted volume, consistent with proposal flooding from non-UNL peers.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMProposeSet)`",
+      "type": "piechart",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (proposal_trusted, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[5m]))",
+          "legendFormat": "Trusted = {{proposal_trusted}} [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Validations Trusted vs Untrusted",
+      "description": "###### What this is:\n*The share of received validations from trusted (UNL) validators versus untrusted sources.*\n\n###### How it's computed:\n*Received-validation rate split by trust status over 5 minutes, shown as proportions.*\n\n###### Reading it:\n*The trusted slice should dominate for a well-configured node.*\n\n###### Healthy range:\n*Workload-dependent; trusted validations expected to be the majority.*\n\n###### Watch for:\n*A shrinking trusted share or a spike of untrusted validations, which can indicate misconfiguration or abuse.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMValidation)`",
+      "type": "piechart",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (validation_trusted, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[5m]))",
+          "legendFormat": "Trusted = {{validation_trusted}} [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Reduce-Relay Peer Selection",
+      "description": "###### What this is:\n*How transaction relay picks peers: chosen relay sources, suppressed peers, and peers with the feature off.*\n\n###### How it's computed:\n*Current peer counts in each category (selected, suppressed, not-enabled), per node.*\n\n###### Reading it:\n*A high suppressed-to-selected ratio means relay is saving bandwidth as intended.*\n\n###### Healthy range:\n*Workload-dependent; suppressed should exceed selected in a well-connected mesh.*\n\n###### Watch for:\n*A large not-enabled count (older peers forcing full relay) or selected climbing while suppressed falls.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerReduceRelayGauge`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "reduce_relay_metrics{metric=\"selected_peers\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Selected [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "reduce_relay_metrics{metric=\"suppressed_peers\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Suppressed [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "reduce_relay_metrics{metric=\"not_enabled_peers\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Not Enabled [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Peer Count",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Reduce-Relay Missing-Tx Frequency",
+      "description": "###### What this is:\n*How often a peer has to fetch a transaction it missed because relay suppressed it.*\n\n###### How it's computed:\n*The reported frequency of on-demand missing-transaction fetches, per node.*\n\n###### Reading it:\n*Lower is better; near-flat means suppression is well tuned.*\n\n###### Healthy range:\n*Workload-dependent; a low, stable value is expected.*\n\n###### Watch for:\n*A rising trend, meaning suppression is too aggressive and the on-demand fetch path is growing.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerReduceRelayGauge`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "reduce_relay_metrics{metric=\"missing_tx_freq\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Missing Tx Freq [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Frequency",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["network", "peer"],
+  "templating": {
+    "list": [
+      {
+        "name": "DS_PROMETHEUS",
+        "type": "datasource",
+        "label": "Prometheus",
+        "query": "prometheus",
+        "regex": "",
+        "current": {},
+        "hide": 0,
+        "refresh": 1,
+        "includeAll": false,
+        "multi": false,
+        "options": []
+      },
+      {
+        "name": "service_name",
+        "label": "Service Name",
+        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
+        "type": "query",
+        "query": "label_values(service_name)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "xrpld",
+          "value": "xrpld"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "deployment_environment",
+        "label": "Environment",
+        "description": "Filter by deployment tier (local/ci/test/prod)",
+        "type": "query",
+        "query": "label_values(deployment_environment)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "xrpl_network_type",
+        "label": "Network",
+        "description": "Filter by XRPL network (mainnet/testnet/devnet)",
+        "type": "query",
+        "query": "label_values(xrpl_network_type)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id — e.g. Node-1)",
+        "type": "query",
+        "query": "label_values(span_calls_total, service_instance_id)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "proposal_trusted",
+        "label": "Proposal Trusted",
+        "description": "Filter by proposal trust status (true = from trusted validator)",
+        "type": "query",
+        "query": "label_values(span_calls_total{span_name=\"peer.proposal.receive\"}, proposal_trusted)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "validation_trusted",
+        "label": "Validation Trusted",
+        "description": "Filter by validation trust status (true = from trusted validator)",
+        "type": "query",
+        "query": "label_values(span_calls_total{span_name=\"peer.validation.receive\"}, validation_trusted)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Peer Network",
+  "uid": "peer-network",
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/peer-quality.json

@@ -0,0 +1,478 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "What this shows: Peer connection quality: latency, divergence, version distribution and upgrade guidance, resource disconnects, and inbound/outbound balance.\nUse it to: Assess how good this node's peer set is and whether connectivity or an upgrade needs attention.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "P90 Peer Latency",
+      "description": "###### What this is:\n*90th-percentile round-trip latency to connected peers, in milliseconds.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the p90 peer latency.*\n\n###### Reading it:\n*Lower is better; green under 200ms, yellow to 500ms, red above.*\n\n###### Healthy range:\n*Under 200ms.*\n\n###### Watch for:\n*Rising latency, which points to network congestion or geographically distant, poorly performing peers.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "peer_quality{metric=\"peer_latency_p90_ms\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "P90 Latency [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Latency (ms)",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10,
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          },
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 200
+              },
+              {
+                "color": "red",
+                "value": 500
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Insane/Diverged Peers",
+      "description": "###### What this is:\n*Count of connected peers whose ledger state has diverged from the network.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the diverged-peer count.*\n\n###### Reading it:\n*Zero is healthy; any count means those peers disagree on ledger state.*\n\n###### Healthy range:\n*0 diverged peers.*\n\n###### Watch for:\n*A persistent non-zero count, which can indicate peers on a fork or misbehaving peers.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "peer_quality{metric=\"peers_insane_count\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Insane Peers [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 3
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Higher Version Peers %",
+      "description": "###### What this is:\n*Percentage of connected peers running a newer rippled version than this node.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the higher-version peer percentage.*\n\n###### Reading it:\n*A high percentage suggests this node is behind and should be upgraded.*\n\n###### Healthy range:\n*Under 30%.*\n\n###### Watch for:\n*A majority of peers on a newer version, a strong upgrade signal.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "peer_quality{metric=\"peers_higher_version_pct\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Higher Version % [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 30
+              },
+              {
+                "color": "red",
+                "value": 60
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Upgrade Recommended",
+      "description": "###### What this is:\n*A flag indicating whether an upgrade is advised based on peer version analysis (Yes/No).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the upgrade-recommended flag.*\n\n###### Reading it:\n*No is healthy; Yes means most peers run a newer version.*\n\n###### Healthy range:\n*No.*\n\n###### Watch for:\n*A Yes state, indicating the node risks falling out of step with the network.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "peer_quality{metric=\"upgrade_recommended\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Upgrade [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": {
+                  "text": "No",
+                  "color": "green"
+                }
+              }
+            },
+            {
+              "type": "value",
+              "options": {
+                "1": {
+                  "text": "Yes",
+                  "color": "red"
+                }
+              }
+            }
+          ],
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Resource Disconnects",
+      "description": "###### What this is:\n*Cumulative count of peers dropped for exceeding resource (load) limits.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the resource-disconnect total over time.*\n\n###### Reading it:\n*A flat line is healthy; a rising line means peers are being dropped for overuse.*\n\n###### Healthy range:\n*Flat / near constant.*\n\n###### Watch for:\n*A steep climb, which flags aggressive or misbehaving peers being shed as backpressure.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 10,
+        "x": 6,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "server_info{metric=\"peer_disconnects_resources\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Disconnects [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "axisLabel": "Disconnects",
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10,
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          },
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Inbound vs Outbound Peers",
+      "description": "###### What this is:\n*Active inbound versus outbound peer connection counts.*\n\n###### How it's computed:\n*Instantaneous gauge readings of active inbound and outbound peer counts.*\n\n###### Reading it:\n*A balanced mix is healthy for connectivity and resilience.*\n\n###### Healthy range:\n*A mix of both; some outbound connections present.*\n\n###### Watch for:\n*All-inbound with no outbound, which usually means NAT or firewall issues block outbound peering.*\n\n###### Source:\n[PeerfinderManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/PeerfinderManager.cpp)\n\n###### Function:\n`Logic Stats ctor`",
+      "type": "bargauge",
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 8
+      },
+      "options": {
+        "orientation": "horizontal",
+        "displayMode": "gradient",
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "peer_finder_active_inbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Inbound [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "peer_finder_active_outbound_peers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}",
+          "legendFormat": "Outbound [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "custom": {}
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "Inbound.*"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "mode": "fixed",
+                  "fixedColor": "blue"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "Outbound.*"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "mode": "fixed",
+                  "fixedColor": "orange"
+                }
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["network", "peer"],
+  "templating": {
+    "list": [
+      {
+        "name": "DS_PROMETHEUS",
+        "type": "datasource",
+        "label": "Prometheus",
+        "query": "prometheus",
+        "regex": "",
+        "current": {},
+        "hide": 0,
+        "refresh": 1,
+        "includeAll": false,
+        "multi": false,
+        "options": []
+      },
+      {
+        "name": "service_name",
+        "label": "Service Name",
+        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
+        "type": "query",
+        "query": "label_values(service_name)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "xrpld",
+          "value": "xrpld"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "deployment_environment",
+        "label": "Environment",
+        "description": "Filter by deployment tier (local/ci/test/prod)",
+        "type": "query",
+        "query": "label_values(deployment_environment)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "xrpl_network_type",
+        "label": "Network",
+        "description": "Filter by XRPL network (mainnet/testnet/devnet)",
+        "type": "query",
+        "query": "label_values(xrpl_network_type)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(peer_quality, service_instance_id)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "Peer Quality",
+  "uid": "peer-quality",
+  "refresh": "5s"
+}

docker/telemetry/grafana/dashboards/rpc-pathfinding.json

@@ -0,0 +1,694 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "What this shows: RPC request throughput and response timing, pathfinding cost, gRPC read traffic, and resource-manager enforcement for this node.\nUse it to: Watch client request load and cost, and confirm resource limits are protecting the node.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "RPC Request Rate",
+      "description": "###### What this is:\n*How many RPC requests the server counts per second.*\n\n###### How it's computed:\n*Per-second rate of the RPC request counter over 5 minutes, per node.*\n\n###### Reading it:\n*A steady line proportional to client demand; cross-checks the trace-based RPC dashboard.*\n\n###### Healthy range:\n*Workload-dependent; tracks client activity.*\n\n###### Watch for:\n*A sudden surge above baseline, consistent with a client flooding the RPC endpoint.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "rate(rpc_requests_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])",
+          "legendFormat": "Requests / Sec [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Response Time",
+      "description": "###### What this is:\n*The 95th-percentile end-to-end RPC response time, including HTTP handling.*\n\n###### How it's computed:\n*95th-percentile of measured response times over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; this is broader than command-only latency.*\n\n###### Healthy range:\n*A few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A sustained climb, indicating handler overload or slow downstream work.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
+          "legendFormat": "P95 Response Time [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Latency (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Response Size",
+      "description": "###### What this is:\n*The 95th-percentile size of RPC response payloads in bytes.*\n\n###### How it's computed:\n*95th-percentile of response payload sizes over 5 minutes, per node.*\n\n###### Reading it:\n*Smaller is cheaper; large responses cost bandwidth and memory.*\n\n###### Healthy range:\n*Workload-dependent; small for status queries, large for bulk data queries.*\n\n###### Watch for:\n*Growth in large responses, consistent with expensive queries or API misuse.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(rpc_size_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
+          "legendFormat": "P95 Response Size [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decbytes",
+          "custom": {
+            "axisLabel": "Size (Bytes)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "RPC Response Time Distribution",
+      "description": "###### What this is:\n*RPC response time at the 90th, 95th, and 99th percentiles together.*\n\n###### How it's computed:\n*Three response-time quantiles over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; the gap between P90 and P99 shows the long tail.*\n\n###### Healthy range:\n*Percentiles clustered low; workload-dependent.*\n\n###### Watch for:\n*A widening P99, revealing long-tail or bimodal slow requests.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.9, sum by (le, service_instance_id) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
+          "legendFormat": "P90 [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
+          "legendFormat": "P95 [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le, service_instance_id) (rate(rpc_time_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
+          "legendFormat": "P99 [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Latency (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Pathfinding Fast Duration",
+      "description": "###### What this is:\n*The 95th-percentile time of the fast pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of fast pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; fast mode trades accuracy for speed.*\n\n###### Healthy range:\n*Typically a few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A rising trend, indicating heavy pathfinding demand or an expensive order book.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
+          "legendFormat": "P95 Fast Pathfind [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Pathfinding Full Duration",
+      "description": "###### What this is:\n*The 95th-percentile time of the full, exhaustive pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of full pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; full mode is much more expensive than fast mode.*\n\n###### Healthy range:\n*Tens to hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained high durations, consistent with pathfinding-heavy clients straining the node.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
+          "legendFormat": "P95 Full Pathfind [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Resource Warnings Rate",
+      "description": "###### What this is:\n*How often the resource manager warns a peer or client for excessive usage, per second.*\n\n###### How it's computed:\n*Per-second rate of resource-warning events over 5 minutes, per node.*\n\n###### Reading it:\n*Green near zero, yellow above 0.1/sec, red above 1/sec.*\n\n###### Healthy range:\n*Near zero in normal operation.*\n\n###### Watch for:\n*A rising rate, consistent with aggressive clients that may need throttling.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/resource/detail/Logic.h)\n\n###### Function:\n`Logic::Stats ctor`",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "rate(warn_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])",
+          "legendFormat": "Warnings / Sec [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.1
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Resource Drops Rate",
+      "description": "###### What this is:\n*How often the resource manager drops or blocks a peer or client for abuse, per second.*\n\n###### How it's computed:\n*Per-second rate of resource-drop events over 5 minutes, per node.*\n\n###### Reading it:\n*Green near zero, yellow above 0.01/sec, red above 0.1/sec.*\n\n###### Healthy range:\n*Zero when no abusive consumers are present.*\n\n###### Watch for:\n*Non-zero values, meaning the node is actively rejecting abusive connections.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/resource/detail/Logic.h)\n\n###### Function:\n`Logic::Stats ctor`",
+      "type": "stat",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "rate(drop_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])",
+          "legendFormat": "Drops / Sec [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "thresholds": {
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.01
+              },
+              {
+                "color": "red",
+                "value": 0.1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "gRPC Request Rate by Method (Spans)",
+      "description": "###### What this is:\n*How many gRPC calls of each method the node serves per second.*\n\n###### How it's computed:\n*Per-second rate of gRPC calls over 5 minutes, grouped by method and node.*\n\n###### Reading it:\n*Non-zero only when reporting/Clio-style clients use the gRPC API.*\n\n###### Healthy range:\n*Workload-dependent; zero without gRPC traffic.*\n\n###### Watch for:\n*A single method spiking, consistent with a heavy or misbehaving gRPC consumer.*\n\n###### Source:\n[GRPCServer.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/GRPCServer.cpp)\n\n###### Function:\n`GRPCServerImpl::CallData::process`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (method, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", method=~\"$grpc_method\", span_name=~\"grpc\\\\..*\"}[5m]))",
+          "legendFormat": "{{method}} [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Calls / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "gRPC Latency P95 by Method (Spans)",
+      "description": "###### What this is:\n*The 95th-percentile latency of each gRPC method.*\n\n###### How it's computed:\n*95th-percentile of gRPC call durations over 5 minutes, grouped by method and node.*\n\n###### Reading it:\n*Lower is better; identifies slow gRPC read paths.*\n\n###### Healthy range:\n*Workload-dependent; scales with ledger data volume served.*\n\n###### Watch for:\n*A method whose latency climbs, indicating an expensive read path.*\n\n###### Source:\n[GRPCServer.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/GRPCServer.cpp)\n\n###### Function:\n`GRPCServerImpl::CallData::process`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, method, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", method=~\"$grpc_method\", span_name=~\"grpc\\\\..*\"}[5m])))",
+          "legendFormat": "{{method}} [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "gRPC Error Rate by Status (Spans)",
+      "description": "###### What this is:\n*gRPC call rate broken down by outcome status (success or error).*\n\n###### How it's computed:\n*Per-second rate of gRPC calls over 5 minutes, grouped by status and node.*\n\n###### Reading it:\n*Success should dominate; the error rate should stay low.*\n\n###### Healthy range:\n*Workload-dependent; errors near the floor.*\n\n###### Watch for:\n*A rising error rate, indicating gRPC clients hitting failures.*\n\n###### Source:\n[GRPCServer.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/GRPCServer.cpp)\n\n###### Function:\n`GRPCServerImpl::CallData::process`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 40
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (grpc_status, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=~\"grpc\\\\..*\", grpc_status!=\"\"}[5m]))",
+          "legendFormat": "{{grpc_status}} [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Calls / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Pathfinding Compute Duration (Spans)",
+      "description": "###### What this is:\n*The 95th-percentile time to compute paths for a single request.*\n\n###### How it's computed:\n*95th-percentile of path-computation durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; complements the fast/full pathfinding timers with span-level detail.*\n\n###### Healthy range:\n*Workload-dependent; grows with order-book depth and request complexity.*\n\n###### Watch for:\n*A rising trend under pathfinding load, indicating expensive path computation.*\n\n###### Source:\n[PathRequest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequest.cpp)\n\n###### Function:\n`PathRequest::doUpdate`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 40
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"pathfind.compute\"}[5m])))",
+          "legendFormat": "P95 Compute [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "custom": {
+            "axisLabel": "Duration (ms)",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Pathfinding Request & Discovery Rate (Spans)",
+      "description": "###### What this is:\n*The rate of client path requests versus path-discovery passes.*\n\n###### How it's computed:\n*Per-second rates of path requests and discovery passes over 5 minutes, per node.*\n\n###### Reading it:\n*Discovery cost tracks request demand; useful for subscription-heavy nodes.*\n\n###### Healthy range:\n*Workload-dependent; both zero without pathfinding clients.*\n\n###### Watch for:\n*Discovery rate climbing out of proportion to requests, a cost driver for subscription-heavy nodes.*\n\n###### Source:\n[PathFind.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/handlers/orderbook/PathFind.cpp) · [PathRequest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequest.cpp)\n\n###### Function:\n`doPathFind ; PathRequest::findPaths`",
+      "type": "timeseries",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 48
+      },
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"pathfind.request\"}[5m]))",
+          "legendFormat": "Requests / Sec [{{service_instance_id}}]"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"pathfind.discover\"}[5m]))",
+          "legendFormat": "Discoveries / Sec [{{service_instance_id}}]"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "custom": {
+            "axisLabel": "Operations / Sec",
+            "spanNulls": true,
+            "insertNulls": false,
+            "showPoints": "auto",
+            "pointSize": 3
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["rpc", "pathfinding"],
+  "templating": {
+    "list": [
+      {
+        "name": "DS_PROMETHEUS",
+        "type": "datasource",
+        "label": "Prometheus",
+        "query": "prometheus",
+        "regex": "",
+        "current": {},
+        "hide": 0,
+        "refresh": 1,
+        "includeAll": false,
+        "multi": false,
+        "options": []
+      },
+      {
+        "name": "service_name",
+        "label": "Service Name",
+        "description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
+        "type": "query",
+        "query": "label_values(service_name)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "xrpld",
+          "value": "xrpld"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "deployment_environment",
+        "label": "Environment",
+        "description": "Filter by deployment tier (local/ci/test/prod)",
+        "type": "query",
+        "query": "label_values(deployment_environment)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "xrpl_network_type",
+        "label": "Network",
+        "description": "Filter by XRPL network (mainnet/testnet/devnet)",
+        "type": "query",
+        "query": "label_values(xrpl_network_type)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "node",
+        "label": "Node",
+        "description": "Filter by rippled node (service.instance.id)",
+        "type": "query",
+        "query": "label_values(rpc_requests_total, service_instance_id)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "name": "grpc_method",
+        "label": "gRPC Method",
+        "description": "Filter by gRPC method (GetLedger, GetLedgerData, GetLedgerDiff, GetLedgerEntry)",
+        "type": "query",
+        "query": "label_values(span_calls_total{span_name=~\"grpc\\\\..*\"}, method)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "title": "RPC & Pathfinding",
+  "uid": "rpc-pathfinding",
+  "refresh": "5s"
+}

docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml

@@ -0,0 +1,65 @@
+# Grafana contact-point provisioning for rippled OTel alerts.
+#
+# Phase 9: Internal metric gap fill — alerting on health-critical metrics.
+#
+# A contact point is where a firing alert is delivered. Two are defined:
+#   xrpld-default   — Slack only; receives warning-severity alerts.
+#   xrpld-critical  — Slack + email; receives critical-severity alerts.
+# The severity split is wired in policies.yaml.
+#
+# Secrets and personal addresses are NOT hard-coded. To enable delivery,
+# replace the disabled placeholder values below with a real Slack webhook
+# and alert email — e.g. by copying docker/telemetry/.env.alerting.example
+# to .env.alerting (gitignored) and swapping the placeholder lines to
+# ${SLACK_WEBHOOK_URL} / ${ALERT_EMAIL_TO}. See the Alerting section of
+# docs/telemetry-runbook.md for setup.
+#
+# The defaults are deliberately non-empty, invalid-but-valid-shaped
+# placeholders (an unroutable webhook host and a .invalid email). Grafana's
+# alerting provisioning validator REQUIRES a non-empty Slack url and email
+# addresses, and it does NOT support ${VAR:-default} expansion — an unset
+# ${VAR} expands to empty and crashes Grafana on startup. The placeholders
+# keep the whole stack booting with zero configuration; alerts simply route
+# nowhere until a real destination is supplied.
+#
+# Email delivery additionally requires SMTP configured on the Grafana
+# service (GF_SMTP_* in docker-compose.yml).
+
+apiVersion: 1
+
+contactPoints:
+  # --- Warning tier: Slack only ---
+  - orgId: 1
+    name: xrpld-default
+    receivers:
+      - uid: xrpld-slack-default
+        type: slack
+        settings:
+          # Disabled placeholder: an unroutable webhook host. A non-empty url
+          # selects Slack webhook mode (no recipient/token required) and keeps
+          # provisioning valid. Replace with a real ${SLACK_WEBHOOK_URL} to
+          # enable delivery.
+          url: https://hooks.slack.invalid/disabled
+          title: "{{ .CommonLabels.alertname }} on {{ .CommonLabels.service_instance_id }}"
+        disableResolveMessage: false
+
+  # --- Critical tier: Slack + email ---
+  - orgId: 1
+    name: xrpld-critical
+    receivers:
+      - uid: xrpld-slack-critical
+        type: slack
+        settings:
+          # Disabled placeholder — see xrpld-slack-default above.
+          url: https://hooks.slack.invalid/disabled
+          title: "[CRITICAL] {{ .CommonLabels.alertname }} on {{ .CommonLabels.service_instance_id }}"
+        disableResolveMessage: false
+      - uid: xrpld-email-critical
+        type: email
+        settings:
+          # Disabled placeholder: a .invalid address keeps the required
+          # `addresses` field non-empty so provisioning validates. Replace
+          # with a real ${ALERT_EMAIL_TO} (and enable SMTP) to deliver.
+          addresses: alerts-disabled@xrpld.invalid
+          singleEmail: true
+        disableResolveMessage: false

docker/telemetry/grafana/provisioning/alerting/policies.yaml

@@ -0,0 +1,44 @@
+# Grafana notification-policy provisioning for rippled OTel alerts.
+#
+# Phase 9: Internal metric gap fill — alerting on health-critical metrics.
+#
+# The notification policy tree decides which contact point receives a firing
+# alert and how alerts are batched. Routing is split by severity:
+#   - default (root)        → xrpld-default  (Slack) — catches warnings.
+#   - severity = critical   → xrpld-critical (Slack + email) — child route.
+# A critical alert matches the child route and is delivered to Slack AND email;
+# everything else falls through to the root receiver.
+#
+# Grouping by alertname + service_instance_id means one notification per
+# (alert, node) pair, so a fleet-wide problem does not collapse into a single
+# ambiguous message.
+
+apiVersion: 1
+
+policies:
+  - orgId: 1
+    receiver: xrpld-default
+    group_by:
+      - alertname
+      - service_instance_id
+    # Wait before sending the first notification for a new group, so a burst of
+    # related series is batched together.
+    group_wait: 30s
+    # Wait before adding new alerts to an already-notified group.
+    group_interval: 5m
+    # Minimum time before re-sending a notification for an alert that is still
+    # firing.
+    repeat_interval: 4h
+    routes:
+      # Critical alerts page harder: Slack + email. matchers uses the
+      # PromQL-style label match "severity = critical" set on the rules.
+      - receiver: xrpld-critical
+        matchers:
+          - severity = critical
+        group_by:
+          - alertname
+          - service_instance_id
+        group_wait: 30s
+        group_interval: 5m
+        # Re-page critical alerts more often than the 4h default while still firing.
+        repeat_interval: 1h

docker/telemetry/grafana/provisioning/alerting/rules.yaml

@@ -0,0 +1,410 @@
+# Grafana alert-rule provisioning for rippled OTel metrics.
+#
+# Phase 9: Internal metric gap fill — alerting on health-critical metrics.
+#
+# Six rules across three node-fatal subsystems: consensus/ledger health,
+# validator health, and the job queue. Every metric referenced here is
+# introduced by phase 9's MetricsRegistry.
+#
+# Rule shape (Grafana server-side evaluation):
+#   A  Prometheus query  — a 5-minute rate / histogram_quantile, aggregated
+#                          `by (service_instance_id)` so each node evaluates
+#                          independently. Alert rules run headless and cannot
+#                          use the dashboards' `$node` template variable.
+#   B  reduce (last)      — collapse A's series to its most recent value.
+#   C  threshold          — the firing condition; `condition: C`.
+#
+# Thresholds are documented in docs/telemetry-runbook.md (Alerting section)
+# and are intended to be tuned.
+
+apiVersion: 1
+
+groups:
+  # ------------------------------------------------------------------ #
+  # Consensus / ledger health                                          #
+  # ------------------------------------------------------------------ #
+  - orgId: 1
+    name: xrpld-consensus
+    folder: xrpld
+    interval: 1m
+    rules:
+      # A closed ledger that later fails validation against the network —
+      # any sustained nonzero rate means this node built history the rest
+      # of the network rejects.
+      - uid: xrpld-ledger-history-mismatch
+        title: LedgerHistoryMismatch
+        condition: C
+        for: 5m
+        noDataState: NoData
+        execErrState: Error
+        labels:
+          severity: critical
+          category: consensus
+        annotations:
+          summary: "Ledger history mismatch on {{ $labels.service_instance_id }}"
+          description: >-
+            Node {{ $labels.service_instance_id }} is recording ledger history
+            mismatches ({{ $values.B.Value }}/s over 5m). The node's built ledger
+            diverges from the validated network chain.
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: sum by (service_instance_id) (rate(ledger_history_mismatch_total[5m]))
+              instant: true
+              range: false
+              intervalMs: 1000
+              maxDataPoints: 43200
+          - refId: B
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: B
+              type: reduce
+              reducer: last
+              expression: A
+              datasource:
+                type: __expr__
+                uid: __expr__
+          - refId: C
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: B
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [0]
+              datasource:
+                type: __expr__
+                uid: __expr__
+
+      # Healthy nodes close a ledger every ~3-5s. Zero closes for 3 minutes
+      # means consensus/ledger advancement is stuck. NoData (metric absent)
+      # also fires — a vanished series here means the node is down.
+      - uid: xrpld-ledger-close-stalled
+        title: LedgerCloseStalled
+        condition: C
+        for: 3m
+        noDataState: Alerting
+        execErrState: Error
+        labels:
+          severity: critical
+          category: consensus
+        annotations:
+          summary: "Ledger closing stalled on {{ $labels.service_instance_id }}"
+          description: >-
+            Node {{ $labels.service_instance_id }} has closed no ledgers for
+            several minutes (5m rate has decayed to zero). Consensus or ledger
+            advancement is stuck.
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: sum by (service_instance_id) (rate(ledgers_closed_total[5m]))
+              instant: true
+              range: false
+              intervalMs: 1000
+              maxDataPoints: 43200
+          - refId: B
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: B
+              type: reduce
+              reducer: last
+              expression: A
+              datasource:
+                type: __expr__
+                uid: __expr__
+          - refId: C
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: B
+              conditions:
+                - evaluator:
+                    type: lt
+                    params: [0.001]
+              datasource:
+                type: __expr__
+                uid: __expr__
+
+  # ------------------------------------------------------------------ #
+  # Validator health                                                   #
+  # ------------------------------------------------------------------ #
+  - orgId: 1
+    name: xrpld-validator
+    folder: xrpld
+    interval: 1m
+    rules:
+      # This validator's own validations are not reaching / agreeing with
+      # the network. A sustained nonzero miss rate risks the validator being
+      # dropped from UNLs.
+      - uid: xrpld-validations-missed
+        title: ValidationsMissed
+        condition: C
+        for: 5m
+        noDataState: NoData
+        execErrState: Error
+        labels:
+          severity: warning
+          category: validator
+        annotations:
+          summary: "Validations missed on {{ $labels.service_instance_id }}"
+          description: >-
+            Node {{ $labels.service_instance_id }} is missing validations
+            ({{ $values.B.Value }}/s over 5m). Its validations are not agreeing with
+            the validated ledger.
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: sum by (service_instance_id) (rate(validation_missed_total[5m]))
+              instant: true
+              range: false
+              intervalMs: 1000
+              maxDataPoints: 43200
+          - refId: B
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: B
+              type: reduce
+              reducer: last
+              expression: A
+              datasource:
+                type: __expr__
+                uid: __expr__
+          - refId: C
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: B
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [0]
+              datasource:
+                type: __expr__
+                uid: __expr__
+
+      # The node has stopped checking incoming validations. Zero checked
+      # validations means it is no longer processing the validation stream
+      # from peers.
+      - uid: xrpld-validations-not-checked
+        title: ValidationsNotChecked
+        condition: C
+        for: 5m
+        noDataState: Alerting
+        execErrState: Error
+        labels:
+          severity: warning
+          category: validator
+        annotations:
+          summary: "No validations checked on {{ $labels.service_instance_id }}"
+          description: >-
+            Node {{ $labels.service_instance_id }} has checked no incoming
+            validations for several minutes (5m rate has decayed to zero). The
+            validation stream from peers may have stopped.
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: sum by (service_instance_id) (rate(validations_checked_total[5m]))
+              instant: true
+              range: false
+              intervalMs: 1000
+              maxDataPoints: 43200
+          - refId: B
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: B
+              type: reduce
+              reducer: last
+              expression: A
+              datasource:
+                type: __expr__
+                uid: __expr__
+          - refId: C
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: B
+              conditions:
+                - evaluator:
+                    type: lt
+                    params: [0.001]
+              datasource:
+                type: __expr__
+                uid: __expr__
+
+  # ------------------------------------------------------------------ #
+  # Job queue / resource health                                        #
+  # ------------------------------------------------------------------ #
+  - orgId: 1
+    name: xrpld-jobqueue
+    folder: xrpld
+    interval: 1m
+    rules:
+      # Transactions are being dropped because the job queue is full — the
+      # node is shedding load it cannot process.
+      - uid: xrpld-jobqueue-tx-overflow
+        title: JobQueueTxOverflow
+        condition: C
+        for: 5m
+        noDataState: NoData
+        execErrState: Error
+        labels:
+          severity: warning
+          category: jobqueue
+        annotations:
+          summary: "Job queue transaction overflow on {{ $labels.service_instance_id }}"
+          description: >-
+            Node {{ $labels.service_instance_id }} is overflowing its transaction
+            job queue ({{ $values.B.Value }}/s over 5m). Transactions are being
+            dropped under load.
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: sum by (service_instance_id) (rate(jq_trans_overflow_total[5m]))
+              instant: true
+              range: false
+              intervalMs: 1000
+              maxDataPoints: 43200
+          - refId: B
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: B
+              type: reduce
+              reducer: last
+              expression: A
+              datasource:
+                type: __expr__
+                uid: __expr__
+          - refId: C
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: B
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [0]
+              datasource:
+                type: __expr__
+                uid: __expr__
+
+      # p99 time a job waits in the queue before running. A sustained p99
+      # above 1s means the node is saturated and work is backing up. Tune the
+      # threshold (in microseconds) to the deployment.
+      - uid: xrpld-jobqueue-latency-high
+        title: JobQueueLatencyHigh
+        condition: C
+        for: 5m
+        noDataState: NoData
+        execErrState: Error
+        labels:
+          severity: warning
+          category: jobqueue
+        annotations:
+          summary: "Job queue latency high on {{ $labels.service_instance_id }}"
+          description: >-
+            Node {{ $labels.service_instance_id }} has a p99 job-queue wait of
+            {{ $values.B.Value }}µs (>1s) over 5m. The node is saturated and jobs are
+            backing up.
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: prometheus
+            model:
+              refId: A
+              expr: histogram_quantile(0.99, sum by (le, service_instance_id) (rate(job_queued_us_bucket[5m])))
+              instant: true
+              range: false
+              intervalMs: 1000
+              maxDataPoints: 43200
+          - refId: B
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: B
+              type: reduce
+              reducer: last
+              expression: A
+              datasource:
+                type: __expr__
+                uid: __expr__
+          - refId: C
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: B
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [1000000]
+              datasource:
+                type: __expr__
+                uid: __expr__

docker/telemetry/grafana/provisioning/dashboards/dashboards.yaml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: xrpld-telemetry
+    orgId: 1
+    folder: xrpld
+    type: file
+    disableDeletion: false
+    editable: true
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: false

docker/telemetry/grafana/provisioning/datasources/loki.yaml

@@ -0,0 +1,35 @@
+# Grafana Loki data source provisioning for rippled log-trace correlation.
+#
+# Loki ingests rippled logs via OTel Collector's filelog receiver.
+# The derivedFields config links trace_id values in log lines back to
+# Tempo traces, enabling one-click log-to-trace navigation in Grafana.
+
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    uid: loki
+    jsonData:
+      derivedFields:
+        # Trace ID is an OTel-issued 32-hex value emitted by Logs::format()
+        # as `trace_id=...`. Grafana treats the captured value as a Tempo
+        # trace ID, opening the trace directly via Tempo's trace-by-id API.
+        - datasourceUid: tempo
+          matcherRegex: "trace_id=(\\w+)"
+          name: TraceID
+          url: "$${__value.raw}"
+        # 64-char uppercase hex tokens in log bodies are XRPL ledger
+        # hashes (or tx hashes). They are NOT OTel trace IDs and cannot
+        # be resolved via the trace-by-id endpoint. Build a Grafana
+        # Explore deep-link that runs TraceQL on the span attribute
+        # `xrpl.consensus.ledger_id` — set on `consensus.round` spans to
+        # the full prev_ledger hash. This finds the round span whose
+        # first 16 bytes were folded into the OTel trace_id by
+        # SpanGuard::hashSpan().
+        - matcherRegex: "\\b([A-F0-9]{64})\\b"
+          name: ConsensusLedgerHash
+          urlDisplayLabel: "Search Tempo"
+          url: '/explore?schemaVersion=1&orgId=1&panes={"tempo":{"datasource":"tempo","queries":[{"refId":"A","queryType":"traceql","query":"{.xrpl.consensus.ledger_id=\"$${__value.raw}\"}"}],"range":{"from":"now-1h","to":"now"}}}'

docker/telemetry/grafana/provisioning/datasources/prometheus.yaml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true

docker/telemetry/grafana/provisioning/datasources/tempo.yaml

@@ -26,6 +26,14 @@ datasources:
       # Prometheus service is added to docker-compose.yml.
       serviceMap:
         datasourceUid: prometheus
+      # Trace-to-log correlation — enables one-click navigation
+      # from a Tempo trace to the corresponding Loki log lines. Filters
+      # by trace_id so only logs from the same trace are shown.
+      tracesToLogs:
+        datasourceUid: loki
+        filterByTraceID: true
+        filterBySpanID: false
+        tags: []
       tracesToMetrics:
         datasourceUid: prometheus
         spanStartTimeShift: "-1h"
@@ -39,10 +47,10 @@ datasources:
             tag: service.name
             operator: "="
             scope: resource
-            type: dynamic
-          # service.instance.id: unique node identifier — defaults to the
-          #   node's public key (e.g., nHB1X37...). Distinguishes individual
-          #   nodes in a multi-node cluster or network.
+            type: static
+          # service.instance.id: unique node identifier — configurable via
+          #   the service_instance_id setting in [telemetry], defaults to the
+          #   node's public key. E.g. "Node-1" or "nHB1X37...".
           - id: node-id
             tag: service.instance.id
             operator: "="
@@ -124,6 +132,12 @@ datasources:
             operator: "="
             scope: span
             type: dynamic
+          # tx_type: transaction type (e.g., "Payment", "OfferCreate").
+          - id: tx-type
+            tag: tx_type
+            operator: "="
+            scope: span
+            type: dynamic
           # Consensus tracing filters
           - id: consensus-mode
             tag: consensus_mode
@@ -140,6 +154,12 @@ datasources:
             operator: "="
             scope: span
             type: static
+          # ledger_hash: ledger hash — scope all spans to a specific closed ledger.
+          - id: ledger-hash
+            tag: ledger_hash
+            operator: "="
+            scope: span
+            type: static
           - id: consensus-close-time-correct
             tag: close_time_correct
             operator: "="
@@ -155,3 +175,54 @@ datasources:
             operator: "="
             scope: span
             type: dynamic
+          - id: consensus-proposers
+            tag: proposers
+            operator: "="
+            scope: span
+            type: dynamic
+          - id: consensus-result
+            tag: consensus_result
+            operator: "="
+            scope: span
+            type: dynamic
+          - id: consensus-mode-old
+            tag: mode_old
+            operator: "="
+            scope: span
+            type: dynamic
+          - id: consensus-mode-new
+            tag: mode_new
+            operator: "="
+            scope: span
+            type: dynamic
+          - id: consensus-ledger-id
+            tag: consensus_ledger_id
+            operator: "="
+            scope: span
+            type: static
+          # Additional transaction and queue filters
+          - id: tx-path
+            tag: path
+            operator: "="
+            scope: span
+            type: dynamic
+          - id: tx-suppressed
+            tag: suppressed
+            operator: "="
+            scope: span
+            type: dynamic
+          - id: peer-version
+            tag: peer_version
+            operator: "="
+            scope: span
+            type: dynamic
+          - id: txq-status
+            tag: txq_status
+            operator: "="
+            scope: span
+            type: dynamic
+          - id: txq-ter-code
+            tag: ter_code
+            operator: "="
+            scope: span
+            type: dynamic

docker/telemetry/integration-test.sh

@@ -0,0 +1,756 @@
+#!/usr/bin/env bash
+# Integration test for rippled OpenTelemetry instrumentation.
+#
+# Launches a 6-node xrpld consensus network with telemetry enabled,
+# exercises RPC / transaction / consensus code paths, then verifies
+# that the expected spans and metrics appear in Tempo and Prometheus.
+#
+# Usage:
+#   bash docker/telemetry/integration-test.sh
+#
+# Prerequisites:
+#   - .build/xrpld built with telemetry=ON
+#   - docker compose (v2)
+#   - curl, jq
+#
+# The script leaves the observability stack and xrpld nodes running
+# so you can manually inspect Tempo (localhost:3200) and Grafana
+# (localhost:3000). Run with --cleanup to tear down instead.
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+XRPLD="$REPO_ROOT/.build/xrpld"
+COMPOSE_FILE="$SCRIPT_DIR/docker-compose.yml"
+STANDALONE_CFG="$SCRIPT_DIR/xrpld-telemetry.cfg"
+WORKDIR="${WORKDIR:-/tmp/xrpld-integration}"
+NUM_NODES=6
+PEER_PORT_BASE=51235
+RPC_PORT_BASE=5005
+CONSENSUS_TIMEOUT=120
+GENESIS_ACCOUNT="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"
+GENESIS_SEED="snoPBrXtMeMyMHUVTgbuqAfg1SUTb"
+DEST_ACCOUNT="" # Generated dynamically via wallet_propose
+TEMPO="http://localhost:3200"
+PROM="http://localhost:9090"
+
+# Counters for pass/fail
+PASS=0
+FAIL=0
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+log() { printf "\033[1;34m[INFO]\033[0m  %s\n" "$*"; }
+ok() {
+    printf "\033[1;32m[PASS]\033[0m  %s\n" "$*"
+    PASS=$((PASS + 1))
+}
+fail() {
+    printf "\033[1;31m[FAIL]\033[0m  %s\n" "$*"
+    FAIL=$((FAIL + 1))
+}
+die() {
+    printf "\033[1;31m[ERROR]\033[0m %s\n" "$*" >&2
+    exit 1
+}
+
+check_span() {
+    local op="$1"
+    local count
+    count=$(curl -sf "$TEMPO/api/search" \
+        --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \
+        --data-urlencode "limit=5" |
+        jq '.traces | length' 2>/dev/null || echo 0)
+    if [ "$count" -gt 0 ]; then
+        ok "$op  ($count traces)"
+    else
+        fail "$op  (0 traces)"
+    fi
+}
+
+# Verify trace_id injection in xrpld log output.
+# Greps all node debug.log files for the "trace_id=<hex> span_id=<hex>"
+# pattern that Logs::format() injects when an active OTel span exists.
+# Also cross-checks that a trace_id found in logs matches a trace in Tempo.
+check_log_correlation() {
+    log "Checking log-trace correlation..."
+
+    local total_matches=0
+    local files_scanned=0
+    local sample_trace_id=""
+
+    for i in $(seq 1 "$NUM_NODES"); do
+        local logfile="$WORKDIR/node$i/debug.log"
+        if [ ! -f "$logfile" ]; then
+            continue
+        fi
+        files_scanned=$((files_scanned + 1))
+        local matches
+        matches=$(grep -c 'trace_id=[a-f0-9]\{32\} span_id=[a-f0-9]\{16\}' "$logfile") || matches=0
+        total_matches=$((total_matches + matches))
+        # Capture the first trace_id we find for cross-referencing with Tempo
+        if [ -z "$sample_trace_id" ] && [ "$matches" -gt 0 ]; then
+            sample_trace_id=$(grep -o 'trace_id=[a-f0-9]\{32\}' "$logfile" | head -1 | cut -d= -f2)
+        fi
+    done
+
+    if [ "$files_scanned" -eq 0 ]; then
+        fail "Log correlation: no debug.log files found in $WORKDIR/node*/"
+        return
+    fi
+
+    if [ "$total_matches" -gt 0 ]; then
+        ok "Log correlation: found $total_matches log lines with trace_id ($files_scanned nodes scanned)"
+    else
+        fail "Log correlation: no trace_id found in any node debug.log ($files_scanned nodes scanned)"
+    fi
+
+    # Cross-check: verify the sample trace_id exists in Tempo
+    if [ -n "$sample_trace_id" ]; then
+        local trace_found
+        # Tempo /api/traces/{id} returns OTLP shape: {"batches":[...]}
+        trace_found=$(curl -sf "$TEMPO/api/traces/$sample_trace_id" |
+            jq '.batches | length' 2>/dev/null) || trace_found=0
+        if [ "$trace_found" -gt 0 ]; then
+            ok "Log-Tempo cross-check: trace_id=$sample_trace_id found in Tempo"
+        else
+            fail "Log-Tempo cross-check: trace_id=$sample_trace_id NOT found in Tempo"
+        fi
+    fi
+}
+
+cleanup() {
+    log "Cleaning up..."
+    # Kill xrpld nodes
+    for i in $(seq 1 "$NUM_NODES"); do
+        local pidfile="$WORKDIR/node$i/xrpld.pid"
+        if [ -f "$pidfile" ]; then
+            kill "$(cat "$pidfile")" 2>/dev/null || true
+            rm -f "$pidfile"
+        fi
+    done
+    # Also kill any straggling xrpld processes from our workdir
+    pkill -f "$WORKDIR" 2>/dev/null || true
+    # Stop docker stack
+    docker compose -f "$COMPOSE_FILE" down 2>/dev/null || true
+    # Remove workdir
+    rm -rf "$WORKDIR"
+    log "Cleanup complete."
+}
+
+# Handle --cleanup flag
+if [ "${1:-}" = "--cleanup" ]; then
+    cleanup
+    exit 0
+fi
+
+# ---------------------------------------------------------------------------
+# Step 0: Prerequisites
+# ---------------------------------------------------------------------------
+log "Checking prerequisites..."
+
+command -v docker >/dev/null 2>&1 || die "docker not found"
+docker compose version >/dev/null 2>&1 || die "docker compose (v2) not found"
+command -v curl >/dev/null 2>&1 || die "curl not found"
+command -v jq >/dev/null 2>&1 || die "jq not found"
+[ -x "$XRPLD" ] || die "xrpld binary not found at $XRPLD (build with telemetry=ON)"
+[ -f "$COMPOSE_FILE" ] || die "docker-compose.yml not found at $COMPOSE_FILE"
+[ -f "$STANDALONE_CFG" ] || die "xrpld-telemetry.cfg not found at $STANDALONE_CFG"
+
+log "All prerequisites met."
+
+# ---------------------------------------------------------------------------
+# Step 1: Clean previous run
+# ---------------------------------------------------------------------------
+log "Cleaning previous run data..."
+for i in $(seq 1 "$NUM_NODES"); do
+    pidfile="$WORKDIR/node$i/xrpld.pid"
+    if [ -f "$pidfile" ]; then
+        kill "$(cat "$pidfile")" 2>/dev/null || true
+    fi
+done
+pkill -f "$WORKDIR" 2>/dev/null || true
+# Kill any xrpld using the standalone config (from key generation)
+pkill -f "xrpld-telemetry.cfg" 2>/dev/null || true
+sleep 2
+rm -rf "$WORKDIR"
+mkdir -p "$WORKDIR"
+
+# ---------------------------------------------------------------------------
+# Step 2: Start observability stack
+# ---------------------------------------------------------------------------
+log "Starting observability stack..."
+# Point the collector's log mount at this test's workdir so it tails the
+# per-node debug.log files this script generates. The compose default
+# (./data/logs) is for user-run xrpld; the test owns its own log root.
+XRPLD_LOG_DIR="$WORKDIR" docker compose -f "$COMPOSE_FILE" up -d
+
+log "Waiting for otel-collector to be ready..."
+for attempt in $(seq 1 30); do
+    # The OTLP HTTP endpoint returns 405 for GET (expects POST), which
+    # means it is listening.  curl -sf would fail on 405, so we check
+    # the HTTP status code explicitly.
+    status=$(curl -so /dev/null -w '%{http_code}' http://localhost:4318/ 2>/dev/null || echo 000)
+    if [ "$status" != "000" ]; then
+        log "otel-collector ready (attempt $attempt, HTTP $status)."
+        break
+    fi
+    if [ "$attempt" -eq 30 ]; then
+        die "otel-collector not ready after 30s"
+    fi
+    sleep 1
+done
+
+log "Waiting for Tempo to be ready..."
+for attempt in $(seq 1 30); do
+    if curl -sf "$TEMPO/ready" >/dev/null 2>&1; then
+        log "Tempo ready (attempt $attempt)."
+        break
+    fi
+    if [ "$attempt" -eq 30 ]; then
+        die "Tempo not ready after 30s"
+    fi
+    sleep 1
+done
+
+# ---------------------------------------------------------------------------
+# Step 3: Generate validator keys
+# ---------------------------------------------------------------------------
+log "Generating $NUM_NODES validator key pairs..."
+
+# Start a temporary standalone xrpld for key generation
+TEMP_DATA="$WORKDIR/temp-keygen"
+mkdir -p "$TEMP_DATA"
+
+# Create a minimal temp config for key generation
+TEMP_CFG="$TEMP_DATA/xrpld.cfg"
+cat >"$TEMP_CFG" <<EOCFG
+[server]
+port_rpc_temp
+
+[port_rpc_temp]
+port = 5099
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[node_db]
+type=NuDB
+path=$TEMP_DATA/nudb
+online_delete=256
+
+[database_path]
+$TEMP_DATA/db
+
+[debug_logfile]
+$TEMP_DATA/debug.log
+
+[ssl_verify]
+0
+EOCFG
+
+"$XRPLD" --conf "$TEMP_CFG" -a --start >"$TEMP_DATA/stdout.log" 2>&1 &
+TEMP_PID=$!
+log "Temporary xrpld started (PID $TEMP_PID), waiting for RPC..."
+
+for attempt in $(seq 1 30); do
+    if curl -sf http://localhost:5099 -d '{"method":"server_info"}' >/dev/null 2>&1; then
+        log "Temporary xrpld RPC ready (attempt $attempt)."
+        break
+    fi
+    if [ "$attempt" -eq 30 ]; then
+        kill "$TEMP_PID" 2>/dev/null || true
+        die "Temporary xrpld RPC not ready after 30s"
+    fi
+    sleep 1
+done
+
+declare -a SEEDS
+declare -a PUBKEYS
+
+for i in $(seq 1 "$NUM_NODES"); do
+    result=$(curl -sf http://localhost:5099 -d '{"method":"validation_create"}')
+    seed=$(echo "$result" | jq -r '.result.validation_seed')
+    pubkey=$(echo "$result" | jq -r '.result.validation_public_key')
+    if [ -z "$seed" ] || [ "$seed" = "null" ]; then
+        kill "$TEMP_PID" 2>/dev/null || true
+        die "Failed to generate key pair $i"
+    fi
+    SEEDS+=("$seed")
+    PUBKEYS+=("$pubkey")
+    log "  Node $i: $pubkey"
+done
+
+kill "$TEMP_PID" 2>/dev/null || true
+wait "$TEMP_PID" 2>/dev/null || true
+rm -rf "$TEMP_DATA"
+log "Key generation complete."
+
+# ---------------------------------------------------------------------------
+# Step 4: Generate node configs and validators.txt
+# ---------------------------------------------------------------------------
+log "Generating node configs..."
+
+# Create shared validators.txt
+VALIDATORS_FILE="$WORKDIR/validators.txt"
+{
+    echo "[validators]"
+    for i in $(seq 0 $((NUM_NODES - 1))); do
+        echo "${PUBKEYS[$i]}"
+    done
+} >"$VALIDATORS_FILE"
+
+# Create per-node configs
+for i in $(seq 1 "$NUM_NODES"); do
+    NODE_DIR="$WORKDIR/node$i"
+    mkdir -p "$NODE_DIR/nudb" "$NODE_DIR/db"
+
+    RPC_PORT=$((RPC_PORT_BASE + i - 1))
+    PEER_PORT=$((PEER_PORT_BASE + i - 1))
+    SEED="${SEEDS[$((i - 1))]}"
+
+    # Build ips_fixed list (all peers except self)
+    IPS_FIXED=""
+    for j in $(seq 1 "$NUM_NODES"); do
+        if [ "$j" -ne "$i" ]; then
+            IPS_FIXED="${IPS_FIXED}127.0.0.1 $((PEER_PORT_BASE + j - 1))
+"
+        fi
+    done
+
+    cat >"$NODE_DIR/xrpld.cfg" <<EOCFG
+[server]
+port_rpc
+port_peer
+
+[port_rpc]
+port = $RPC_PORT
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[port_peer]
+port = $PEER_PORT
+ip = 0.0.0.0
+protocol = peer
+
+[node_db]
+type=NuDB
+path=$NODE_DIR/nudb
+online_delete=256
+
+[database_path]
+$NODE_DIR/db
+
+[debug_logfile]
+$NODE_DIR/debug.log
+
+[validation_seed]
+$SEED
+
+[validators_file]
+$VALIDATORS_FILE
+
+[ips_fixed]
+${IPS_FIXED}
+[peer_private]
+1
+
+[telemetry]
+enabled=1
+service_instance_id=Node-${i}
+endpoint=http://localhost:4318/v1/traces
+exporter=otlp_http
+batch_size=512
+batch_delay_ms=2000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1
+metrics_endpoint=http://localhost:4318/v1/metrics
+
+[insight]
+server=otel
+endpoint=http://localhost:4318/v1/metrics
+prefix=rippled
+service_instance_id=Node-${i}
+
+[insight]
+server=statsd
+address=127.0.0.1:8125
+prefix=rippled
+
+[rpc_startup]
+{ "command": "log_level", "severity": "warning" }
+
+[ssl_verify]
+0
+EOCFG
+
+    log "  Node $i config: RPC=$RPC_PORT, Peer=$PEER_PORT"
+done
+
+# ---------------------------------------------------------------------------
+# Step 5: Start all 6 nodes
+# ---------------------------------------------------------------------------
+log "Starting $NUM_NODES xrpld nodes..."
+
+for i in $(seq 1 "$NUM_NODES"); do
+    NODE_DIR="$WORKDIR/node$i"
+    "$XRPLD" --conf "$NODE_DIR/xrpld.cfg" --start >"$NODE_DIR/stdout.log" 2>&1 &
+    echo $! >"$NODE_DIR/xrpld.pid"
+    log "  Node $i started (PID $(cat "$NODE_DIR/xrpld.pid"))"
+done
+
+# Give nodes a moment to initialize
+sleep 5
+
+# ---------------------------------------------------------------------------
+# Step 6: Wait for consensus
+# ---------------------------------------------------------------------------
+log "Waiting for nodes to reach 'proposing' state (timeout: ${CONSENSUS_TIMEOUT}s)..."
+
+start_time=$(date +%s)
+nodes_ready=0
+
+while [ "$nodes_ready" -lt "$NUM_NODES" ]; do
+    elapsed=$(($(date +%s) - start_time))
+    if [ "$elapsed" -ge "$CONSENSUS_TIMEOUT" ]; then
+        fail "Consensus timeout after ${CONSENSUS_TIMEOUT}s ($nodes_ready/$NUM_NODES nodes ready)"
+        log "Continuing with partial consensus..."
+        break
+    fi
+
+    nodes_ready=0
+    for i in $(seq 1 "$NUM_NODES"); do
+        RPC_PORT=$((RPC_PORT_BASE + i - 1))
+        state=$(curl -sf "http://localhost:$RPC_PORT" \
+            -d '{"method":"server_info"}' 2>/dev/null |
+            jq -r '.result.info.server_state' 2>/dev/null || echo "unreachable")
+        if [ "$state" = "proposing" ]; then
+            nodes_ready=$((nodes_ready + 1))
+        fi
+    done
+    printf "\r  %d/%d nodes proposing (%ds elapsed)..." "$nodes_ready" "$NUM_NODES" "$elapsed"
+    if [ "$nodes_ready" -lt "$NUM_NODES" ]; then
+        sleep 3
+    fi
+done
+echo ""
+
+if [ "$nodes_ready" -eq "$NUM_NODES" ]; then
+    ok "All $NUM_NODES nodes reached 'proposing' state"
+else
+    fail "Only $nodes_ready/$NUM_NODES nodes reached 'proposing' state"
+fi
+
+# ---------------------------------------------------------------------------
+# Step 6b: Wait for validated ledger
+# ---------------------------------------------------------------------------
+log "Waiting for first validated ledger..."
+for attempt in $(seq 1 60); do
+    val_seq=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
+        -d '{"method":"server_info"}' 2>/dev/null |
+        jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
+    if [ "$val_seq" -gt 2 ] 2>/dev/null; then
+        ok "First validated ledger: seq $val_seq"
+        break
+    fi
+    if [ "$attempt" -eq 60 ]; then
+        fail "No validated ledger after 60s"
+    fi
+    sleep 1
+done
+
+# ---------------------------------------------------------------------------
+# Step 7: Exercise RPC spans (Phase 2)
+# ---------------------------------------------------------------------------
+log "Exercising RPC spans..."
+
+curl -sf "http://localhost:$RPC_PORT_BASE" \
+    -d '{"method":"server_info"}' >/dev/null
+curl -sf "http://localhost:$RPC_PORT_BASE" \
+    -d '{"method":"server_state"}' >/dev/null
+curl -sf "http://localhost:$RPC_PORT_BASE" \
+    -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' >/dev/null
+
+log "RPC commands sent. Waiting 5s for batch export..."
+sleep 5
+
+# ---------------------------------------------------------------------------
+# Step 8: Submit transaction (Phase 3)
+# ---------------------------------------------------------------------------
+log "Submitting Payment transaction..."
+
+# Generate a destination wallet
+log "  Generating destination wallet..."
+wallet_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
+    -d '{"method":"wallet_propose"}')
+DEST_ACCOUNT=$(echo "$wallet_result" | jq -r '.result.account_id' 2>/dev/null)
+if [ -z "$DEST_ACCOUNT" ] || [ "$DEST_ACCOUNT" = "null" ]; then
+    fail "Could not generate destination wallet"
+    DEST_ACCOUNT="rrrrrrrrrrrrrrrrrrrrrhoLvTp" # ACCOUNT_ZERO fallback
+fi
+log "  Destination: $DEST_ACCOUNT"
+
+# Get genesis account info
+acct_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
+    -d "{\"method\":\"account_info\",\"params\":[{\"account\":\"$GENESIS_ACCOUNT\"}]}")
+seq_num=$(echo "$acct_result" | jq -r '.result.account_data.Sequence' 2>/dev/null || echo "unknown")
+log "  Genesis account sequence: $seq_num"
+
+# Submit payment
+submit_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
+    -d "{\"method\":\"submit\",\"params\":[{\"secret\":\"$GENESIS_SEED\",\"tx_json\":{\"TransactionType\":\"Payment\",\"Account\":\"$GENESIS_ACCOUNT\",\"Destination\":\"$DEST_ACCOUNT\",\"Amount\":\"10000000\"}}]}")
+
+engine_result=$(echo "$submit_result" | jq -r '.result.engine_result' 2>/dev/null || echo "unknown")
+tx_hash=$(echo "$submit_result" | jq -r '.result.tx_json.hash' 2>/dev/null || echo "unknown")
+
+if [ "$engine_result" = "tesSUCCESS" ] || [ "$engine_result" = "terQUEUED" ]; then
+    ok "Transaction submitted: $engine_result (hash: ${tx_hash:0:16}...)"
+else
+    fail "Transaction submission: $engine_result"
+    log "  Full response: $(echo "$submit_result" | jq -c .result 2>/dev/null)"
+fi
+
+log "Waiting 15s for consensus round + batch export..."
+sleep 15
+
+# ---------------------------------------------------------------------------
+# Step 9: Verify Tempo traces
+# ---------------------------------------------------------------------------
+log "Verifying spans in Tempo..."
+
+# Check service registration
+services=$(curl -sf "$TEMPO/api/v2/search/tag/resource.service.name/values" |
+    jq -r '.tagValues[].value' 2>/dev/null || echo "")
+if echo "$services" | grep -q "xrpld"; then
+    ok "Service 'xrpld' registered in Tempo"
+else
+    fail "Service 'xrpld' NOT found in Tempo (found: $services)"
+fi
+
+log ""
+log "--- Phase 2: RPC Spans ---"
+check_span "rpc.request"
+check_span "rpc.process"
+check_span "rpc.command.server_info"
+check_span "rpc.command.server_state"
+check_span "rpc.command.ledger"
+
+log ""
+log "--- Phase 3: Transaction Spans ---"
+check_span "tx.process"
+check_span "tx.receive"
+check_span "tx.apply"
+
+log ""
+log "--- Phase 4: Consensus Spans ---"
+check_span "consensus.proposal.send"
+check_span "consensus.ledger_close"
+check_span "consensus.accept"
+check_span "consensus.validation.send"
+
+log ""
+log "--- Phase 5: Ledger Spans ---"
+check_span "ledger.build"
+check_span "ledger.validate"
+check_span "ledger.store"
+
+log ""
+log "--- Phase 5: Peer Spans (trace_peer=1) ---"
+check_span "peer.proposal.receive"
+check_span "peer.validation.receive"
+
+# ---------------------------------------------------------------------------
+# Step 9b: Verify log-trace correlation (Phase 8)
+# ---------------------------------------------------------------------------
+log ""
+log "--- Log-Trace Correlation ---"
+check_log_correlation
+
+# ---------------------------------------------------------------------------
+# Step 10: Verify Prometheus spanmetrics
+# ---------------------------------------------------------------------------
+log ""
+log "--- Phase 5: Spanmetrics ---"
+log "Waiting 20s for Prometheus scrape cycle..."
+sleep 20
+
+calls_count=$(curl -sf "$PROM/api/v1/query?query=span_calls_total" |
+    jq '.data.result | length' 2>/dev/null || echo 0)
+if [ "$calls_count" -gt 0 ]; then
+    ok "Prometheus: span_calls_total ($calls_count series)"
+else
+    fail "Prometheus: span_calls_total (0 series)"
+fi
+
+duration_count=$(curl -sf "$PROM/api/v1/query?query=span_duration_milliseconds_count" |
+    jq '.data.result | length' 2>/dev/null || echo 0)
+if [ "$duration_count" -gt 0 ]; then
+    ok "Prometheus: duration histogram ($duration_count series)"
+else
+    fail "Prometheus: duration histogram (0 series)"
+fi
+
+# Check Grafana
+if curl -sf http://localhost:3000/api/health >/dev/null 2>&1; then
+    ok "Grafana: healthy at localhost:3000"
+else
+    fail "Grafana: not reachable at localhost:3000"
+fi
+
+# ---------------------------------------------------------------------------
+# Step 10b: Verify native OTel metrics in Prometheus (beast::insight)
+# ---------------------------------------------------------------------------
+log ""
+log "--- Phase 7: Native OTel Metrics (beast::insight via OTLP) ---"
+log "Waiting 20s for OTLP metric export + Prometheus scrape..."
+sleep 20
+
+check_otel_metric() {
+    local metric_name="$1"
+    local result
+    result=$(curl -sf "$PROM/api/v1/query?query=$metric_name" |
+        jq '.data.result | length' 2>/dev/null || echo 0)
+    if [ "$result" -gt 0 ]; then
+        ok "OTel: $metric_name ($result series)"
+    else
+        fail "OTel: $metric_name (0 series)"
+    fi
+}
+
+# Node health gauges (ObservableGauge — no _total suffix)
+check_otel_metric "ledgermaster_validated_ledger_age"
+check_otel_metric "ledgermaster_published_ledger_age"
+check_otel_metric "job_count"
+
+# State accounting
+check_otel_metric "state_accounting_full_duration"
+
+# Peer finder
+check_otel_metric "peer_finder_active_inbound_peers"
+check_otel_metric "peer_finder_active_outbound_peers"
+
+# RPC counters (Counter — Prometheus adds _total suffix automatically)
+check_otel_metric "rpc_requests_total"
+
+# Overlay traffic
+check_otel_metric "total_bytes_in"
+
+# Verify StatsD receiver is NOT required (no statsd receiver in pipeline)
+log ""
+log "--- Verify StatsD receiver is not required ---"
+# StatsD listens on UDP 8125, so probe with a UDP-aware tool, not curl (TCP).
+if command -v ss >/dev/null 2>&1; then
+    if ss -ulnp 2>/dev/null | grep -q ":8125"; then
+        fail "StatsD port 8125 appears to be listening (should not be needed)"
+    else
+        ok "StatsD port 8125 is not listening (not required)"
+    fi
+else
+    log "ss not found -- skipping StatsD UDP port check"
+fi
+
+# ---------------------------------------------------------------------------
+# Step 10c: Verify Phase 9 OTel SDK Metrics
+# ---------------------------------------------------------------------------
+log ""
+log "--- Phase 9: OTel SDK Metrics (MetricsRegistry) ---"
+log "Waiting 15s for OTel metric export + Prometheus scrape..."
+sleep 15
+
+check_otel_metric() {
+    local metric_name="$1"
+    local result
+    result=$(curl -sf "$PROM/api/v1/query?query=$metric_name" |
+        jq '.data.result | length' 2>/dev/null || echo 0)
+    if [ "$result" -gt 0 ]; then
+        ok "OTel: $metric_name ($result series)"
+    else
+        fail "OTel: $metric_name (0 series)"
+    fi
+}
+
+# Task 9.1: NodeStore I/O
+check_otel_metric 'nodestore_state{metric="node_reads_total"}'
+check_otel_metric 'nodestore_state{metric="write_load"}'
+
+# Task 9.2: Cache hit rates
+check_otel_metric 'cache_metrics{metric="SLE_hit_rate"}'
+check_otel_metric 'cache_metrics{metric="treenode_cache_size"}'
+
+# Task 9.3: TxQ metrics
+check_otel_metric 'txq_metrics{metric="txq_count"}'
+check_otel_metric 'txq_metrics{metric="txq_reference_fee_level"}'
+
+# Task 9.4: Per-RPC metrics
+check_otel_metric "rpc_method_started_total"
+check_otel_metric "rpc_method_finished_total"
+
+# Task 9.5: Per-job metrics
+check_otel_metric "job_queued_total"
+check_otel_metric "job_finished_total"
+
+# Task 9.6: Counted object instances
+check_otel_metric "object_count"
+
+# Task 9.7: Load factor breakdown
+check_otel_metric 'load_factor_metrics{metric="load_factor"}'
+check_otel_metric 'load_factor_metrics{metric="load_factor_server"}'
+
+# Task 7.15 / Phase 9: ValidationTracker rolling-window agreement gauge.
+# MetricsRegistry::registerValidationAgreementGauge() publishes
+# validation_agreement with a `metric` label for each window
+# (1h / 24h / 7d) plus the matching agreement/miss counts. The 7-day
+# window matches the external xrpl-validator-dashboard parity target.
+check_otel_metric 'validation_agreement{metric="agreement_pct_1h"}'
+check_otel_metric 'validation_agreement{metric="agreement_pct_24h"}'
+check_otel_metric 'validation_agreement{metric="agreement_pct_7d"}'
+check_otel_metric 'validation_agreement{metric="agreements_1h"}'
+check_otel_metric 'validation_agreement{metric="missed_1h"}'
+check_otel_metric 'validation_agreement{metric="agreements_24h"}'
+check_otel_metric 'validation_agreement{metric="missed_24h"}'
+check_otel_metric 'validation_agreement{metric="agreements_7d"}'
+check_otel_metric 'validation_agreement{metric="missed_7d"}'
+
+# ---------------------------------------------------------------------------
+# Step 11: Summary
+# ---------------------------------------------------------------------------
+echo ""
+echo "==========================================================="
+echo "  INTEGRATION TEST RESULTS"
+echo "==========================================================="
+printf "  \033[1;32mPASSED: %d\033[0m\n" "$PASS"
+printf "  \033[1;31mFAILED: %d\033[0m\n" "$FAIL"
+echo "==========================================================="
+echo ""
+echo "  Observability stack is running:"
+echo ""
+echo "    Tempo:         http://localhost:3200"
+echo "    Grafana:       http://localhost:3000"
+echo "    Prometheus:    http://localhost:9090"
+echo "    Loki:          http://localhost:3100"
+echo ""
+echo "  xrpld nodes (6) are running:"
+for i in $(seq 1 "$NUM_NODES"); do
+    RPC_PORT=$((RPC_PORT_BASE + i - 1))
+    PEER_PORT=$((PEER_PORT_BASE + i - 1))
+    echo "    Node $i: RPC=localhost:$RPC_PORT  Peer=:$PEER_PORT  PID=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null || echo 'unknown')"
+done
+echo ""
+echo "  To tear down:"
+echo "    bash docker/telemetry/integration-test.sh --cleanup"
+echo ""
+echo "==========================================================="
+
+if [ "$FAIL" -gt 0 ]; then
+    exit 1
+fi

docker/telemetry/otel-collector-config.grafanacloud.yaml

@@ -0,0 +1,230 @@
+# OpenTelemetry Collector configuration — Grafana Cloud dual-export variant.
+#
+# Identical to otel-collector-config.yaml (local Tempo/Prometheus/Loki) but
+# ALSO ships all three signals to Grafana Cloud over a single OTLP/HTTP
+# endpoint. Local backends are kept so the on-box stack still works as a
+# fallback; remove the local exporters from the pipelines below if you want
+# cloud-only.
+#
+# Selecting this config is a RUNTIME choice — no code or image changes:
+#   Local only (default):
+#     docker compose -f docker/telemetry/docker-compose.yml up -d
+#   Local + Grafana Cloud:
+#     docker compose -f docker/telemetry/docker-compose.yml \
+#                    -f docker/telemetry/docker-compose.grafanacloud.yaml up -d
+#
+# The override file mounts THIS config over the collector's config path and
+# injects the three required secrets as environment variables:
+#   GRAFANA_CLOUD_OTLP_ENDPOINT  e.g. https://otlp-gateway-<zone>.grafana.net/otlp
+#   GRAFANA_CLOUD_INSTANCE_ID    numeric Grafana Cloud stack/instance id
+#   GRAFANA_CLOUD_API_TOKEN      a Cloud Access Policy token with metrics/traces/logs:write
+# Find all three under Grafana Cloud -> Connections -> "OpenTelemetry (OTLP)".
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+  # Basic-auth for the Grafana Cloud OTLP gateway: username = instance id,
+  # password = API token. Values come from the container environment so no
+  # secret is committed to the repo.
+  basicauth/grafanacloud:
+    client_auth:
+      username: ${env:GRAFANA_CLOUD_INSTANCE_ID}
+      password: ${env:GRAFANA_CLOUD_API_TOKEN}
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+  filelog:
+    include:
+      - /var/log/xrpld/*/debug.log
+    operators:
+      - type: regex_parser
+        regex: '^(?P<timestamp>\S+\s+\S+)\s+\S+\s+(?:(?P<partition>\S+):)?(?P<severity>\S+)\s+(?:trace_id=(?P<trace_id>[a-f0-9]+)\s+span_id=(?P<span_id>[a-f0-9]+)\s+)?(?P<message>.*)$'
+        timestamp:
+          parse_from: attributes.timestamp
+          layout: "%Y-%b-%d %H:%M:%S.%f"
+          location: UTC
+
+processors:
+  batch:
+    timeout: 1s
+    send_batch_size: 100
+  # Box-only (uncommitted): probabilistic 0.5% tail sampling on the
+  # trace-storage branch only. spanmetrics-derived metrics see 100% of spans
+  # (they run on a separate branch) so RED remain exact; Tempo/Cloud stores
+  # ~1/200 of traces to keep long-running node cost bounded.
+  tail_sampling:
+    decision_wait: 10s
+    num_traces: 50000
+    policies:
+      - name: half-percent
+        type: probabilistic
+        probabilistic:
+          sampling_percentage: 0.5
+  resource/logs:
+    attributes:
+      - key: service.name
+        value: xrpld
+        action: upsert
+      - key: job
+        value: xrpld
+        action: upsert
+  # Deployment-tier tagging. Each collector serves ONE environment and ONE
+  # network, so it stamps both onto every signal it forwards to Grafana
+  # Cloud, letting one cloud stack hold data from many collectors and
+  # filter by tier.
+  #   - deployment.environment: the collector IS the environment (local, ci,
+  #     test, prod), so it is authoritative -> upsert (overwrite).
+  #   - xrpl.network.type: the xrpld node knows its own chain and already
+  #     stamps this, so the collector only fills it when absent -> insert.
+  #     This keeps a node's real network (e.g. a local node on mainnet)
+  #     from being overwritten by a collector's default.
+  # Replace the placeholder values per collector; see the "Deployment
+  # Tiers" section of docs/telemetry-runbook.md.
+  resource/tier:
+    attributes:
+      - key: deployment.environment
+        value: local
+        action: upsert
+      - key: xrpl.network.type
+        value: mainnet
+        action: insert
+  # Strip SDK-injected resource attributes (telemetry.sdk.language/name/version).
+  # The OpenTelemetry SDK auto-adds these to every Resource; they carry no
+  # operational value, so drop them for every signal on every backend.
+  resource/stripsdk:
+    attributes:
+      - key: telemetry.sdk.language
+        action: delete
+      - key: telemetry.sdk.name
+        action: delete
+      - key: telemetry.sdk.version
+        action: delete
+  # Grafana Cloud ingests metrics via OTLP (no Prometheus scrape), so the
+  # tier/instance resource attributes never become series labels the way the
+  # local prometheus exporter promotes them (resource_to_telemetry_conversion
+  # is Prometheus-exporter-only). Copy them onto datapoint labels so the
+  # dashboards' $node / $deployment_environment / $xrpl_network_type filters
+  # resolve on Cloud exactly as they do locally.
+  #
+  # NOTE: service.name is intentionally NOT copied here. Cloud's OTLP ingest
+  # already promotes service.name to the `service_name` label, and adding a
+  # datapoint attribute with the same name causes Mimir to concatenate both
+  # values as `xrpld;xrpld`. service.instance.id does not exhibit the same
+  # doubling, so the copy for $node is kept.
+  transform/cloudlabels:
+    metric_statements:
+      - context: datapoint
+        statements:
+          - set(attributes["service_instance_id"], resource.attributes["service.instance.id"])
+          - set(attributes["deployment_environment"], resource.attributes["deployment.environment"])
+          - set(attributes["xrpl_network_type"], resource.attributes["xrpl.network.type"])
+
+connectors:
+  spanmetrics:
+    namespace: "span"
+    # Resource attributes that define the resource-metrics grouping key.
+    # All resource attributes propagate to the output metrics regardless;
+    # label promotion is handled by resource_to_telemetry_conversion on the
+    # prometheus exporter. Listing the per-node id and tier keys keeps
+    # series from distinct nodes/tiers grouped separately.
+    resource_metrics_key_attributes:
+      - service.instance.id
+      - deployment.environment
+      - xrpl.network.type
+    histogram:
+      explicit:
+        buckets: [1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 5s]
+    dimensions:
+      - name: command
+      - name: rpc_status
+      - name: consensus_mode
+      - name: close_time_correct
+      - name: consensus_state
+      - name: local
+      - name: suppressed
+      - name: proposal_trusted
+      - name: validation_trusted
+      - name: tx_type
+      - name: ter_result
+      - name: stage
+      - name: txq_status
+      - name: load_type
+      - name: is_batch
+      - name: mode_new
+      - name: consensus_stalled
+      - name: consensus_phase
+      - name: consensus_result
+      - name: method
+      - name: grpc_role
+      - name: grpc_status
+      - name: outcome
+      - name: acquire_reason
+
+exporters:
+  otlp/tempo:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+  otlphttp/loki:
+    endpoint: http://loki:3100/otlp
+  prometheus:
+    endpoint: 0.0.0.0:8889
+    # Promote resource attributes (deployment.environment, xrpl.network.type,
+    # service.name, service.instance.id) to Prometheus labels so the local
+    # scrape surface can be filtered by tier. (Grafana Cloud ingests the
+    # attributes natively via OTLP, so this only affects the local export.)
+    resource_to_telemetry_conversion:
+      enabled: true
+  # Single OTLP/HTTP exporter to Grafana Cloud. The gateway fans the three
+  # signals out to hosted Tempo (traces), Mimir/Prometheus (metrics), and
+  # Loki (logs). Retry + queue guard against transient gateway errors.
+  otlphttp/grafanacloud:
+    endpoint: ${env:GRAFANA_CLOUD_OTLP_ENDPOINT}
+    auth:
+      authenticator: basicauth/grafanacloud
+    retry_on_failure:
+      enabled: true
+    sending_queue:
+      enabled: true
+
+service:
+  extensions: [health_check, basicauth/grafanacloud]
+  pipelines:
+    # Each pipeline keeps its local exporter(s) AND adds Grafana Cloud.
+    # For cloud-only, drop debug/otlp/tempo, prometheus, and otlphttp/loki
+    # from the respective exporter lists.
+    # 100% of spans feed the spanmetrics connector so span-derived RED
+    # metrics stay exact. No tail sampling on this branch.
+    traces/metrics:
+      receivers: [otlp]
+      processors: [resource/tier, resource/stripsdk, batch]
+      exporters: [spanmetrics]
+    # Trace-STORAGE branch: 0.5% probabilistic tail sampling before Tempo
+    # and Grafana Cloud, so stored trace volume is ~1/200 of ingested spans.
+    traces/store:
+      receivers: [otlp]
+      processors: [tail_sampling, resource/tier, resource/stripsdk, batch]
+      exporters: [otlp/tempo, otlphttp/grafanacloud]
+    # The local Prometheus scrape promotes tier/instance resource attrs to
+    # labels via resource_to_telemetry_conversion; Grafana Cloud (OTLP) does
+    # not, so it runs a separate pipeline that copies them onto datapoint
+    # labels via transform/cloudlabels. Splitting avoids the local scrape and
+    # the transform both writing a service_instance_id label on the same series.
+    metrics/local:
+      receivers: [otlp, spanmetrics]
+      processors: [resource/tier, resource/stripsdk, batch]
+      exporters: [prometheus]
+    metrics/cloud:
+      receivers: [otlp, spanmetrics]
+      processors:
+        [resource/tier, resource/stripsdk, transform/cloudlabels, batch]
+      exporters: [otlphttp/grafanacloud]
+    logs:
+      receivers: [filelog]
+      processors: [resource/logs, resource/tier, resource/stripsdk, batch]
+      exporters: [otlphttp/loki, otlphttp/grafanacloud]

docker/telemetry/otel-collector-config.yaml

@@ -1,9 +1,28 @@
 # OpenTelemetry Collector configuration for xrpld development.
 #
-# Pipeline: OTLP receiver -> batch processor -> debug + Tempo.
+# Pipelines:
+#   traces: OTLP receiver -> batch processor -> debug + Tempo + spanmetrics
+#   metrics: OTLP receiver + spanmetrics connector -> Prometheus exporter
+#   logs:   filelog receiver -> batch processor -> otlphttp/Loki (Phase 8)
+#
 # xrpld sends traces via OTLP/HTTP to port 4318. The collector batches
-# them and forwards to Tempo via OTLP/gRPC on the Docker network. Tempo
-# is queryable via Grafana Explore using TraceQL.
+# them, forwards to Tempo, and derives RED metrics via the spanmetrics
+# connector, which Prometheus scrapes on port 8889.
+#
+# xrpld sends beast::insight metrics natively via OTLP/HTTP to port 4318
+# (same endpoint as traces). The OTLP receiver feeds both the traces and
+# metrics pipelines. Metrics are exported to Prometheus alongside
+# span-derived metrics.
+#
+# The filelog receiver tails xrpld's debug.log files under
+# /var/log/xrpld/ (mounted from the host). A regex_parser operator
+# extracts timestamp, partition, severity, and optional trace_id/span_id
+# fields injected by Logs::format(). Parsed logs are exported to Grafana
+# Loki for log-trace correlation.
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
 
 receivers:
   otlp:
@@ -12,11 +31,43 @@ receivers:
         endpoint: 0.0.0.0:4317
       http:
         endpoint: 0.0.0.0:4318
+  # Filelog receiver tails xrpld debug.log files for log-trace
+  # correlation. Extracts structured fields (timestamp, partition, severity,
+  # trace_id, span_id, message) via regex. The trace_id and span_id are
+  # optional — only present when the log was emitted within an active span.
+  filelog:
+    include: [/var/log/xrpld/*/debug.log]
+    operators:
+      # Log format emitted by Logs::format() is:
+      #   YYYY-Mmm-DD HH:MM:SS.ffffff UTC <partition>:<severity> [trace_id=... span_id=...] <message>
+      # The `partition:` prefix is omitted when partition is empty, so the
+      # capture group is non-capturing optional. Fractional seconds up to 6
+      # digits are parsed via the `%f` strptime directive.
+      - type: regex_parser
+        regex: '^(?P<timestamp>\S+\s+\S+)\s+\S+\s+(?:(?P<partition>\S+):)?(?P<severity>\S+)\s+(?:trace_id=(?P<trace_id>[a-f0-9]+)\s+span_id=(?P<span_id>[a-f0-9]+)\s+)?(?P<message>.*)$'
+        timestamp:
+          parse_from: attributes.timestamp
+          layout: "%Y-%b-%d %H:%M:%S.%f"
+          location: UTC
 
 processors:
   batch:
     timeout: 1s
     send_batch_size: 100
+  resource/logs:
+    attributes:
+      - key: service.name
+        value: xrpld
+        action: upsert
+      # Loki 3.x OTLP ingestion converts `service.name` to the label
+      # `service_name`. The runbook and integration-test queries use the
+      # canonical Loki label `job` so operators can paste `{job="xrpld"}`
+      # without guessing the otel-to-loki naming convention. Upsert the
+      # `job` resource attribute here so it round-trips through OTLP
+      # into Loki as the `job` label.
+      - key: job
+        value: xrpld
+        action: upsert
   # Deployment-tier tagging. Each collector serves ONE environment and ONE
   # network, so it stamps both onto every signal it forwards. This lets a
   # single Grafana stack hold data from many collectors and filter by tier.
@@ -58,6 +109,53 @@ processors:
       - key: pathfind_dest_account
         action: hash
 
+connectors:
+  spanmetrics:
+    namespace: "span"
+    # Resource attributes that define the resource-metrics grouping key.
+    # All resource attributes are propagated onto the output metrics
+    # regardless; promotion to Prometheus labels is handled by the
+    # prometheus exporter's resource_to_telemetry_conversion. Listing the
+    # per-node id and the deployment-tier keys keeps series from distinct
+    # nodes/tiers grouped separately.
+    resource_metrics_key_attributes:
+      - service.instance.id
+      - deployment.environment
+      - xrpl.network.type
+    histogram:
+      explicit:
+        buckets: [1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 5s]
+    dimensions:
+      - name: command
+      - name: rpc_status
+      - name: consensus_mode
+      - name: close_time_correct
+      - name: consensus_state
+      - name: local
+      - name: suppressed
+      - name: proposal_trusted
+      - name: validation_trusted
+      - name: tx_type
+      - name: ter_result
+      # Apply-pipeline stage (preflight|preclaim|apply) — splits the
+      # tx.preflight/tx.preclaim/tx.transactor span RED metrics per stage.
+      - name: stage
+      - name: txq_status
+      - name: load_type
+      - name: is_batch
+      # Consensus lifecycle dimensions (low cardinality, bounded value sets).
+      - name: mode_new
+      - name: consensus_stalled
+      - name: consensus_phase
+      - name: consensus_result
+      # gRPC surface dimensions (bounded: method names, role, status).
+      - name: method
+      - name: grpc_role
+      - name: grpc_status
+      # ledger.acquire dimensions (bounded: outcome, acquire reason).
+      - name: outcome
+      - name: acquire_reason
+
 exporters:
   debug:
     verbosity: detailed
@@ -65,10 +163,19 @@ exporters:
     endpoint: tempo:4317
     tls:
       insecure: true
-
-extensions:
-  health_check:
-    endpoint: 0.0.0.0:13133
+  # Export logs to Grafana Loki via OTLP/HTTP. Loki 3.x supports
+  # native OTLP ingestion on its /otlp endpoint, replacing the removed
+  # loki exporter (dropped in otel-collector-contrib v0.147.0).
+  otlphttp/loki:
+    endpoint: http://loki:3100/otlp
+  prometheus:
+    endpoint: 0.0.0.0:8889
+    # Promote resource attributes (deployment.environment, xrpl.network.type,
+    # service.name, service.instance.id) to Prometheus labels. Without this
+    # the tier attributes stamped by resource/tier stay on the resource and
+    # never reach the metric series, so dashboards could not filter by them.
+    resource_to_telemetry_conversion:
+      enabled: true
 
 service:
   extensions: [health_check]
@@ -76,4 +183,14 @@ service:
     traces:
       receivers: [otlp]
       processors: [resource/tier, resource/stripsdk, attributes/hash, batch]
-      exporters: [debug, otlp/tempo]
+      exporters: [debug, otlp/tempo, spanmetrics]
+    metrics:
+      receivers: [otlp, spanmetrics]
+      processors: [resource/tier, resource/stripsdk, batch]
+      exporters: [prometheus]
+    # Log pipeline ingests xrpld debug.log via filelog receiver,
+    # batches entries, and exports to Loki for log-trace correlation.
+    logs:
+      receivers: [filelog]
+      processors: [resource/logs, resource/tier, resource/stripsdk, batch]
+      exporters: [otlphttp/loki]

docker/telemetry/prometheus.yml

@@ -0,0 +1,9 @@
+# Prometheus configuration for scraping spanmetrics from OTel Collector.
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: otel-collector
+    static_configs:
+      - targets: ["otel-collector:8889"]

docker/telemetry/tempo.yaml

@@ -17,6 +17,14 @@ stream_over_http_enabled: true
 server:
   http_listen_port: 3200
 
+# Raise the TraceQL metrics query range limit. The default
+# query_frontend.metrics.max_duration is 3h, so a dashboard set to a longer
+# window (e.g. 6h/12h) fails with "range exceeds 3h0m0s". 168h matches the
+# search max_duration and gives dashboards generous headroom.
+query_frontend:
+  metrics:
+    max_duration: 168h
+
 distributor:
   receivers:
     otlp:
@@ -31,9 +39,11 @@ compactor:
   compaction:
     block_retention: 1h
 
-# Enable metrics generator for service graph and span metrics.
-# Produces RED metrics (rate, errors, duration) per service/span,
-# feeding Grafana's service map visualization.
+# Enable metrics generator for service graph, span metrics, and the
+# local-blocks processor. Produces RED metrics (rate, errors, duration) per
+# service/span for the service map, and keeps recent trace blocks queryable so
+# TraceQL metrics queries (quantile_over_time, count_over_time, etc. via
+# /api/metrics/query_range) work.
 metrics_generator:
   registry:
     external_labels:
@@ -44,6 +54,18 @@ metrics_generator:
     # to enable remote_write for service graph metrics:
     # remote_write:
     #   - url: http://prometheus:9090/api/v1/write
+  # Separate WAL the local-blocks processor flushes traces to for metrics
+  # queries. Required when flush_to_storage is true.
+  traces_storage:
+    path: /var/tempo/generator/traces
+  processor:
+    local_blocks:
+      # xrpld consensus/transaction spans are SPAN_KIND_INTERNAL. By default
+      # local-blocks keeps only server spans for TraceQL metrics, so attribute
+      # aggregations over internal spans return nothing. Keep all spans.
+      filter_server_spans: false
+      # Flush recent blocks to traces_storage so query_range can read them.
+      flush_to_storage: true
 
 overrides:
   defaults:
@@ -51,6 +73,7 @@ overrides:
       processors:
         - service-graphs
         - span-metrics
+        - local-blocks
 
 storage:
   trace:

docker/telemetry/validators-devnet.txt

@@ -0,0 +1,11 @@
+# Devnet validator list configuration.
+#
+# Uses the Devnet validator list publisher operated by Ripple.
+# This fetches the current set of trusted validators for the
+# XRP Ledger Devnet automatically.
+
+[validator_list_sites]
+https://vl.devnet.rippletest.net
+
+[validator_list_keys]
+EDBB54B0D9AEE071BB37784AF5A9E7CC49AC7A0EFCE868C54532BCB966B9CFC13B

docker/telemetry/validators-mainnet.txt

@@ -0,0 +1,15 @@
+# Mainnet validator list configuration.
+#
+# Uses the production XRPL validator list publishers operated by
+# Ripple and the XRPL Foundation. These fetch the current set of
+# trusted validators for XRPL Mainnet automatically.
+
+[validator_list_sites]
+https://vl.ripple.com
+https://unl.xrplf.org
+
+[validator_list_keys]
+# vl.ripple.com
+ED2677ABFFD1B33AC6FBC3062B71F1E8397C1505E1C42C64D11AD1B28FF73F4734
+# unl.xrplf.org
+ED42AEC58B701EEBB77356FFFEC26F83C1F0407263530F068C7C73D392C7E06FD1

docker/telemetry/workload/README.md

@@ -0,0 +1,371 @@
+# Telemetry Workload Tools
+
+Synthetic workload generation and validation tools for xrpld's OpenTelemetry telemetry stack. These tools validate that all spans, metrics, dashboards, and log-trace correlation work end-to-end under controlled load.
+
+## Quick Start
+
+```bash
+# Build xrpld with telemetry enabled
+conan install . --build=missing -o telemetry=True
+cmake --preset default -Dtelemetry=ON
+cmake --build --preset default
+
+# Run full validation (starts everything, runs load, validates)
+docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld
+
+# Cleanup when done
+docker/telemetry/workload/run-full-validation.sh --cleanup
+```
+
+## Architecture
+
+The validation suite runs a multi-node xrpld cluster as local processes alongside
+a Docker Compose telemetry stack. The cluster exercises consensus, peer-to-peer
+spans (proposals, validations), and all metric pipelines.
+
+```
+run-full-validation.sh (shell orchestrator)
+  |
+  |-- docker-compose.workload.yaml
+  |     |-- otel-collector (traces via OTLP + StatsD receiver)
+  |     |-- tempo (trace backend + TraceQL search API)
+  |     |-- prometheus (metrics scraping)
+  |     |-- grafana (dashboards, provisioned automatically)
+  |
+  |-- generate-validator-keys.sh
+  |     -> validator-keys.json, validators.txt
+  |
+  |-- Nx xrpld nodes (local processes, full telemetry)
+  |     - Each node: [telemetry] enabled=1, trace_rpc/consensus/transactions
+  |     - [signing_support] true (server-side signing for tx_submitter)
+  |     - Peer discovery via [ips] (not [ips_fixed]) for active peer counts
+  |
+  |-- workload_orchestrator.py (phased load execution)
+  |     |-- rpc_load_generator.py (WebSocket RPC traffic)
+  |     |-- tx_submitter.py (transaction diversity)
+  |     -> workload-report.json + per-phase reports
+  |
+  |-- validate_telemetry.py (pass/fail checks)
+  |     -> validation-report.json
+  |
+  |-- benchmark.sh (baseline vs telemetry comparison)
+        -> benchmark-report-*.md
+```
+
+## Workload Profiles
+
+The workload orchestrator (`workload_orchestrator.py`) reads named profiles
+from `workload-profiles.json` and executes sequential load phases. Within
+each phase, the RPC generator and TX submitter run concurrently.
+
+### Available Profiles
+
+| Profile           | Phases | Duration                     | Purpose                                                     |
+| ----------------- | ------ | ---------------------------- | ----------------------------------------------------------- |
+| `full-validation` | 6      | ~5 min + 1 min propagation   | Full 18-dashboard coverage with burst/idle/plateau patterns |
+| `quick-smoke`     | 1      | ~30s + 30s propagation       | Fast CI smoke test                                          |
+| `stress`          | 3      | ~3.5 min + 1 min propagation | Heavy sustained load for benchmarking                       |
+
+### full-validation Phases
+
+| Phase        | RPC Rate | TX TPS | Duration | Dashboard Coverage                              |
+| ------------ | -------- | ------ | -------- | ----------------------------------------------- |
+| warmup       | 5 RPS    | —      | 30s      | Node Health, Validator Health (baseline gauges) |
+| steady-state | 30 RPS   | 3 TPS  | 60s      | All dashboards (plateau data)                   |
+| rpc-burst    | 100 RPS  | —      | 30s      | Job Queue, RPC Performance (latency spikes)     |
+| tx-flood     | 5 RPS    | 20 TPS | 30s      | Fee Market & TxQ, Transaction Overview          |
+| mixed-peak   | 50 RPS   | 10 TPS | 60s      | Consensus Health, Ledger Operations             |
+| cooldown     | 5 RPS    | —      | 30s      | Recovery patterns, state transitions            |
+
+### Custom Profiles
+
+Add profiles to `workload-profiles.json`:
+
+```json
+{
+  "profiles": {
+    "my-custom": {
+      "description": "Custom profile for specific testing",
+      "phases": [
+        {
+          "name": "phase-name",
+          "description": "What this phase exercises",
+          "duration_sec": 60,
+          "rpc": { "rate": 50, "weights": { "server_info": 80, "fee": 20 } },
+          "tx": { "tps": 5, "weights": { "Payment": 100 } }
+        }
+      ],
+      "propagation_wait_sec": 30
+    }
+  }
+}
+```
+
+Set `"rpc"` or `"tx"` to `null` to skip that generator for a phase.
+Custom `"weights"` override the default command/transaction distribution.
+
+## Tools Reference
+
+### run-full-validation.sh
+
+Orchestrates the complete validation pipeline. Starts the telemetry stack, starts a multi-node xrpld cluster, generates load, and validates the results.
+
+```bash
+# Full validation with defaults (uses full-validation profile)
+./run-full-validation.sh --xrpld /path/to/xrpld
+
+# Quick smoke test
+./run-full-validation.sh --xrpld /path/to/xrpld --profile quick-smoke
+
+# Stress test with benchmarks
+./run-full-validation.sh --xrpld /path/to/xrpld --profile stress --with-benchmark
+
+# Skip Loki checks (if Phase 8 not deployed)
+./run-full-validation.sh --xrpld /path/to/xrpld --skip-loki
+```
+
+### workload_orchestrator.py
+
+Reads a named profile from `workload-profiles.json` and executes sequential
+load phases. Within each phase, `rpc_load_generator.py` and `tx_submitter.py`
+run as concurrent subprocesses. Produces per-phase reports and a combined
+summary.
+
+```bash
+# Run with a specific profile
+python3 workload_orchestrator.py --profile full-validation
+
+# Multiple endpoints
+python3 workload_orchestrator.py --profile full-validation \
+    --endpoints ws://localhost:6006 ws://localhost:6007
+
+# Save combined report
+python3 workload_orchestrator.py --profile stress --report /tmp/report.json
+```
+
+### rpc_load_generator.py
+
+Generates RPC traffic matching realistic production distribution. Uses
+xrpld's **native WebSocket command format** (`{"command": ...}`) with flat
+parameters — the same format as `tx_submitter.py`.
+
+- 40% health checks (server_info, fee)
+- 30% wallet queries (account_info, account_lines, account_objects)
+- 15% explorer queries (ledger, ledger_data)
+- 10% transaction lookups (tx, account_tx)
+- 5% DEX queries (book_offers, amm_info)
+
+```bash
+# Basic usage
+python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120
+
+# Multiple endpoints (round-robin)
+python3 rpc_load_generator.py \
+    --endpoints ws://localhost:6006 ws://localhost:6007 \
+    --rate 100 --duration 300
+
+# Custom weights
+python3 rpc_load_generator.py --endpoints ws://localhost:6006 \
+    --weights '{"server_info": 80, "account_info": 20}'
+```
+
+### tx_submitter.py
+
+Submits diverse transaction types to exercise the full span and metric surface.
+Uses xrpld's **native WebSocket command format** (`{"command": ...}`) rather
+than JSON-RPC format. The response payload is inside the `"result"` key, with
+`"status"` at the top level.
+
+Supported transaction types:
+
+- Payment (XRP transfers) — exercises `tx.process`, `tx.receive`, `tx.apply`
+- OfferCreate / OfferCancel (DEX activity)
+- TrustSet (trust line creation)
+- NFTokenMint / NFTokenCreateOffer (NFT activity)
+- EscrowCreate / EscrowFinish (escrow lifecycle)
+- AMMCreate / AMMDeposit (AMM pool operations)
+
+Requires `[signing_support] true` in the node config for server-side signing.
+
+```bash
+# Basic usage
+python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120
+
+# Custom mix
+python3 tx_submitter.py --endpoint ws://localhost:6006 \
+    --weights '{"Payment": 60, "OfferCreate": 20, "TrustSet": 20}'
+```
+
+### validate_telemetry.py
+
+Automated validation that all expected telemetry data exists. Every metric and span is required — if it doesn't fire, the validation fails.
+
+- **Span validation**: All span types from `expected_spans.json` with required attributes and parent-child hierarchies
+- **Metric validation**: All metrics from `expected_metrics.json` — SpanMetrics, StatsD gauges/counters/histograms, Phase 9 OTLP metrics. Every listed metric must have > 0 series. Uses the Prometheus `/api/v1/series` endpoint (not instant queries) to avoid false negatives from stale gauges.
+- **Log-trace correlation**: trace_id/span_id in Loki logs (requires Loki)
+- **Dashboard validation**: All 10 Grafana dashboards load with panels
+
+```bash
+# Run all validations
+python3 validate_telemetry.py --report /tmp/report.json
+
+# Skip Loki checks
+python3 validate_telemetry.py --skip-loki --report /tmp/report.json
+```
+
+### OTel Timings Regression Gate
+
+`capture_timings.py` + `compare_to_baseline.py` implement a regression gate
+that compares OTel-derived per-span/per-RPC/per-job timings against a
+committed baseline. Unlike `benchmark.sh` (which measures the overhead of
+enabling telemetry on the current binary), this gate catches **xrpld
+performance regressions over time** by diffing against a stored baseline
+from a prior run.
+
+How it runs inside the validation pipeline:
+
+1. `run-full-validation.sh` executes the normal workload and validation suite.
+2. After validation, `capture_timings.py` queries Prometheus for every
+   metric in `regression-metrics.json` and writes `reports/timings.json`.
+3. `compare_to_baseline.py` reads `timings.json`,
+   `baselines/baseline-timings.json`, and `regression-thresholds.json`,
+   then either:
+   - Prints the paste-me JSON block (when the baseline is a placeholder
+     or empty) and exits 0.
+   - Prints a delta table, writes `reports/regression-report.json`, and
+     exits non-zero if any metric breached both the percentage AND
+     absolute bound.
+
+Bootstrapping a baseline:
+
+1. Push the branch. The `Telemetry Validation` CI run prints the full
+   timings JSON under "Paste into `baselines/baseline-timings.json`" in
+   the workflow Step Summary.
+2. Open a PR copying that JSON block verbatim into
+   `baselines/baseline-timings.json`. Reviewer approval is the audit gate.
+3. Subsequent runs compare against it; the gate fails on regression.
+
+Per-run tuning:
+
+- `--skip-regression` disables the gate (local exploration only).
+- `REGRESSION_WINDOW` env var overrides the default Prometheus `rate()`
+  window (`3m`). Keep close to the workload duration.
+- Metric surface lives in `regression-metrics.json`; thresholds in
+  `regression-thresholds.json`; both are reviewed changes.
+
+See [`baselines/README.md`](./baselines/README.md) for the baseline
+lifecycle and refresh process.
+
+### benchmark.sh
+
+Compares baseline (no telemetry) vs telemetry-enabled performance:
+
+```bash
+./benchmark.sh --xrpld /path/to/xrpld --duration 300
+```
+
+Thresholds (configurable via environment):
+
+| Metric            | Threshold | Env Variable                |
+| ----------------- | --------- | --------------------------- |
+| CPU overhead      | < 3%      | BENCH_CPU_OVERHEAD_PCT      |
+| Memory overhead   | < 5MB     | BENCH_MEM_OVERHEAD_MB       |
+| RPC p99 latency   | < 2ms     | BENCH_RPC_LATENCY_IMPACT_MS |
+| Throughput impact | < 5%      | BENCH_TPS_IMPACT_PCT        |
+| Consensus impact  | < 1%      | BENCH_CONSENSUS_IMPACT_PCT  |
+
+## Reading Validation Reports
+
+The validation report (`validation-report.json`) is structured as:
+
+```json
+{
+  "summary": {
+    "total": 45,
+    "passed": 42,
+    "failed": 3,
+    "all_passed": false
+  },
+  "checks": [
+    {
+      "name": "span.rpc.ws_message",
+      "category": "span",
+      "passed": true,
+      "message": "rpc.ws_message: 15 traces found",
+      "details": { "trace_count": 15 }
+    }
+  ]
+}
+```
+
+Categories:
+
+- **span**: Span type existence and attribute validation
+- **metric**: Prometheus metric existence
+- **log**: Log-trace correlation checks
+- **dashboard**: Grafana dashboard accessibility
+
+## CI Integration
+
+The validation runs as a GitHub Actions workflow (`.github/workflows/telemetry-validation.yml`):
+
+- Triggered manually or on pushes to telemetry branches
+- Builds xrpld, starts the full stack, runs load, validates
+- Uploads reports as artifacts
+- Posts summary to PR
+
+## Configuration Files
+
+| File                              | Purpose                                                       |
+| --------------------------------- | ------------------------------------------------------------- |
+| `workload-profiles.json`          | Named load profiles with phase definitions                    |
+| `expected_spans.json`             | Span inventory (names, attributes, hierarchies, config flags) |
+| `expected_metrics.json`           | Metric inventory — every listed metric must be present        |
+| `test_accounts.json`              | Test account roles (keys generated at runtime)                |
+| `regression-metrics.json`         | Metric surface for the OTel regression gate                   |
+| `regression-thresholds.json`      | Per-metric regression bounds (pct AND abs)                    |
+| `baselines/baseline-timings.json` | Committed baseline — populated from first CI run              |
+| `requirements.txt`                | Python dependencies                                           |
+
+### expected_metrics.json Format
+
+```json
+{
+  "category_name": {
+    "description": "Human-readable description.",
+    "metrics": ["metric_1", "metric_2"]
+  }
+}
+```
+
+Every metric listed must produce > 0 Prometheus series during the validation run. If a metric doesn't fire, the workload generators need to produce enough load to trigger it.
+
+### expected_spans.json Format
+
+Each span entry defines its name, category, parent (for hierarchy validation),
+required attributes, and the `config_flag` that must be enabled:
+
+```json
+{
+  "name": "rpc.command.*",
+  "category": "rpc",
+  "parent": "rpc.process",
+  "required_attributes": ["command", "version", "rpc_role", "rpc_status"],
+  "config_flag": "trace_rpc"
+}
+```
+
+## Node Configuration Notes
+
+The orchestrator (`run-full-validation.sh`) generates node configs with:
+
+- `[telemetry] enabled=1` with all trace categories (`trace_rpc`, `trace_consensus`, `trace_transactions`)
+- `[signing_support] true` — required for `tx_submitter.py` to submit signed transactions via WebSocket
+- `[ips]` (not `[ips_fixed]`) — ensures peer connections are counted in `Peer_Finder_Active_Inbound/Outbound_Peers` metrics (fixed peers are excluded from these counters by design)
+
+## StatsD Gauge Behaviour
+
+Beast::insight StatsD gauges only emit when their value _changes_ from the previous sample. This can cause two problems in the validation environment:
+
+1. **Initial-zero gauges** — if a gauge value is 0 from startup and never changes, the gauge would never emit. To address this, `StatsDGaugeImpl` initializes `m_dirty = true`, ensuring the first flush always emits the initial value.
+2. **Stale gauges** — once a gauge stabilizes (e.g., peer count stays at 1), it stops emitting new data points. Prometheus marks it stale after ~5 minutes. The validation script uses the Prometheus `/api/v1/series` endpoint instead of instant queries to catch such gauges.

docker/telemetry/workload/baselines/README.md

@@ -0,0 +1,72 @@
+# Performance Baselines
+
+This directory holds the committed baseline file used by the OTel-driven regression gate.
+
+## How the gate works
+
+After the validation suite runs, `capture_timings.py` queries Prometheus for the timings
+declared in [`../regression-metrics.json`](../regression-metrics.json) and writes a
+`timings.json`. Then `compare_to_baseline.py` reads [`baseline-timings.json`](./baseline-timings.json),
+[`../regression-thresholds.json`](../regression-thresholds.json), and the captured
+`timings.json`. The comparator picks one of two modes automatically:
+
+- **Placeholder baseline** (`"placeholder": true` or empty `metrics`): the comparator
+  prints the captured timings JSON in exactly the format expected for this file, then
+  exits 0 without gating. This is how we bootstrap the baseline.
+- **Populated baseline**: the comparator diffs per-metric, enforces the thresholds
+  (regression = current exceeds baseline on BOTH the percentage AND absolute bound),
+  and exits non-zero on any regression.
+
+The regression gate runs against whatever workload profile `run-full-validation.sh`
+was invoked with. Capture and comparison are profile-agnostic — they only read
+Prometheus — so all existing profiles (`full-validation`, `quick-smoke`, `stress`)
+continue to work unchanged.
+
+## Bootstrapping the baseline
+
+1. Merge a CI run with a `"placeholder": true` baseline. The telemetry-validation
+   workflow runs, fails no gate, and prints the captured timings block to the workflow
+   Step Summary under the heading `### Paste into baselines/baseline-timings.json`.
+2. Open a new PR. Copy the full JSON block from the Step Summary (or download the
+   `timings.json` artifact) into this file, replacing the placeholder contents. The
+   JSON is emitted in the exact byte-for-byte format this file expects — sorted keys,
+   2-space indent, trailing newline.
+3. The committed baseline PR needs reviewer approval just like any other code change.
+   This is the primary audit point for "who moved the performance bar."
+
+## Refreshing the baseline
+
+Refresh when a legitimate performance change lands on `develop` (for example, a
+deliberate rewrite that changes a span's structure). The process is identical to
+bootstrapping: run CI with the current baseline, inspect the delta, and if the
+new numbers should become the norm, open a PR pasting the fresh timings into
+`baseline-timings.json`. The reviewer decides whether the new baseline is acceptable.
+
+Do **not** edit `baseline-timings.json` by hand outside of this process — every entry
+should trace back to a real CI run so variance characteristics are preserved.
+
+## Schema
+
+```json
+{
+  "schema_version": 1,
+  "captured_at": "2026-04-24T17:30:00Z",
+  "window": "3m",
+  "git_sha": "<SHA of the commit that produced these numbers>",
+  "profile": "<workload profile used>",
+  "metrics": {
+    "span.tx.process.p99": { "value": 12.4, "unit": "ms" },
+    "rpc.server_info.p95": { "value": 850.0, "unit": "us" },
+    "job.transaction.queued.p95": { "value": 1500.0, "unit": "us" }
+  }
+}
+```
+
+Placeholder baselines additionally include `"placeholder": true`. The comparator
+detects this field (or an empty `metrics` object) to switch into "populate" mode
+instead of enforcing thresholds. Remove the `placeholder` key when pasting real
+captured timings.
+
+Missing metrics (value `null`) in a captured run do not count as regressions — they
+are reported separately in `regression-report.json` under `missing_in_current`.
+This keeps the gate robust when a profile doesn't exercise every span on every run.

docker/telemetry/workload/baselines/baseline-timings.json

@@ -0,0 +1,133 @@
+{
+  "captured_at": "2026-06-05T18:41:52Z",
+  "git_sha": "fd1c8c6060f7a15cc9e65b16f99629d9ab7ac7dc",
+  "metrics": {
+    "job.acceptLedger.queued.p95": {
+      "unit": "us",
+      "value": 96.78571428571428
+    },
+    "job.acceptLedger.running.p95": {
+      "unit": "us",
+      "value": 10562.499999999945
+    },
+    "job.transaction.queued.p95": {
+      "unit": "us",
+      "value": 478.96551724137925
+    },
+    "job.transaction.running.p95": {
+      "unit": "us",
+      "value": 494.1361256544502
+    },
+    "span.consensus.accept.p50": {
+      "unit": "ms",
+      "value": 1.059405940594059
+    },
+    "span.consensus.accept.p95": {
+      "unit": "ms",
+      "value": 9.749999999999996
+    },
+    "span.consensus.accept.p99": {
+      "unit": "ms",
+      "value": 23.704545454545432
+    },
+    "span.consensus.ledger_close.p50": {
+      "unit": "ms",
+      "value": 0.5284697508896797
+    },
+    "span.consensus.ledger_close.p95": {
+      "unit": "ms",
+      "value": 1.511111111111103
+    },
+    "span.consensus.ledger_close.p99": {
+      "unit": "ms",
+      "value": 7.878571428571429
+    },
+    "span.ledger.build.p50": {
+      "unit": "ms",
+      "value": 0.7412060301507538
+    },
+    "span.ledger.build.p95": {
+      "unit": "ms",
+      "value": 4.611111111111112
+    },
+    "span.ledger.build.p99": {
+      "unit": "ms",
+      "value": 7.541666666666674
+    },
+    "span.ledger.store.p50": {
+      "unit": "ms",
+      "value": 0.5
+    },
+    "span.ledger.store.p95": {
+      "unit": "ms",
+      "value": 0.95
+    },
+    "span.ledger.store.p99": {
+      "unit": "ms",
+      "value": 0.9900000000000001
+    },
+    "span.ledger.validate.p50": {
+      "unit": "ms",
+      "value": 0.5283687943262412
+    },
+    "span.ledger.validate.p95": {
+      "unit": "ms",
+      "value": 1.3666666666666627
+    },
+    "span.ledger.validate.p99": {
+      "unit": "ms",
+      "value": 6.699999999999978
+    },
+    "span.rpc.process.p50": {
+      "unit": "ms",
+      "value": null
+    },
+    "span.rpc.process.p95": {
+      "unit": "ms",
+      "value": null
+    },
+    "span.rpc.process.p99": {
+      "unit": "ms",
+      "value": null
+    },
+    "span.rpc.ws_message.p50": {
+      "unit": "ms",
+      "value": 0.5026522773001647
+    },
+    "span.rpc.ws_message.p95": {
+      "unit": "ms",
+      "value": 0.9550393268703128
+    },
+    "span.rpc.ws_message.p99": {
+      "unit": "ms",
+      "value": 0.9952515090543261
+    },
+    "span.tx.apply.p50": {
+      "unit": "ms",
+      "value": 0.6330472103004292
+    },
+    "span.tx.apply.p95": {
+      "unit": "ms",
+      "value": 4.203389830508474
+    },
+    "span.tx.apply.p99": {
+      "unit": "ms",
+      "value": 5.083333333333319
+    },
+    "span.tx.process.p50": {
+      "unit": "ms",
+      "value": 0.5042801992591597
+    },
+    "span.tx.process.p95": {
+      "unit": "ms",
+      "value": 0.9581323781882418
+    },
+    "span.tx.process.p99": {
+      "unit": "ms",
+      "value": 0.998474791584883
+    }
+  },
+  "profile": "full-validation",
+  "schema_version": 1,
+  "window": "3m"
+}

docker/telemetry/workload/benchmark.sh

@@ -0,0 +1,396 @@
+#!/usr/bin/env bash
+# benchmark.sh — Performance benchmark for rippled telemetry overhead.
+#
+# Runs two identical workloads against a rippled cluster:
+#   1. Baseline: telemetry disabled ([telemetry] enabled=0)
+#   2. Telemetry: full telemetry enabled (traces + StatsD + all categories)
+#
+# Compares CPU, memory, RPC latency, TPS, and consensus round time.
+# Outputs a Markdown table with pass/fail against configured thresholds.
+#
+# Usage:
+#   ./benchmark.sh --xrpld /path/to/xrpld --duration 300
+#
+# Thresholds (configurable via environment variables):
+#   BENCH_CPU_OVERHEAD_PCT=3       CPU overhead < 3%
+#   BENCH_MEM_OVERHEAD_MB=5        Memory overhead < 5MB
+#   BENCH_RPC_LATENCY_IMPACT_MS=2  RPC p99 latency impact < 2ms
+#   BENCH_TPS_IMPACT_PCT=5         Throughput impact < 5%
+#   BENCH_CONSENSUS_IMPACT_PCT=1   Consensus round time impact < 1%
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Colored output helpers
+# ---------------------------------------------------------------------------
+log() { printf "\033[1;34m[BENCH]\033[0m  %s\n" "$*"; }
+ok() { printf "\033[1;32m[BENCH]\033[0m  %s\n" "$*"; }
+warn() { printf "\033[1;33m[BENCH]\033[0m  %s\n" "$*"; }
+fail() { printf "\033[1;31m[BENCH]\033[0m  %s\n" "$*"; }
+die() {
+    printf "\033[1;31m[BENCH]\033[0m  %s\n" "$*" >&2
+    exit 1
+}
+
+# ---------------------------------------------------------------------------
+# Defaults and thresholds
+# ---------------------------------------------------------------------------
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+
+# Configurable thresholds via environment variables.
+CPU_THRESHOLD="${BENCH_CPU_OVERHEAD_PCT:-3}"
+MEM_THRESHOLD="${BENCH_MEM_OVERHEAD_MB:-5}"
+RPC_THRESHOLD="${BENCH_RPC_LATENCY_IMPACT_MS:-2}"
+TPS_THRESHOLD="${BENCH_TPS_IMPACT_PCT:-5}"
+CONSENSUS_THRESHOLD="${BENCH_CONSENSUS_IMPACT_PCT:-1}"
+
+XRPLD="${BENCH_XRPLD:-$REPO_ROOT/.build/xrpld}"
+DURATION=300
+NUM_NODES=3
+WORKDIR="/tmp/xrpld-benchmark"
+RESULTS_DIR="$SCRIPT_DIR/benchmark-results"
+RPC_PORT_BASE=5020
+PEER_PORT_BASE=51250
+
+# ---------------------------------------------------------------------------
+# Argument parsing
+# ---------------------------------------------------------------------------
+usage() {
+    echo "Usage: $0 [OPTIONS]"
+    echo ""
+    echo "Options:"
+    echo "  --xrpld PATH      Path to xrpld binary (default: \$REPO_ROOT/.build/xrpld)"
+    echo "  --duration SECS   Benchmark duration per run (default: 300)"
+    echo "  --nodes NUM       Number of validator nodes (default: 3)"
+    echo "  --output DIR      Results output directory"
+    echo "  -h, --help        Show this help"
+    exit 0
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --xrpld)
+            XRPLD="$2"
+            shift 2
+            ;;
+        --duration)
+            DURATION="$2"
+            shift 2
+            ;;
+        --nodes)
+            NUM_NODES="$2"
+            shift 2
+            ;;
+        --output)
+            RESULTS_DIR="$2"
+            shift 2
+            ;;
+        -h | --help) usage ;;
+        *) die "Unknown option: $1" ;;
+    esac
+done
+
+# Validate prerequisites.
+[ -x "$XRPLD" ] || die "xrpld not found at $XRPLD"
+command -v jq >/dev/null 2>&1 || die "jq not found"
+command -v bc >/dev/null 2>&1 || die "bc not found"
+command -v curl >/dev/null 2>&1 || die "curl not found"
+
+mkdir -p "$RESULTS_DIR"
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+
+# ---------------------------------------------------------------------------
+# Node cluster management
+# ---------------------------------------------------------------------------
+start_cluster() {
+    local telemetry_enabled="$1"
+    local label="$2"
+
+    log "Starting $NUM_NODES-node cluster ($label, telemetry=$telemetry_enabled)..."
+
+    rm -rf "$WORKDIR"
+    mkdir -p "$WORKDIR"
+
+    # Generate keys using first node.
+    bash "$SCRIPT_DIR/generate-validator-keys.sh" "$XRPLD" "$NUM_NODES" "$WORKDIR"
+
+    # Build per-node configs.
+    for i in $(seq 1 "$NUM_NODES"); do
+        local node_dir="$WORKDIR/node$i"
+        mkdir -p "$node_dir/nudb" "$node_dir/db"
+
+        local rpc_port
+        rpc_port=$((RPC_PORT_BASE + i - 1))
+        local peer_port
+        peer_port=$((PEER_PORT_BASE + i - 1))
+        local seed
+        seed=$(jq -r ".[$((i - 1))].seed" "$WORKDIR/validator-keys.json")
+
+        # Build ips_fixed list.
+        local ips_fixed=""
+        for j in $(seq 1 "$NUM_NODES"); do
+            if [ "$j" -ne "$i" ]; then
+                ips_fixed="${ips_fixed}127.0.0.1 $((PEER_PORT_BASE + j - 1))
+"
+            fi
+        done
+
+        # Build telemetry section.
+        local telemetry_section=""
+        if [ "$telemetry_enabled" = "1" ]; then
+            telemetry_section="
+[telemetry]
+enabled=1
+service_instance_id=bench-node-${i}
+endpoint=http://localhost:4318/v1/traces
+exporter=otlp_http
+batch_size=512
+batch_delay_ms=2000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1
+
+[insight]
+server=otel
+endpoint=http://localhost:4318/v1/metrics
+prefix=xrpld"
+        else
+            telemetry_section="
+[telemetry]
+enabled=0"
+        fi
+
+        cat >"$node_dir/xrpld.cfg" <<EOCFG
+[server]
+port_rpc
+port_peer
+
+[port_rpc]
+port = $rpc_port
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[port_peer]
+port = $peer_port
+ip = 0.0.0.0
+protocol = peer
+
+[node_db]
+type=NuDB
+path=$node_dir/nudb
+online_delete=256
+
+[database_path]
+$node_dir/db
+
+[debug_logfile]
+$node_dir/debug.log
+
+[validation_seed]
+$seed
+
+[validators_file]
+$WORKDIR/validators.txt
+
+[ips_fixed]
+${ips_fixed}
+[peer_private]
+1
+${telemetry_section}
+
+[rpc_startup]
+{ "command": "log_level", "severity": "warning" }
+
+[ssl_verify]
+0
+EOCFG
+
+        "$XRPLD" --conf "$node_dir/xrpld.cfg" --start >"$node_dir/stdout.log" 2>&1 &
+        echo $! >"$node_dir/xrpld.pid"
+    done
+
+    # Wait for consensus.
+    log "Waiting for consensus..."
+    for attempt in $(seq 1 120); do
+        local ready=0
+        for i in $(seq 1 "$NUM_NODES"); do
+            local port
+            port=$((RPC_PORT_BASE + i - 1))
+            local state
+            state=$(curl -sf "http://localhost:$port" \
+                -d '{"method":"server_info"}' 2>/dev/null |
+                jq -r '.result.info.server_state' 2>/dev/null || echo "")
+            if [ "$state" = "proposing" ]; then
+                ready=$((ready + 1))
+            fi
+        done
+        if [ "$ready" -ge "$NUM_NODES" ]; then
+            ok "All $NUM_NODES nodes proposing (attempt $attempt)"
+            break
+        fi
+        if [ "$attempt" -eq 120 ]; then
+            warn "Consensus timeout — $ready/$NUM_NODES nodes ready"
+        fi
+        sleep 1
+    done
+
+    # Let the cluster stabilize.
+    sleep 5
+}
+
+stop_cluster() {
+    log "Stopping cluster..."
+    for i in $(seq 1 "$NUM_NODES"); do
+        local pidfile="$WORKDIR/node$i/xrpld.pid"
+        if [ -f "$pidfile" ]; then
+            kill "$(cat "$pidfile")" 2>/dev/null || true
+        fi
+    done
+    pkill -f "$WORKDIR" 2>/dev/null || true
+    sleep 3
+}
+
+# Build RPC ports CSV string.
+rpc_ports_csv() {
+    local ports=""
+    for i in $(seq 1 "$NUM_NODES"); do
+        [ -n "$ports" ] && ports="$ports,"
+        ports="$ports$((RPC_PORT_BASE + i - 1))"
+    done
+    echo "$ports"
+}
+
+# ---------------------------------------------------------------------------
+# Run benchmark
+# ---------------------------------------------------------------------------
+log "="
+log "  rippled Telemetry Performance Benchmark"
+log "  Nodes: $NUM_NODES | Duration: ${DURATION}s | Binary: $XRPLD"
+log "="
+
+# --- Baseline run ---
+BASELINE_FILE="$RESULTS_DIR/baseline-${TIMESTAMP}.json"
+start_cluster "0" "baseline"
+bash "$SCRIPT_DIR/collect_system_metrics.sh" "$(rpc_ports_csv)" "$DURATION" "$BASELINE_FILE"
+stop_cluster
+
+# --- Telemetry run ---
+TELEMETRY_FILE="$RESULTS_DIR/telemetry-${TIMESTAMP}.json"
+start_cluster "1" "telemetry"
+bash "$SCRIPT_DIR/collect_system_metrics.sh" "$(rpc_ports_csv)" "$DURATION" "$TELEMETRY_FILE"
+stop_cluster
+
+# ---------------------------------------------------------------------------
+# Compare results
+# ---------------------------------------------------------------------------
+log "Comparing results..."
+
+read_metric() {
+    local file="$1"
+    local key="$2"
+    jq -r ".$key // 0" "$file"
+}
+
+BASE_CPU=$(read_metric "$BASELINE_FILE" "cpu_pct_avg")
+TELE_CPU=$(read_metric "$TELEMETRY_FILE" "cpu_pct_avg")
+CPU_DELTA=$(echo "scale=2; $TELE_CPU - $BASE_CPU" | bc 2>/dev/null || echo "0")
+
+BASE_MEM=$(read_metric "$BASELINE_FILE" "memory_rss_mb_peak")
+TELE_MEM=$(read_metric "$TELEMETRY_FILE" "memory_rss_mb_peak")
+MEM_DELTA=$(echo "scale=2; $TELE_MEM - $BASE_MEM" | bc 2>/dev/null || echo "0")
+
+BASE_RPC=$(read_metric "$BASELINE_FILE" "rpc_p99_ms")
+TELE_RPC=$(read_metric "$TELEMETRY_FILE" "rpc_p99_ms")
+RPC_DELTA=$(echo "scale=2; $TELE_RPC - $BASE_RPC" | bc 2>/dev/null || echo "0")
+
+BASE_TPS=$(read_metric "$BASELINE_FILE" "tps")
+TELE_TPS=$(read_metric "$TELEMETRY_FILE" "tps")
+if [[ "$(echo "$BASE_TPS > 0" | bc 2>/dev/null)" = "1" ]]; then
+    TPS_IMPACT=$(echo "scale=2; ($BASE_TPS - $TELE_TPS) / $BASE_TPS * 100" | bc 2>/dev/null || echo "0")
+else
+    TPS_IMPACT="0"
+fi
+
+BASE_CONS=$(read_metric "$BASELINE_FILE" "consensus_round_p95_ms")
+TELE_CONS=$(read_metric "$TELEMETRY_FILE" "consensus_round_p95_ms")
+if [[ "$(echo "$BASE_CONS > 0" | bc 2>/dev/null)" = "1" ]]; then
+    CONS_IMPACT=$(echo "scale=2; ($TELE_CONS - $BASE_CONS) / $BASE_CONS * 100" | bc 2>/dev/null || echo "0")
+else
+    CONS_IMPACT="0"
+fi
+
+# ---------------------------------------------------------------------------
+# Pass/fail checks
+# ---------------------------------------------------------------------------
+PASS_COUNT=0
+FAIL_COUNT=0
+
+check_threshold() {
+    local name="$1"
+    local actual="$2"
+    local threshold="$3"
+    local unit="$4"
+
+    # Compare: actual <= threshold
+    if [[ "$(echo "$actual <= $threshold" | bc 2>/dev/null)" = "1" ]]; then
+        ok "$name: ${actual}${unit} <= ${threshold}${unit} PASS"
+        PASS_COUNT=$((PASS_COUNT + 1))
+        echo "PASS"
+    else
+        fail "$name: ${actual}${unit} > ${threshold}${unit} FAIL"
+        FAIL_COUNT=$((FAIL_COUNT + 1))
+        echo "FAIL"
+    fi
+}
+
+CPU_RESULT=$(check_threshold "CPU overhead" "$CPU_DELTA" "$CPU_THRESHOLD" "%")
+MEM_RESULT=$(check_threshold "Memory overhead" "$MEM_DELTA" "$MEM_THRESHOLD" "MB")
+RPC_RESULT=$(check_threshold "RPC p99 impact" "$RPC_DELTA" "$RPC_THRESHOLD" "ms")
+TPS_RESULT=$(check_threshold "TPS impact" "$TPS_IMPACT" "$TPS_THRESHOLD" "%")
+CONS_RESULT=$(check_threshold "Consensus impact" "$CONS_IMPACT" "$CONSENSUS_THRESHOLD" "%")
+
+# ---------------------------------------------------------------------------
+# Output Markdown table
+# ---------------------------------------------------------------------------
+REPORT_FILE="$RESULTS_DIR/benchmark-report-${TIMESTAMP}.md"
+
+cat >"$REPORT_FILE" <<EOMD
+# Telemetry Performance Benchmark Report
+
+**Date**: $(date -u +"%Y-%m-%d %H:%M:%S UTC")
+**Nodes**: $NUM_NODES | **Duration**: ${DURATION}s per run
+**Binary**: $XRPLD
+
+## Results
+
+| Metric | Baseline | Telemetry | Delta | Threshold | Result |
+|--------|----------|-----------|-------|-----------|--------|
+| CPU (avg %) | ${BASE_CPU}% | ${TELE_CPU}% | ${CPU_DELTA}% | < ${CPU_THRESHOLD}% | ${CPU_RESULT} |
+| Memory RSS (peak MB) | ${BASE_MEM} MB | ${TELE_MEM} MB | ${MEM_DELTA} MB | < ${MEM_THRESHOLD} MB | ${MEM_RESULT} |
+| RPC p99 Latency (ms) | ${BASE_RPC} ms | ${TELE_RPC} ms | ${RPC_DELTA} ms | < ${RPC_THRESHOLD} ms | ${RPC_RESULT} |
+| Throughput (TPS) | ${BASE_TPS} | ${TELE_TPS} | ${TPS_IMPACT}% | < ${TPS_THRESHOLD}% | ${TPS_RESULT} |
+| Consensus Round p95 (ms) | ${BASE_CONS} ms | ${TELE_CONS} ms | ${CONS_IMPACT}% | < ${CONSENSUS_THRESHOLD}% | ${CONS_RESULT} |
+
+## Summary
+
+- **Passed**: $PASS_COUNT / $((PASS_COUNT + FAIL_COUNT))
+- **Failed**: $FAIL_COUNT / $((PASS_COUNT + FAIL_COUNT))
+
+## Raw Data
+
+- Baseline: \`$(basename "$BASELINE_FILE")\`
+- Telemetry: \`$(basename "$TELEMETRY_FILE")\`
+EOMD
+
+ok "Benchmark report written to $REPORT_FILE"
+cat "$REPORT_FILE"
+
+# Exit with failure if any check failed.
+if [ "$FAIL_COUNT" -gt 0 ]; then
+    exit 1
+fi

docker/telemetry/workload/capture_timings.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+"""Capture OTel-derived timings from Prometheus for the regression gate.
+
+Queries Prometheus for every metric declared in ``regression-metrics.json``
+and writes the results to a JSON file in the exact schema
+``baseline-timings.json`` expects. When a user wants to refresh the
+baseline, they copy a CI run's ``timings.json`` artifact (or the block
+printed to the workflow step summary) into
+``baselines/baseline-timings.json`` in a reviewable PR.
+
+Output schema (stable — ``compare_to_baseline.py`` reads it verbatim)::
+
+    {
+        "schema_version": 1,
+        "captured_at": "2026-04-24T17:30:00Z",
+        "window": "3m",
+        "git_sha": "<from $GITHUB_SHA or `git rev-parse HEAD`>",
+        "profile": "regression",
+        "metrics": {
+            "span.tx.process.p99": {"value": 12.4, "unit": "ms"},
+            "rpc.server_info.p95": {"value": 850.0, "unit": "us"},
+            ...
+        }
+    }
+
+Usage::
+
+    python3 capture_timings.py \\
+        --prometheus http://localhost:9090 \\
+        --metrics regression-metrics.json \\
+        --output /tmp/timings.json \\
+        --window 3m \\
+        --profile regression
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import os
+import subprocess
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import aiohttp
+
+from prom_queries import build_query_plan, run_query_plan
+
+logger = logging.getLogger("capture_timings")
+
+SCHEMA_VERSION = 1
+
+
+async def capture(
+    prom_url: str,
+    metrics_path: Path,
+    window: str,
+    profile: str,
+) -> dict:
+    """Build and execute the query plan, return the full report dict."""
+    plan = build_query_plan(metrics_path, window=window)
+    logger.info("Capturing %d metrics from %s (window=%s)", len(plan), prom_url, window)
+
+    async with aiohttp.ClientSession() as session:
+        metrics = await run_query_plan(session, prom_url, plan)
+
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "captured_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "window": window,
+        "git_sha": _detect_git_sha(),
+        "profile": profile,
+        "metrics": dict(sorted(metrics.items())),
+    }
+
+
+def _detect_git_sha() -> str:
+    """Return the current commit SHA from env or git, else ``"unknown"``.
+
+    Prefers ``GITHUB_SHA`` (set in Actions), falls back to ``git rev-parse``.
+    Silent fallback is fine here — a missing SHA only affects the captured
+    metadata, not the comparison logic.
+    """
+    env_sha = os.environ.get("GITHUB_SHA")
+    if env_sha:
+        return env_sha
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+            check=False,
+        )
+        if result.returncode == 0:
+            return result.stdout.strip()
+    except (OSError, subprocess.SubprocessError):
+        pass
+    return "unknown"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--prometheus",
+        default="http://localhost:9090",
+        help="Prometheus base URL (default: http://localhost:9090)",
+    )
+    parser.add_argument(
+        "--metrics",
+        type=Path,
+        default=Path(__file__).parent / "regression-metrics.json",
+        help="Path to regression-metrics.json",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        required=True,
+        help="Where to write the captured timings JSON",
+    )
+    parser.add_argument(
+        "--window",
+        default="3m",
+        help="Prometheus rate() window (default: 3m)",
+    )
+    parser.add_argument(
+        "--profile",
+        default="regression",
+        help="Workload profile used during capture (metadata only)",
+    )
+    parser.add_argument(
+        "--min-capture-ratio",
+        type=float,
+        default=0.5,
+        help="Fail if fewer than this fraction of metrics are captured (default: 0.5)",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable debug logging",
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(levelname)s %(name)s: %(message)s",
+    )
+
+    report = asyncio.run(
+        capture(
+            prom_url=args.prometheus,
+            metrics_path=args.metrics,
+            window=args.window,
+            profile=args.profile,
+        )
+    )
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with open(args.output, "w") as f:
+        json.dump(report, f, indent=2, sort_keys=True)
+        f.write("\n")
+
+    captured = sum(1 for v in report["metrics"].values() if v["value"] is not None)
+    total = len(report["metrics"])
+    logger.info("Wrote %s (%d/%d metrics captured)", args.output, captured, total)
+
+    if total > 0 and (captured / total) < args.min_capture_ratio:
+        logger.error(
+            "Only %d/%d (%.0f%%) metrics captured — below the %.0f%% minimum. "
+            "Is Prometheus reachable at %s?",
+            captured,
+            total,
+            captured / total * 100,
+            args.min_capture_ratio * 100,
+            args.prometheus,
+        )
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docker/telemetry/workload/collect_system_metrics.sh

@@ -0,0 +1,236 @@
+#!/usr/bin/env bash
+# collect_system_metrics.sh — Collect CPU, memory, and RPC latency metrics
+# from running xrpld nodes for benchmark comparison.
+#
+# Samples system metrics at regular intervals and writes a JSON summary.
+# Used by benchmark.sh for baseline vs telemetry comparison.
+#
+# Usage:
+#   ./collect_system_metrics.sh <rpc_ports_csv> <duration_seconds> <output_file>
+#
+# Example:
+#   ./collect_system_metrics.sh "5005,5006,5007" 300 /tmp/metrics-baseline.json
+#
+# Output JSON format:
+#   {
+#     "cpu_pct_avg": 12.5,
+#     "memory_rss_mb_peak": 450.2,
+#     "rpc_p99_ms": 15.3,
+#     "tps": 4.8,
+#     "consensus_round_p95_ms": 3200,
+#     "samples": 60
+#   }
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Colored output helpers
+# ---------------------------------------------------------------------------
+log() { printf "\033[1;34m[METRICS]\033[0m %s\n" "$*"; }
+ok() { printf "\033[1;32m[METRICS]\033[0m %s\n" "$*"; }
+die() {
+    printf "\033[1;31m[METRICS]\033[0m %s\n" "$*" >&2
+    exit 1
+}
+
+# ---------------------------------------------------------------------------
+# Argument parsing
+# ---------------------------------------------------------------------------
+usage() {
+    echo "Usage: $0 <rpc_ports_csv> <duration_seconds> <output_file>"
+    echo ""
+    echo "Arguments:"
+    echo "  rpc_ports_csv     Comma-separated RPC ports (e.g., 5005,5006,5007)"
+    echo "  duration_seconds  How long to collect metrics"
+    echo "  output_file       Path to write JSON results"
+    exit 1
+}
+
+if [ $# -lt 3 ]; then
+    usage
+fi
+
+RPC_PORTS_CSV="$1"
+DURATION="$2"
+OUTPUT_FILE="$3"
+
+IFS=',' read -ra RPC_PORTS <<<"$RPC_PORTS_CSV"
+SAMPLE_INTERVAL=5
+SAMPLES=$((DURATION / SAMPLE_INTERVAL))
+
+log "Collecting metrics for ${DURATION}s (${SAMPLES} samples, ${#RPC_PORTS[@]} nodes)..."
+
+# ---------------------------------------------------------------------------
+# Temporary files for aggregation
+# ---------------------------------------------------------------------------
+TMPDIR_METRICS="$(mktemp -d)"
+CPU_FILE="$TMPDIR_METRICS/cpu.txt"
+MEM_FILE="$TMPDIR_METRICS/mem.txt"
+RPC_FILE="$TMPDIR_METRICS/rpc.txt"
+LEDGER_FILE="$TMPDIR_METRICS/ledger.txt"
+
+touch "$CPU_FILE" "$MEM_FILE" "$RPC_FILE" "$LEDGER_FILE"
+
+cleanup() {
+    rm -rf "$TMPDIR_METRICS"
+}
+trap cleanup EXIT
+
+# ---------------------------------------------------------------------------
+# Get initial ledger sequence for TPS calculation
+# ---------------------------------------------------------------------------
+INITIAL_SEQ=0
+INITIAL_TIME=$(date +%s)
+for port in "${RPC_PORTS[@]}"; do
+    seq=$(curl -sf "http://localhost:$port" \
+        -d '{"method":"server_info"}' 2>/dev/null |
+        jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
+    if [ "$seq" -gt "$INITIAL_SEQ" ]; then
+        INITIAL_SEQ=$seq
+    fi
+done
+log "Initial validated ledger seq: $INITIAL_SEQ"
+
+# ---------------------------------------------------------------------------
+# Sampling loop
+# ---------------------------------------------------------------------------
+for sample in $(seq 1 "$SAMPLES"); do
+    # Collect CPU usage for xrpld processes.
+    # Uses ps to find all xrpld processes and average their CPU%.
+    cpu_sum=0
+    cpu_count=0
+    while IFS= read -r line; do
+        cpu_val=$(echo "$line" | awk '{print $1}')
+        if [ -n "$cpu_val" ] && [ "$cpu_val" != "0.0" ]; then
+            cpu_sum=$(echo "$cpu_sum + $cpu_val" | bc 2>/dev/null || echo "$cpu_sum")
+            cpu_count=$((cpu_count + 1))
+        fi
+    done < <(ps aux 2>/dev/null | grep '[x]rpld' | awk '{print $3}')
+
+    if [ "$cpu_count" -gt 0 ]; then
+        cpu_avg=$(echo "scale=2; $cpu_sum / $cpu_count" | bc 2>/dev/null || echo "0")
+        echo "$cpu_avg" >>"$CPU_FILE"
+    fi
+
+    # Collect memory RSS for xrpld processes.
+    while IFS= read -r line; do
+        rss_kb=$(echo "$line" | awk '{print $1}')
+        if [ -n "$rss_kb" ] && [ "$rss_kb" != "0" ]; then
+            rss_mb=$(echo "scale=2; $rss_kb / 1024" | bc 2>/dev/null || echo "0")
+            echo "$rss_mb" >>"$MEM_FILE"
+        fi
+    done < <(ps aux 2>/dev/null | grep '[x]rpld' | awk '{print $6}')
+
+    # Collect RPC latency from each node.
+    for port in "${RPC_PORTS[@]}"; do
+        start_ms=$(date +%s%N)
+        curl -sf "http://localhost:$port" \
+            -d '{"method":"server_info"}' >/dev/null 2>&1 || true
+        end_ms=$(date +%s%N)
+        latency_ms=$(((end_ms - start_ms) / 1000000))
+        echo "$latency_ms" >>"$RPC_FILE"
+    done
+
+    # Record current validated ledger seq.
+    for port in "${RPC_PORTS[@]}"; do
+        seq=$(curl -sf "http://localhost:$port" \
+            -d '{"method":"server_info"}' 2>/dev/null |
+            jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
+        echo "$seq" >>"$LEDGER_FILE"
+        break # Only need one node's seq per sample.
+    done
+
+    # Progress indicator.
+    if [ $((sample % 10)) -eq 0 ]; then
+        log "  Sample $sample/$SAMPLES..."
+    fi
+
+    sleep "$SAMPLE_INTERVAL"
+done
+
+# ---------------------------------------------------------------------------
+# Compute aggregated metrics
+# ---------------------------------------------------------------------------
+log "Computing aggregated metrics..."
+
+# CPU average.
+if [ -s "$CPU_FILE" ]; then
+    CPU_AVG=$(awk '{ sum += $1; n++ } END { if (n>0) printf "%.2f", sum/n; else print "0" }' "$CPU_FILE")
+else
+    CPU_AVG="0"
+fi
+
+# Memory peak RSS (MB).
+if [ -s "$MEM_FILE" ]; then
+    MEM_PEAK=$(sort -n "$MEM_FILE" | tail -1)
+else
+    MEM_PEAK="0"
+fi
+
+# RPC latency p99 (ms).
+if [ -s "$RPC_FILE" ]; then
+    RPC_COUNT=$(wc -l <"$RPC_FILE")
+    P99_INDEX=$(echo "scale=0; $RPC_COUNT * 99 / 100" | bc)
+    RPC_P99=$(sort -n "$RPC_FILE" | sed -n "${P99_INDEX}p")
+    [ -z "$RPC_P99" ] && RPC_P99="0"
+else
+    RPC_P99="0"
+fi
+
+# TPS calculation from ledger sequence advancement.
+FINAL_SEQ=0
+for port in "${RPC_PORTS[@]}"; do
+    seq=$(curl -sf "http://localhost:$port" \
+        -d '{"method":"server_info"}' 2>/dev/null |
+        jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
+    if [ "$seq" -gt "$FINAL_SEQ" ]; then
+        FINAL_SEQ=$seq
+    fi
+done
+FINAL_TIME=$(date +%s)
+ELAPSED=$((FINAL_TIME - INITIAL_TIME))
+LEDGER_ADVANCE=$((FINAL_SEQ - INITIAL_SEQ))
+if [ "$ELAPSED" -gt 0 ] && [ "$LEDGER_ADVANCE" -gt 0 ]; then
+    # Rough TPS: assume ~avg_txs_per_ledger * ledgers / elapsed.
+    # Without tx count, use ledger close rate as proxy.
+    TPS=$(echo "scale=2; $LEDGER_ADVANCE / $ELAPSED" | bc 2>/dev/null || echo "0")
+else
+    TPS="0"
+fi
+
+# Consensus round time p95 (from ledger close interval).
+# Approximate by looking at ledger sequence progression intervals.
+if [ -s "$LEDGER_FILE" ]; then
+    # Calculate intervals between consecutive ledger sequences.
+    LEDGER_COUNT=$(wc -l <"$LEDGER_FILE")
+    # Rough estimate: DURATION / number_of_distinct_ledgers * 1000 ms
+    UNIQUE_LEDGERS=$(sort -u "$LEDGER_FILE" | wc -l)
+    if [ "$UNIQUE_LEDGERS" -gt 1 ]; then
+        CONSENSUS_P95=$(echo "scale=0; $DURATION * 1000 / ($UNIQUE_LEDGERS - 1)" | bc 2>/dev/null || echo "0")
+    else
+        CONSENSUS_P95="0"
+    fi
+else
+    CONSENSUS_P95="0"
+fi
+
+# ---------------------------------------------------------------------------
+# Write output JSON
+# ---------------------------------------------------------------------------
+cat >"$OUTPUT_FILE" <<EOF_JSON
+{
+  "cpu_pct_avg": $CPU_AVG,
+  "memory_rss_mb_peak": $MEM_PEAK,
+  "rpc_p99_ms": $RPC_P99,
+  "tps": $TPS,
+  "consensus_round_p95_ms": $CONSENSUS_P95,
+  "samples": $SAMPLES,
+  "duration_seconds": $DURATION,
+  "node_count": ${#RPC_PORTS[@]},
+  "initial_ledger_seq": $INITIAL_SEQ,
+  "final_ledger_seq": $FINAL_SEQ
+}
+EOF_JSON
+
+ok "Metrics written to $OUTPUT_FILE"
+cat "$OUTPUT_FILE"

docker/telemetry/workload/compare_to_baseline.py

@@ -0,0 +1,401 @@
+#!/usr/bin/env python3
+"""Compare captured OTel timings against a committed baseline.
+
+Operating modes (chosen automatically based on the baseline file contents):
+
+1. **No baseline** — if ``baseline-timings.json`` has an empty
+   ``metrics`` object (or is marked with ``"placeholder": true``), this
+   script is in "populate" mode. It prints the captured timings JSON in
+   the exact format expected for pasting into
+   ``baselines/baseline-timings.json``, then exits 0. No regression check.
+
+2. **Populated baseline** — per-metric percentage AND absolute deltas are
+   computed against thresholds from ``regression-thresholds.json``. A
+   regression occurs when BOTH bounds are breached for the same quantile.
+   Prints a human-readable table and writes a full JSON report.
+   Exits 1 if any regression was detected, else 0.
+
+Inputs:
+    --timings     Captured timings JSON (from capture_timings.py)
+    --baseline    Committed baseline JSON
+    --thresholds  Threshold policy JSON
+    --report      Where to write regression-report.json (optional)
+
+Exit codes:
+    0 — No baseline (paste-me emitted), OR baseline populated and no regression
+    1 — Regression detected (at least one metric breached both bounds)
+    2 — Internal error (e.g. bad JSON, baseline/current key mismatch)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+from dataclasses import dataclass, asdict
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("compare_to_baseline")
+
+
+@dataclass
+class MetricDelta:
+    """Single metric's baseline-vs-current comparison outcome.
+
+    Attributes:
+        key:                Flat metric key (e.g. span.tx.process.p99).
+        baseline:           Baseline value (may be None if unpopulated).
+        current:            Current run value (may be None if not captured).
+        delta:              current - baseline (None if either side None).
+        pct_change:         100 * delta / baseline (None if baseline ≤ 0).
+        unit:               Unit from baseline (preserved as-is).
+        threshold_pct:      Resolved per-metric pct threshold.
+        threshold_abs:      Resolved per-metric absolute threshold.
+        regressed:          True iff both bounds breached.
+        note:               Human-readable classification when not regressed.
+    """
+
+    key: str
+    baseline: float | None
+    current: float | None
+    delta: float | None
+    pct_change: float | None
+    unit: str
+    threshold_pct: float | None
+    threshold_abs: float | None
+    regressed: bool
+    note: str
+
+
+def load_json(path: Path) -> dict:
+    with open(path) as f:
+        return json.load(f)
+
+
+def is_placeholder(baseline: dict) -> bool:
+    """A baseline is a placeholder if explicitly marked OR metrics are empty."""
+    if baseline.get("placeholder") is True:
+        return True
+    return not baseline.get("metrics")
+
+
+def print_paste_me(timings: dict) -> None:
+    """Print captured timings in the exact baseline-timings.json format.
+
+    The output between the two banner lines is the file contents to paste,
+    byte-for-byte — sorted keys, 2-space indent, trailing newline.
+    """
+    banner = "=" * 72
+    print(banner, file=sys.stderr)
+    print(
+        "  NO BASELINE FOUND — paste the JSON below into",
+        file=sys.stderr,
+    )
+    print(
+        "  docker/telemetry/workload/baselines/baseline-timings.json",
+        file=sys.stderr,
+    )
+    print(banner, file=sys.stderr)
+
+    print(json.dumps(timings, indent=2, sort_keys=True))
+
+    print(banner, file=sys.stderr)
+    print(
+        "  (End of paste-me JSON. Gate did NOT run — baseline is empty.)",
+        file=sys.stderr,
+    )
+    print(banner, file=sys.stderr)
+
+
+def resolve_thresholds(
+    key: str,
+    thresholds: dict,
+) -> tuple[float | None, float | None]:
+    """Return ``(pct_threshold, abs_threshold)`` for a metric key.
+
+    Per-metric overrides win over defaults. Returns ``(None, None)`` if no
+    threshold is defined for this category/quantile — such metrics are
+    captured but never gate the build.
+    """
+    parts = key.split(".")
+    if len(parts) < 3:
+        return (None, None)
+    category_key = parts[0]
+    quantile_key = parts[-1]
+
+    category_map = {
+        "span": "span",
+        "rpc": "rpc_method",
+        "job": "job_queue",
+    }
+    cat = category_map.get(category_key)
+    if cat is None:
+        return (None, None)
+
+    override_key = f"{category_key}.{'.'.join(parts[1:-1])}"
+    overrides = thresholds.get("overrides", {})
+    defaults = thresholds.get("defaults", {}).get(cat, {})
+
+    rule = overrides.get(override_key, {}).get(quantile_key)
+    if rule is None:
+        rule = defaults.get(quantile_key)
+    if rule is None:
+        return (None, None)
+
+    pct = rule.get("max_pct_increase")
+    abs_bound = rule.get("max_abs_increase_ms")
+    if abs_bound is None:
+        abs_bound = rule.get("max_abs_increase_us")
+    return (pct, abs_bound)
+
+
+def _skip_delta(
+    key: str,
+    baseline: float | None,
+    current: float | None,
+    unit: str,
+    thresholds: dict,
+    note: str,
+) -> MetricDelta:
+    """Build a MetricDelta for cases where comparison is not possible."""
+    pct_threshold, abs_threshold = resolve_thresholds(key, thresholds)
+    return MetricDelta(
+        key=key,
+        baseline=baseline,
+        current=current,
+        delta=None,
+        pct_change=None,
+        unit=unit,
+        threshold_pct=pct_threshold,
+        threshold_abs=abs_threshold,
+        regressed=False,
+        note=note,
+    )
+
+
+def compute_delta(
+    key: str,
+    baseline_entry: dict | None,
+    current_entry: dict | None,
+    thresholds: dict,
+) -> MetricDelta:
+    """Compute a MetricDelta for one metric key.
+
+    A regression requires BOTH bounds to be breached simultaneously. This
+    tolerates small-value noise: a 100% increase on a 0.5 ms metric
+    (to 1.0 ms) is not a regression under a 5 ms absolute bound.
+    """
+    baseline = baseline_entry.get("value") if baseline_entry else None
+    current = current_entry.get("value") if current_entry else None
+    unit = (baseline_entry or current_entry or {}).get("unit", "")
+
+    if baseline is None and current is None:
+        return _skip_delta(
+            key, None, None, unit, thresholds, "no data (neither baseline nor current)"
+        )
+
+    if baseline is None:
+        return _skip_delta(
+            key, None, current, unit, thresholds, "new metric (not in baseline)"
+        )
+
+    if current is None:
+        return _skip_delta(
+            key, baseline, None, unit, thresholds, "not captured in current run"
+        )
+
+    pct_threshold, abs_threshold = resolve_thresholds(key, thresholds)
+    delta = current - baseline
+    pct_change = (delta / baseline * 100.0) if baseline > 0 else None
+
+    if pct_threshold is None or abs_threshold is None:
+        return MetricDelta(
+            key=key,
+            baseline=baseline,
+            current=current,
+            delta=delta,
+            pct_change=pct_change,
+            unit=unit,
+            threshold_pct=pct_threshold,
+            threshold_abs=abs_threshold,
+            regressed=False,
+            note="no threshold configured",
+        )
+
+    pct_breach = pct_change is not None and pct_change > pct_threshold
+    abs_breach = delta > abs_threshold
+    regressed = pct_breach and abs_breach
+
+    if regressed:
+        note = "REGRESSION"
+    elif delta < 0:
+        note = "improved"
+    else:
+        note = "within bounds"
+
+    return MetricDelta(
+        key=key,
+        baseline=baseline,
+        current=current,
+        delta=delta,
+        pct_change=pct_change,
+        unit=unit,
+        threshold_pct=pct_threshold,
+        threshold_abs=abs_threshold,
+        regressed=regressed,
+        note=note,
+    )
+
+
+def print_summary(deltas: list[MetricDelta]) -> None:
+    """Print a sorted, human-readable table of per-metric results."""
+    regressions = [d for d in deltas if d.regressed]
+    improvements = [
+        d
+        for d in deltas
+        if d.delta is not None and d.delta < 0 and d.baseline not in (None, 0)
+    ]
+    improvements.sort(key=lambda d: d.pct_change or 0)
+    regressions.sort(key=lambda d: -(d.pct_change or 0))
+
+    print("=" * 72)
+    print(f"  Regression check: {len(regressions)} regression(s) detected")
+    print("=" * 72)
+
+    if regressions:
+        print("\nRegressions (breached BOTH pct AND absolute bounds):")
+        _print_table(regressions)
+
+    if improvements:
+        top = improvements[:5]
+        print("\nTop improvements:")
+        _print_table(top)
+
+    missing = [d for d in deltas if d.note == "not captured in current run"]
+    if missing:
+        print(f"\n{len(missing)} baseline metric(s) not captured in current run:")
+        for d in missing:
+            print(f"  {d.key}")
+
+
+def _print_table(rows: list[MetricDelta]) -> None:
+    """Print a fixed-width table for a list of deltas."""
+    header = f"  {'METRIC':<45} {'BASE':>10} {'CUR':>10} {'Δ':>10} {'%':>8} UNIT"
+    print(header)
+    print("  " + "-" * (len(header) - 2))
+    for d in rows:
+        base = f"{d.baseline:.2f}" if d.baseline is not None else "-"
+        cur = f"{d.current:.2f}" if d.current is not None else "-"
+        delta = f"{d.delta:+.2f}" if d.delta is not None else "-"
+        pct = f"{d.pct_change:+.1f}%" if d.pct_change is not None else "-"
+        print(f"  {d.key:<45} {base:>10} {cur:>10} {delta:>10} {pct:>8} {d.unit}")
+
+
+def write_report(
+    deltas: list[MetricDelta],
+    report_path: Path,
+    baseline: dict,
+    timings: dict,
+) -> None:
+    """Write regression-report.json — machine-readable artifact for CI."""
+    regressions = [d for d in deltas if d.regressed]
+    payload = {
+        "schema_version": 1,
+        "baseline_captured_at": baseline.get("captured_at"),
+        "baseline_git_sha": baseline.get("git_sha"),
+        "current_captured_at": timings.get("captured_at"),
+        "current_git_sha": timings.get("git_sha"),
+        "window": timings.get("window"),
+        "profile": timings.get("profile"),
+        "summary": {
+            "total": len(deltas),
+            "regressions": len(regressions),
+            "improvements": sum(
+                1
+                for d in deltas
+                if d.delta is not None and d.delta < 0 and d.baseline not in (None, 0)
+            ),
+            "missing_in_current": sum(
+                1 for d in deltas if d.note == "not captured in current run"
+            ),
+        },
+        "metrics": [asdict(d) for d in deltas],
+    }
+    report_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(report_path, "w") as f:
+        json.dump(payload, f, indent=2, sort_keys=True)
+        f.write("\n")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--timings",
+        type=Path,
+        required=True,
+        help="Captured timings JSON (from capture_timings.py)",
+    )
+    parser.add_argument(
+        "--baseline",
+        type=Path,
+        required=True,
+        help="Committed baseline-timings.json",
+    )
+    parser.add_argument(
+        "--thresholds",
+        type=Path,
+        default=Path(__file__).parent / "regression-thresholds.json",
+        help="Threshold policy JSON",
+    )
+    parser.add_argument(
+        "--report",
+        type=Path,
+        default=None,
+        help="Where to write regression-report.json (optional)",
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(levelname)s %(name)s: %(message)s",
+    )
+
+    try:
+        timings = load_json(args.timings)
+        baseline = load_json(args.baseline)
+        thresholds = load_json(args.thresholds)
+    except (OSError, json.JSONDecodeError) as exc:
+        logger.error("failed to load inputs: %s", exc)
+        return 2
+
+    if is_placeholder(baseline):
+        print_paste_me(timings)
+        return 0
+
+    baseline_metrics = baseline.get("metrics", {})
+    current_metrics = timings.get("metrics", {})
+
+    all_keys = sorted(set(baseline_metrics) | set(current_metrics))
+    deltas = [
+        compute_delta(
+            key,
+            baseline_metrics.get(key),
+            current_metrics.get(key),
+            thresholds,
+        )
+        for key in all_keys
+    ]
+
+    print_summary(deltas)
+
+    if args.report:
+        write_report(deltas, args.report, baseline, timings)
+        logger.info("wrote %s", args.report)
+
+    return 1 if any(d.regressed for d in deltas) else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docker/telemetry/workload/expected_metrics.json

@@ -0,0 +1,145 @@
+{
+  "description": "Expected metric inventory for xrpld telemetry validation. Metric names have no prefix (the xrpld_ prefix was removed). beast::insight metrics are lowercased by formatName. Sourced from the live Grafana dashboards and MetricsRegistry.cpp.",
+  "spanmetrics": {
+    "description": "SpanMetrics-derived RED metrics from the OTel Collector spanmetrics connector.",
+    "metrics": [
+      "span_calls_total",
+      "span_duration_milliseconds_bucket",
+      "span_duration_milliseconds_count",
+      "span_duration_milliseconds_sum"
+    ],
+    "required_labels": [
+      "span_name",
+      "status_code",
+      "service_name",
+      "span_kind"
+    ],
+    "dimension_labels": [
+      "command",
+      "rpc_status",
+      "consensus_mode",
+      "local",
+      "proposal_trusted",
+      "validation_trusted",
+      "tx_type",
+      "ter_result",
+      "stage",
+      "txq_status",
+      "close_time_correct",
+      "consensus_state",
+      "suppressed"
+    ],
+    "_dimension_labels_note": "Bare label names as configured in otel-collector-config.yaml spanmetrics dimensions. Informational only (not asserted by the validator)."
+  },
+  "statsd_gauges": {
+    "description": "beast::insight gauges exported via OTLP/HTTP to the collector (server=otel).",
+    "metrics": [
+      "ledgermaster_validated_ledger_age",
+      "ledgermaster_published_ledger_age",
+      "state_accounting_full_duration",
+      "peer_finder_active_inbound_peers",
+      "peer_finder_active_outbound_peers",
+      "jobq_job_count"
+    ]
+  },
+  "statsd_counters": {
+    "description": "beast::insight counters exported via OTLP/HTTP. The OTel Prometheus exporter appends _total to monotonic counters.",
+    "metrics": ["rpc_requests_total", "ledger_fetches_total"]
+  },
+  "overlay_traffic": {
+    "description": "Overlay traffic metrics (subset — full list has 45+ categories).",
+    "metrics": [
+      "total_bytes_in",
+      "total_bytes_out",
+      "total_messages_in",
+      "total_messages_out"
+    ]
+  },
+  "phase9_nodestore": {
+    "description": "Phase 9 NodeStore I/O observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label distinguishing sub-metrics.",
+    "metrics": ["nodestore_state"]
+  },
+  "phase9_cache": {
+    "description": "Phase 9 cache hit rate observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
+    "metrics": ["cache_metrics"]
+  },
+  "phase9_txq": {
+    "description": "Phase 9 transaction queue observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
+    "metrics": ["txq_metrics"]
+  },
+  "phase9_rpc_method": {
+    "description": "Phase 9 per-RPC-method counters (MetricsRegistry via OTLP).",
+    "metrics": ["rpc_method_started_total"]
+  },
+  "phase9_objects": {
+    "description": "Phase 9 counted object instances observable gauge (MetricsRegistry via OTLP).",
+    "metrics": ["object_count"]
+  },
+  "phase9_load": {
+    "description": "Phase 9 fee escalation and load factor observable gauge (MetricsRegistry via OTLP).",
+    "metrics": ["load_factor_metrics"]
+  },
+  "parity_validation_agreement": {
+    "description": "External dashboard parity: validation agreement percentages (MetricsRegistry).",
+    "metrics": [
+      "validation_agreement{metric=\"agreement_pct_1h\"}",
+      "validation_agreement{metric=\"agreement_pct_24h\"}"
+    ]
+  },
+  "parity_validator_health": {
+    "description": "External dashboard parity: validator health indicators (MetricsRegistry).",
+    "metrics": [
+      "validator_health{metric=\"amendment_blocked\"}",
+      "validator_health{metric=\"unl_expiry_days\"}"
+    ]
+  },
+  "parity_peer_quality": {
+    "description": "External dashboard parity: peer quality metrics (MetricsRegistry).",
+    "metrics": [
+      "peer_quality{metric=\"peer_latency_p90_ms\"}",
+      "peer_quality{metric=\"peers_insane_count\"}"
+    ]
+  },
+  "parity_ledger_economy": {
+    "description": "External dashboard parity: ledger economy metrics (MetricsRegistry).",
+    "metrics": [
+      "ledger_economy{metric=\"base_fee_xrp\"}",
+      "ledger_economy{metric=\"transaction_rate\"}"
+    ]
+  },
+  "parity_state_tracking": {
+    "description": "External dashboard parity: server state tracking (MetricsRegistry).",
+    "metrics": ["state_tracking{metric=\"state_value\"}"]
+  },
+  "parity_counters": {
+    "description": "External dashboard parity: monotonic counters (MetricsRegistry).",
+    "metrics": [
+      "ledgers_closed_total",
+      "validations_sent_total",
+      "state_changes_total"
+    ]
+  },
+  "parity_storage": {
+    "description": "External dashboard parity: storage detail metrics (MetricsRegistry).",
+    "metrics": ["storage_detail{metric=\"nudb_bytes\"}"]
+  },
+  "grafana_dashboards": {
+    "description": "All Grafana dashboards that must render data (UIDs as provisioned on disk under docker/telemetry/grafana/dashboards/).",
+    "uids": [
+      "rpc-performance",
+      "transaction-overview",
+      "consensus-health",
+      "ledger-operations",
+      "peer-network",
+      "peer-quality",
+      "fee-market",
+      "job-queue",
+      "validator-health",
+      "node-health",
+      "network-traffic",
+      "rpc-pathfinding",
+      "overlay-traffic-detail",
+      "ledger-data-sync"
+    ]
+  }
+}

docker/telemetry/workload/expected_spans.json

@@ -0,0 +1,419 @@
+{
+  "description": "Expected span inventory for xrpld telemetry validation. Attribute keys follow the 2026-05-13 span-attr naming redesign (bare/underscore form; dotted xrpl.* reserved for resource attributes). Sourced from the *SpanNames.h headers. Spans marked \"optional\": true are conditional — they only fire under traffic the harness may not produce (e.g. gRPC client, missing-ledger fetch, mode transitions) and are not failed when absent.",
+  "spans": [
+    {
+      "name": "rpc.ws_message",
+      "category": "rpc",
+      "parent": null,
+      "required_attributes": ["command"],
+      "config_flag": "trace_rpc",
+      "note": "WebSocket RPC root span. The load generator uses WS, so this is the RPC entry span (not rpc.http_request, which needs an HTTP/JSON-RPC client)."
+    },
+    {
+      "name": "rpc.process",
+      "category": "rpc",
+      "parent": "rpc.ws_message",
+      "required_attributes": [],
+      "config_flag": "trace_rpc"
+    },
+    {
+      "name": "rpc.command.*",
+      "category": "rpc",
+      "parent": "rpc.process",
+      "required_attributes": ["command", "version", "rpc_role", "rpc_status"],
+      "config_flag": "trace_rpc",
+      "note": "Wildcard — matches rpc.command.server_info, rpc.command.ledger, etc."
+    },
+    {
+      "name": "rpc.http_request",
+      "category": "rpc",
+      "parent": null,
+      "required_attributes": ["request_payload_size"],
+      "config_flag": "trace_rpc",
+      "optional": true,
+      "note": "HTTP/JSON-RPC root span. The harness load generator is WebSocket-only, so this does not fire."
+    },
+    {
+      "name": "tx.process",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["tx_hash", "local", "path"],
+      "config_flag": "trace_transactions"
+    },
+    {
+      "name": "tx.receive",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["tx_hash", "peer_id", "suppressed"],
+      "config_flag": "trace_transactions",
+      "note": "Cross-node span: parent context propagated from the sender's tx.process via protobuf. Also carries tx_type and peer_version. tx_status is only set when a tx is suppressed/known-bad, so it is not a required attribute on every tx.receive."
+    },
+    {
+      "name": "tx.apply",
+      "category": "transaction",
+      "parent": "ledger.build",
+      "required_attributes": ["tx_count", "tx_failed"],
+      "config_flag": "trace_transactions",
+      "note": "Apply-step span inside BuildLedger. Carries tx_count/tx_failed (ledger_seq lives on the parent ledger.build span)."
+    },
+    {
+      "name": "tx.preflight",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["stage", "tx_type", "ter_result"],
+      "config_flag": "trace_transactions",
+      "note": "Apply-pipeline stage span (stage=preflight). Shares a deterministic trace_id (txID[0:16]) with tx.preclaim/tx.transactor."
+    },
+    {
+      "name": "tx.preclaim",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["stage", "tx_type", "ter_result"],
+      "config_flag": "trace_transactions",
+      "note": "Apply-pipeline stage span (stage=preclaim)."
+    },
+    {
+      "name": "tx.transactor",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["stage", "tx_type"],
+      "config_flag": "trace_transactions",
+      "note": "Apply-pipeline stage span (stage=apply). Also carries applied."
+    },
+    {
+      "name": "txq.enqueue",
+      "category": "transaction",
+      "parent": "tx.process",
+      "required_attributes": ["tx_hash", "tx_type", "txq_status"],
+      "config_flag": "trace_transactions",
+      "optional": true,
+      "note": "Only fires when a tx is queued (fee below open-ledger level). Requires fee escalation — driven by the txq-burst workload phase. tx_hash/tx_type/txq_status are set on every code path; fee_level_paid/required_fee_level are conditional (TxQ.cpp ~895-898, after the rejected and applied_direct early exits), so they are NOT guaranteed on every txq.enqueue span and cannot be required."
+    },
+    {
+      "name": "txq.apply_direct",
+      "category": "transaction",
+      "parent": "txq.enqueue",
+      "required_attributes": [],
+      "config_flag": "trace_transactions",
+      "optional": true,
+      "note": "Child of txq.enqueue when the tx applies directly without queueing."
+    },
+    {
+      "name": "txq.batch_clear",
+      "category": "transaction",
+      "parent": "txq.enqueue",
+      "required_attributes": ["num_cleared"],
+      "config_flag": "trace_transactions",
+      "optional": true
+    },
+    {
+      "name": "txq.accept",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["queue_size", "ledger_changed"],
+      "config_flag": "trace_transactions",
+      "optional": true,
+      "note": "Ledger-close accept loop. Fires on the consensus thread; only meaningful when the queue is non-empty."
+    },
+    {
+      "name": "txq.accept_tx",
+      "category": "transaction",
+      "parent": "txq.accept",
+      "required_attributes": [
+        "tx_hash",
+        "ter_code",
+        "retries_remaining",
+        "txq_status"
+      ],
+      "config_flag": "trace_transactions",
+      "optional": true
+    },
+    {
+      "name": "txq.cleanup",
+      "category": "transaction",
+      "parent": null,
+      "required_attributes": ["ledger_seq", "expired_count"],
+      "config_flag": "trace_transactions",
+      "optional": true
+    },
+    {
+      "name": "consensus.round",
+      "category": "consensus",
+      "parent": null,
+      "required_attributes": [
+        "consensus_ledger_id",
+        "ledger_seq",
+        "consensus_mode",
+        "consensus_round_id",
+        "consensus_phase"
+      ],
+      "config_flag": "trace_consensus",
+      "note": "Root consensus span created per round. Also carries trace_strategy, previous_ledger_seq, previous_proposers, previous_round_time_ms."
+    },
+    {
+      "name": "consensus.phase.open",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": [],
+      "config_flag": "trace_consensus"
+    },
+    {
+      "name": "consensus.proposal.send",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": ["consensus_round"],
+      "config_flag": "trace_consensus",
+      "note": "Also carries is_bow_out."
+    },
+    {
+      "name": "consensus.ledger_close",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": ["ledger_seq", "consensus_mode"],
+      "config_flag": "trace_consensus",
+      "note": "Also carries tx_count_open, close_time_resolution_ms."
+    },
+    {
+      "name": "consensus.establish",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": [
+        "converge_percent",
+        "establish_count",
+        "proposers",
+        "disputes_count"
+      ],
+      "config_flag": "trace_consensus"
+    },
+    {
+      "name": "consensus.update_positions",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": [
+        "converge_percent",
+        "proposers",
+        "disputes_count"
+      ],
+      "config_flag": "trace_consensus"
+    },
+    {
+      "name": "consensus.check",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": [
+        "agree_count",
+        "disagree_count",
+        "threshold_percent",
+        "consensus_result"
+      ],
+      "config_flag": "trace_consensus"
+    },
+    {
+      "name": "consensus.accept",
+      "category": "consensus",
+      "parent": "consensus.round",
+      "required_attributes": ["proposers", "round_time_ms", "quorum"],
+      "config_flag": "trace_consensus"
+    },
+    {
+      "name": "consensus.accept.apply",
+      "category": "consensus",
+      "parent": "consensus.accept",
+      "required_attributes": [
+        "ledger_seq",
+        "close_time",
+        "parent_close_time",
+        "close_time_self",
+        "close_time_vote_bins",
+        "resolution_direction"
+      ],
+      "config_flag": "trace_consensus",
+      "note": "Also carries close_time_correct, close_resolution_ms, consensus_state, proposing, round_time_ms, tx_count."
+    },
+    {
+      "name": "consensus.validation.send",
+      "category": "consensus",
+      "parent": null,
+      "required_attributes": [
+        "ledger_seq",
+        "proposing",
+        "ledger_hash",
+        "full_validation"
+      ],
+      "config_flag": "trace_consensus",
+      "note": "follows-from consensus.accept. ledger_hash and full_validation are shared with peer.validation.receive (same keys, told apart by span name). Also carries validation_sign_time."
+    },
+    {
+      "name": "consensus.proposal.receive",
+      "category": "consensus",
+      "parent": null,
+      "required_attributes": [],
+      "config_flag": "trace_consensus",
+      "note": "Context-propagated from the sending peer. No required local attributes."
+    },
+    {
+      "name": "consensus.validation.receive",
+      "category": "consensus",
+      "parent": null,
+      "required_attributes": [],
+      "config_flag": "trace_consensus",
+      "note": "Context-propagated from the sending peer. No required local attributes."
+    },
+    {
+      "name": "consensus.mode_change",
+      "category": "consensus",
+      "parent": null,
+      "required_attributes": ["mode_old", "mode_new"],
+      "config_flag": "trace_consensus",
+      "optional": true,
+      "note": "Only fires on an operating-mode transition; a steady cluster rarely changes mode after warmup."
+    },
+    {
+      "name": "ledger.build",
+      "category": "ledger",
+      "parent": null,
+      "required_attributes": [
+        "ledger_seq",
+        "close_time",
+        "close_time_correct",
+        "close_resolution_ms"
+      ],
+      "config_flag": "trace_ledger",
+      "note": "tx_count/tx_failed live on the child tx.apply span, not here."
+    },
+    {
+      "name": "ledger.validate",
+      "category": "ledger",
+      "parent": null,
+      "required_attributes": ["ledger_seq", "validations"],
+      "config_flag": "trace_ledger"
+    },
+    {
+      "name": "ledger.store",
+      "category": "ledger",
+      "parent": null,
+      "required_attributes": ["ledger_seq"],
+      "config_flag": "trace_ledger"
+    },
+    {
+      "name": "ledger.acquire",
+      "category": "ledger",
+      "parent": null,
+      "required_attributes": [
+        "ledger_seq",
+        "acquire_reason",
+        "timeouts",
+        "peer_count",
+        "outcome"
+      ],
+      "config_flag": "trace_ledger",
+      "optional": true,
+      "note": "Only fires when a node must fetch a missing ledger (InboundLedger). A healthy local cluster rarely back-fills history."
+    },
+    {
+      "name": "peer.proposal.receive",
+      "category": "peer",
+      "parent": null,
+      "required_attributes": ["peer_id", "proposal_trusted"],
+      "config_flag": "trace_peer"
+    },
+    {
+      "name": "peer.validation.receive",
+      "category": "peer",
+      "parent": null,
+      "required_attributes": [
+        "peer_id",
+        "validation_trusted",
+        "ledger_hash",
+        "full_validation"
+      ],
+      "config_flag": "trace_peer",
+      "note": "ledger_hash and full_validation are shared with consensus.validation.send (same keys, told apart by span name)."
+    },
+    {
+      "name": "pathfind.request",
+      "category": "pathfind",
+      "parent": null,
+      "required_attributes": [
+        "pathfind_source_account",
+        "pathfind_dest_account"
+      ],
+      "config_flag": "trace_rpc",
+      "note": "Fires on ripple_path_find / path_find RPC. Driven by the ripple_path_find load in rpc_load_generator.py."
+    },
+    {
+      "name": "pathfind.compute",
+      "category": "pathfind",
+      "parent": "pathfind.request",
+      "required_attributes": ["pathfind_fast"],
+      "config_flag": "trace_rpc",
+      "optional": true,
+      "note": "Only fires when PathRequest::doUpdate runs a computation; the self-to-self XRP probe from the load generator returns early without computing paths in a fresh cluster with no liquidity."
+    },
+    {
+      "name": "pathfind.discover",
+      "category": "pathfind",
+      "parent": "pathfind.compute",
+      "required_attributes": ["pathfind_search_level", "pathfind_num_paths"],
+      "config_flag": "trace_rpc",
+      "optional": true,
+      "note": "Graph exploration; only fires under pathfind.compute, which needs real path liquidity not present in the fresh test cluster."
+    },
+    {
+      "name": "pathfind.update_all",
+      "category": "pathfind",
+      "parent": null,
+      "required_attributes": ["pathfind_ledger_index", "pathfind_num_requests"],
+      "config_flag": "trace_rpc",
+      "optional": true,
+      "note": "Async recomputation at ledger close; only fires when there are active path_find subscriptions (the one-shot ripple_path_find load does not register one)."
+    },
+    {
+      "name": "grpc.*",
+      "category": "grpc",
+      "parent": null,
+      "required_attributes": ["method", "grpc_role", "grpc_status"],
+      "config_flag": "trace_rpc",
+      "optional": true,
+      "note": "Wildcard — grpc.<MethodName>. The harness has no gRPC client, so these do not fire. Tracked for completeness."
+    }
+  ],
+  "parent_child_relationships": [
+    {
+      "parent": "rpc.ws_message",
+      "child": "rpc.process",
+      "description": "WebSocket message contains processing span",
+      "skip": true,
+      "skip_reason": "rpc.ws_message and rpc.process run on different threads (the WS handler posts a coroutine to JobQueue for processing). Span context is not propagated across the thread boundary. Requires a C++ fix to capture and forward the span context through the coroutine lambda."
+    },
+    {
+      "parent": "rpc.process",
+      "child": "rpc.command.*",
+      "description": "Processing span contains per-command span"
+    },
+    {
+      "parent": "ledger.build",
+      "child": "tx.apply",
+      "description": "Ledger build contains transaction application"
+    },
+    {
+      "parent": "consensus.round",
+      "child": "consensus.accept",
+      "description": "Consensus round contains the accept sub-span"
+    },
+    {
+      "parent": "consensus.accept",
+      "child": "consensus.accept.apply",
+      "description": "Accept contains the ledger-apply sub-span"
+    },
+    {
+      "parent": "pathfind.request",
+      "child": "pathfind.compute",
+      "description": "Pathfind request contains the compute sub-span",
+      "skip": true,
+      "skip_reason": "pathfind.compute only fires when a path computation actually runs; the self-to-self XRP probe in a fresh cluster with no liquidity returns before computing, so the child is not emitted under the harness workload."
+    }
+  ],
+  "total_span_types": 40,
+  "total_unique_attributes": 58
+}

docker/telemetry/workload/generate-validator-keys.sh

@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+# generate-validator-keys.sh — Generate validator key pairs for the workload harness.
+#
+# Uses a temporary standalone xrpld instance to call `validation_create` RPC
+# for each node. Outputs a JSON file mapping node index to seed + public key.
+#
+# Usage:
+#   ./generate-validator-keys.sh <xrpld_binary> <num_nodes> <output_dir>
+#
+# Output:
+#   <output_dir>/validator-keys.json — JSON array of {index, seed, public_key}
+#   <output_dir>/validators.txt     — [validators] section for xrpld.cfg
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Colored output helpers
+# ---------------------------------------------------------------------------
+log() { printf "\033[1;34m[KEYGEN]\033[0m %s\n" "$*"; }
+ok() { printf "\033[1;32m[KEYGEN]\033[0m %s\n" "$*"; }
+die() {
+    printf "\033[1;31m[KEYGEN]\033[0m %s\n" "$*" >&2
+    exit 1
+}
+
+# ---------------------------------------------------------------------------
+# Argument parsing
+# ---------------------------------------------------------------------------
+usage() {
+    echo "Usage: $0 <xrpld_binary> <num_nodes> <output_dir>"
+    echo ""
+    echo "Arguments:"
+    echo "  xrpld_binary  Path to xrpld binary (built with telemetry=ON)"
+    echo "  num_nodes     Number of validator key pairs to generate (1-20)"
+    echo "  output_dir    Directory to write validator-keys.json and validators.txt"
+    exit 1
+}
+
+if [ $# -lt 3 ]; then
+    usage
+fi
+
+XRPLD="$1"
+NUM_NODES="$2"
+OUTPUT_DIR="$3"
+
+# Validate arguments
+[ -x "$XRPLD" ] || die "xrpld binary not found or not executable: $XRPLD"
+[[ "$NUM_NODES" =~ ^[0-9]+$ ]] || die "num_nodes must be a positive integer"
+[ "$NUM_NODES" -ge 1 ] && [ "$NUM_NODES" -le 20 ] || die "num_nodes must be between 1 and 20"
+
+mkdir -p "$OUTPUT_DIR"
+
+# ---------------------------------------------------------------------------
+# Start a temporary standalone xrpld for key generation
+# ---------------------------------------------------------------------------
+TEMP_DIR="$(mktemp -d)"
+TEMP_PORT=5099
+TEMP_CFG="$TEMP_DIR/xrpld.cfg"
+
+log "Starting temporary xrpld for key generation (port $TEMP_PORT)..."
+
+cat >"$TEMP_CFG" <<EOCFG
+[server]
+port_rpc_keygen
+
+[port_rpc_keygen]
+port = $TEMP_PORT
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[node_db]
+type=NuDB
+path=$TEMP_DIR/nudb
+online_delete=256
+
+[database_path]
+$TEMP_DIR/db
+
+[debug_logfile]
+$TEMP_DIR/debug.log
+
+[ssl_verify]
+0
+EOCFG
+
+"$XRPLD" --conf "$TEMP_CFG" -a --start >"$TEMP_DIR/stdout.log" 2>&1 &
+TEMP_PID=$!
+
+# Ensure cleanup on exit
+cleanup_temp() {
+    kill "$TEMP_PID" 2>/dev/null || true
+    wait "$TEMP_PID" 2>/dev/null || true
+    rm -rf "$TEMP_DIR"
+}
+trap cleanup_temp EXIT
+
+# Wait for RPC to become available
+for attempt in $(seq 1 30); do
+    if curl -sf "http://localhost:$TEMP_PORT" \
+        -d '{"method":"server_info"}' >/dev/null 2>&1; then
+        log "Temporary xrpld RPC ready (attempt $attempt)."
+        break
+    fi
+    if [ "$attempt" -eq 30 ]; then
+        die "Temporary xrpld RPC not ready after 30s"
+    fi
+    sleep 1
+done
+
+# ---------------------------------------------------------------------------
+# Generate key pairs
+# ---------------------------------------------------------------------------
+log "Generating $NUM_NODES validator key pairs..."
+
+KEYS_JSON="["
+VALIDATORS_TXT="[validators]"
+
+for i in $(seq 1 "$NUM_NODES"); do
+    result=$(curl -sf "http://localhost:$TEMP_PORT" \
+        -d '{"method":"validation_create"}')
+    seed=$(echo "$result" | jq -r '.result.validation_seed')
+    pubkey=$(echo "$result" | jq -r '.result.validation_public_key')
+
+    if [ -z "$seed" ] || [ "$seed" = "null" ]; then
+        die "Failed to generate key pair for node $i"
+    fi
+
+    log "  Node $i: ${pubkey:0:20}..."
+
+    # Build JSON entry
+    entry="{\"index\": $i, \"seed\": \"$seed\", \"public_key\": \"$pubkey\"}"
+    if [ "$i" -gt 1 ]; then
+        KEYS_JSON="$KEYS_JSON,"
+    fi
+    KEYS_JSON="$KEYS_JSON$entry"
+
+    VALIDATORS_TXT="$VALIDATORS_TXT
+$pubkey"
+done
+
+KEYS_JSON="$KEYS_JSON]"
+
+# ---------------------------------------------------------------------------
+# Write output files
+# ---------------------------------------------------------------------------
+echo "$KEYS_JSON" | jq '.' >"$OUTPUT_DIR/validator-keys.json"
+echo "$VALIDATORS_TXT" >"$OUTPUT_DIR/validators.txt"
+
+ok "Generated $NUM_NODES key pairs:"
+ok "  Keys:       $OUTPUT_DIR/validator-keys.json"
+ok "  Validators: $OUTPUT_DIR/validators.txt"

docker/telemetry/workload/prom_queries.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""Shared Prometheus query helpers for the regression gate.
+
+Single source of truth for how regression metrics are computed. Both
+``capture_timings.py`` and any future tooling consume this module so metric
+name → PromQL expression stays consistent.
+
+Design:
+- Every captured metric has a key in the form ``{category}.{name}.p{quantile}``
+  (e.g. ``span.tx.process.p99``). Keys are flat strings so JSON diffing is
+  trivial.
+- Quantile queries go through ``histogram_quantile`` over the standard
+  ``_bucket`` series. The rate window is a parameter (defaults to the
+  capture window, not Prometheus's default 5m) so short CI runs are usable.
+- The catalogue of what to capture lives in ``regression-metrics.json`` —
+  this module only knows how to translate that JSON into HTTP queries.
+
+Usage::
+
+    import asyncio, aiohttp
+    from prom_queries import build_query_plan, run_query_plan
+
+    plan = build_query_plan("regression-metrics.json", window="3m")
+    async with aiohttp.ClientSession() as s:
+        timings = await run_query_plan(s, "http://localhost:9090", plan)
+    # timings = {"span.tx.process.p99": {"value": 12.4, "unit": "ms"}, ...}
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import aiohttp
+
+logger = logging.getLogger("prom_queries")
+
+
+@dataclass(frozen=True)
+class QueryEntry:
+    """One metric to capture from Prometheus.
+
+    Attributes:
+        key:      Flat output key, e.g. ``span.tx.process.p99``.
+        promql:   The PromQL expression to send to /api/v1/query.
+        unit:     Unit of the returned value, e.g. ``ms`` or ``us``.
+                  Baseline JSON preserves this so the comparator can
+                  sanity-check unit drift.
+    """
+
+    key: str
+    promql: str
+    unit: str
+
+
+def _build_simple_entries(
+    cfg: dict,
+    prefix: str,
+    window: str,
+) -> list[QueryEntry]:
+    """Build QueryEntry list for a single-template category (spans, rpc)."""
+    tmpl = cfg.get("_query_template", "")
+    unit = cfg.get("_unit", "ms")
+    entries: list[QueryEntry] = []
+    for name in cfg.get("names", []):
+        for q in cfg.get("_quantiles", []):
+            expr = (
+                tmpl.replace("{quantile}", _format_quantile(q))
+                .replace("{name}", name)
+                .replace("{window}", window)
+            )
+            entries.append(
+                QueryEntry(
+                    key=f"{prefix}.{name}.p{_quantile_label(q)}",
+                    promql=expr,
+                    unit=unit,
+                )
+            )
+    return entries
+
+
+def _build_job_entries(cfg: dict, window: str) -> list[QueryEntry]:
+    """Build QueryEntry list for the job_queue category (multi-phase)."""
+    unit = cfg.get("_unit", "us")
+    phases = cfg.get("_phases", ["queued", "running"])
+    tmpl_map = {
+        "queued": cfg.get("_queued_template", ""),
+        "running": cfg.get("_running_template", ""),
+    }
+    entries: list[QueryEntry] = []
+    for name in cfg.get("names", []):
+        for phase in phases:
+            tmpl = tmpl_map.get(phase, "")
+            if not tmpl:
+                continue
+            for q in cfg.get("_quantiles", []):
+                expr = (
+                    tmpl.replace("{quantile}", _format_quantile(q))
+                    .replace("{name}", name)
+                    .replace("{window}", window)
+                )
+                entries.append(
+                    QueryEntry(
+                        key=f"job.{name}.{phase}.p{_quantile_label(q)}",
+                        promql=expr,
+                        unit=unit,
+                    )
+                )
+    return entries
+
+
+def build_query_plan(metrics_path: str | Path, window: str = "3m") -> list[QueryEntry]:
+    """Translate regression-metrics.json into a list of PromQL queries.
+
+    Args:
+        metrics_path: Path to ``regression-metrics.json``.
+        window:       Rate window passed to ``rate()``. For short CI runs
+                      keep this close to the test duration so the bucket
+                      counts are meaningful. Default 3m matches the
+                      ``regression`` workload profile.
+
+    Returns:
+        A list of ``QueryEntry`` values, one per (metric × quantile).
+    """
+    with open(metrics_path) as f:
+        cfg = json.load(f)
+
+    plan: list[QueryEntry] = []
+    plan.extend(_build_simple_entries(cfg.get("spans", {}), "span", window))
+    plan.extend(_build_simple_entries(cfg.get("rpc_methods", {}), "rpc", window))
+    plan.extend(_build_job_entries(cfg.get("job_queue", {}), window))
+    return plan
+
+
+async def run_query_plan(
+    session: aiohttp.ClientSession,
+    prom_url: str,
+    plan: list[QueryEntry],
+) -> dict[str, dict[str, Any]]:
+    """Execute a query plan and return a flat ``key → {value, unit}`` map.
+
+    Queries that return no data (NaN, empty result) are still included in
+    the output with ``value: null`` — the comparator treats missing values
+    as "not yet observed" rather than as a regression. This keeps the
+    baseline schema stable across runs with different load levels.
+
+    Args:
+        session:  Shared aiohttp session.
+        prom_url: Base URL of Prometheus (e.g. ``http://localhost:9090``).
+        plan:     Output of :func:`build_query_plan`.
+
+    Returns:
+        Mapping from metric key to ``{"value": float|None, "unit": str}``.
+    """
+    results: dict[str, dict[str, Any]] = {}
+    for entry in plan:
+        value = await _instant_query(session, prom_url, entry.promql)
+        results[entry.key] = {"value": value, "unit": entry.unit}
+    return results
+
+
+async def _instant_query(
+    session: aiohttp.ClientSession,
+    prom_url: str,
+    promql: str,
+) -> float | None:
+    """POST an instant query to Prometheus; return the scalar value or None.
+
+    None is returned for NaN, empty results, or HTTP errors — every call
+    site treats None identically ("no data captured").
+    """
+    url = f"{prom_url.rstrip('/')}/api/v1/query"
+    try:
+        async with session.post(
+            url, data={"query": promql}, timeout=aiohttp.ClientTimeout(total=30)
+        ) as resp:
+            if resp.status != 200:
+                logger.warning("query HTTP %d: %s", resp.status, promql)
+                return None
+            body = await resp.json()
+    except (aiohttp.ClientError, TimeoutError) as exc:
+        logger.warning("query failed: %s — %s", promql, exc)
+        return None
+
+    if body.get("status") != "success":
+        logger.warning("query status=%s: %s", body.get("status"), promql)
+        return None
+
+    result = body.get("data", {}).get("result", [])
+    if not result:
+        return None
+
+    raw = result[0].get("value", [None, None])[1]
+    if raw is None or raw in ("NaN", "+Inf", "-Inf"):
+        return None
+    try:
+        return float(raw)
+    except (TypeError, ValueError):
+        return None
+
+
+def _format_quantile(q: float) -> str:
+    """Format a quantile for PromQL (``0.99`` → ``"0.99"``)."""
+    return f"{q:g}"
+
+
+def _quantile_label(q: float) -> str:
+    """Format a quantile for the output key (``0.95`` → ``"95"``)."""
+    return str(int(round(q * 100)))

docker/telemetry/workload/regression-metrics.json

@@ -0,0 +1,28 @@
+{
+  "_description": "Metric surface for the OTel-driven regression gate. Each entry names a metric, the quantiles to capture, and how to query Prometheus. The comparator compares current run against baseline-timings.json under these exact keys.",
+  "_key_format": "{category}.{name}.p{quantile}  (e.g. span.tx.process.p99, rpc.server_info.p95, job.transaction.queued.p95)",
+  "spans": {
+    "_query_template": "histogram_quantile({quantile}, sum by (le) (rate(span_duration_milliseconds_bucket{span_name=\"{name}\"}[{window}])))",
+    "_unit": "ms",
+    "_quantiles": [0.5, 0.95, 0.99],
+    "names": [
+      "rpc.ws_message",
+      "rpc.process",
+      "tx.process",
+      "tx.apply",
+      "ledger.build",
+      "ledger.validate",
+      "ledger.store",
+      "consensus.ledger_close",
+      "consensus.accept"
+    ]
+  },
+  "job_queue": {
+    "_queued_template": "histogram_quantile({quantile}, sum by (le) (rate(job_queued_us_bucket{job_type=\"{name}\"}[{window}])))",
+    "_running_template": "histogram_quantile({quantile}, sum by (le) (rate(job_running_us_bucket{job_type=\"{name}\"}[{window}])))",
+    "_unit": "us",
+    "_quantiles": [0.95],
+    "_phases": ["queued", "running"],
+    "names": ["transaction", "acceptLedger"]
+  }
+}

docker/telemetry/workload/regression-thresholds.json

@@ -0,0 +1,26 @@
+{
+  "_description": "Per-metric regression thresholds. A metric regresses when current - baseline exceeds BOTH the percentage and absolute bounds (AND, not OR — this tolerates small-value noise). Defaults apply unless a per-metric override exists.",
+  "_bucket_note": "SpanMetrics latency histograms use explicit buckets [1,5,10,25,50,100,250,500,1000,5000]ms. A quantile sitting near a low-end boundary can jump a full bucket (e.g. 1ms->5ms) between runs with no real change, so absolute span bounds are set to ~2 low-end bucket widths (10ms) to tolerate that quantization noise while still catching genuine multi-bucket regressions. The job_queue running bound is widened similarly — per-ledger apply work scales with TxQ burst load.",
+  "defaults": {
+    "span": {
+      "p50": { "max_pct_increase": 50.0, "max_abs_increase_ms": 10.0 },
+      "p95": { "max_pct_increase": 50.0, "max_abs_increase_ms": 10.0 },
+      "p99": { "max_pct_increase": 50.0, "max_abs_increase_ms": 15.0 }
+    },
+    "job_queue": {
+      "p95": { "max_pct_increase": 50.0, "max_abs_increase_us": 20000.0 }
+    }
+  },
+  "overrides": {
+    "span.consensus.ledger_close": {
+      "p50": { "max_pct_increase": 5.0, "max_abs_increase_ms": 200.0 },
+      "p95": { "max_pct_increase": 5.0, "max_abs_increase_ms": 500.0 },
+      "p99": { "max_pct_increase": 5.0, "max_abs_increase_ms": 1000.0 }
+    },
+    "span.consensus.accept": {
+      "p50": { "max_pct_increase": 5.0, "max_abs_increase_ms": 200.0 },
+      "p95": { "max_pct_increase": 5.0, "max_abs_increase_ms": 500.0 },
+      "p99": { "max_pct_increase": 5.0, "max_abs_increase_ms": 1000.0 }
+    }
+  }
+}

docker/telemetry/workload/requirements.txt

@@ -0,0 +1,6 @@
+# Python dependencies for Phase 10 workload tools.
+#
+# Install: pip install -r requirements.txt
+
+websockets>=12.0
+aiohttp>=3.9.0

docker/telemetry/workload/rpc_load_generator.py

@@ -0,0 +1,465 @@
+#!/usr/bin/env python3
+"""RPC Load Generator for rippled telemetry validation.
+
+Connects to one or more rippled WebSocket endpoints and fires all traced
+RPC commands at configurable rates with realistic production-like
+distribution.
+
+Command distribution (default weights):
+  40%  Health checks:   server_info, fee
+  30%  Wallet queries:  account_info, account_lines, account_objects
+  15%  Explorer:        ledger, ledger_data
+  10%  TX lookups:      tx, account_tx
+   5%  DEX queries:     book_offers, amm_info
+   3%  Pathfinding:     ripple_path_find
+
+Usage:
+    python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120
+
+    # Multiple endpoints (round-robin):
+    python3 rpc_load_generator.py \\
+        --endpoints ws://localhost:6006 ws://localhost:6007 \\
+        --rate 100 --duration 300
+
+    # Custom weights:
+    python3 rpc_load_generator.py --endpoints ws://localhost:6006 \\
+        --weights '{"server_info":60,"account_info":30,"ledger":10}'
+"""
+
+import argparse
+import asyncio
+import json
+import logging
+import random
+import sys
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any
+
+import websockets
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+# Default command distribution matching realistic production ratios.
+# Keys are RPC command names; values are relative weights.
+DEFAULT_WEIGHTS: dict[str, int] = {
+    # 40% health checks
+    "server_info": 25,
+    "fee": 15,
+    # 30% wallet queries
+    "account_info": 15,
+    "account_lines": 8,
+    "account_objects": 7,
+    # 15% explorer
+    "ledger": 10,
+    "ledger_data": 5,
+    # 10% tx lookups
+    "tx": 5,
+    "account_tx": 5,
+    # 5% DEX queries
+    "book_offers": 3,
+    "amm_info": 2,
+    # Pathfinding — exercises the pathfind.request/compute/discover spans.
+    # ripple_path_find is the synchronous (one-shot) variant that fits this
+    # fire-one-request WS client; path_find is a streaming subscription.
+    "ripple_path_find": 3,
+}
+
+# Well-known genesis account for queries that require an account parameter.
+GENESIS_ACCOUNT = "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"
+
+logger = logging.getLogger("rpc_load_generator")
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class LoadStats:
+    """Tracks request counts and latencies during a load run.
+
+    Attributes:
+        total_sent:     Total RPC requests dispatched.
+        total_success:  Requests that returned a valid result.
+        total_errors:   Requests that returned an error or timed out.
+        latencies:      Per-command list of round-trip times in seconds.
+        command_counts: Per-command request count.
+    """
+
+    total_sent: int = 0
+    total_success: int = 0
+    total_errors: int = 0
+    latencies: dict[str, list[float]] = field(default_factory=dict)
+    command_counts: dict[str, int] = field(default_factory=dict)
+
+    def record(self, command: str, latency: float, success: bool) -> None:
+        """Record the outcome of a single RPC call."""
+        self.total_sent += 1
+        if success:
+            self.total_success += 1
+        else:
+            self.total_errors += 1
+        self.latencies.setdefault(command, []).append(latency)
+        self.command_counts[command] = self.command_counts.get(command, 0) + 1
+
+    def summary(self) -> dict[str, Any]:
+        """Return a summary dict suitable for JSON serialization."""
+        per_command: dict[str, Any] = {}
+        for cmd, lats in self.latencies.items():
+            sorted_lats = sorted(lats)
+            n = len(sorted_lats)
+            per_command[cmd] = {
+                "count": self.command_counts.get(cmd, 0),
+                "p50_ms": round(sorted_lats[n // 2] * 1000, 2) if n else 0,
+                "p95_ms": (round(sorted_lats[int(n * 0.95)] * 1000, 2) if n else 0),
+                "p99_ms": (round(sorted_lats[int(n * 0.99)] * 1000, 2) if n else 0),
+            }
+        return {
+            "total_sent": self.total_sent,
+            "total_success": self.total_success,
+            "total_errors": self.total_errors,
+            "error_rate_pct": (
+                round(self.total_errors / self.total_sent * 100, 2)
+                if self.total_sent
+                else 0
+            ),
+            "per_command": per_command,
+        }
+
+
+# ---------------------------------------------------------------------------
+# RPC command builders
+# ---------------------------------------------------------------------------
+
+
+def build_rpc_request(command: str) -> dict[str, Any]:
+    """Build a native WebSocket command request for the given command.
+
+    Uses rippled's native WS format (``{"command": ...}``) with flat
+    parameters, NOT the JSON-RPC format (``{"method": ..., "params": [...]}``).
+
+    Args:
+        command: The rippled RPC command name.
+
+    Returns:
+        A dict representing the native WebSocket request body.
+    """
+    req: dict[str, Any] = {"command": command}
+
+    if command in ("server_info", "fee"):
+        pass  # No params needed.
+    elif command == "account_info":
+        req["account"] = GENESIS_ACCOUNT
+    elif command == "account_lines":
+        req["account"] = GENESIS_ACCOUNT
+    elif command == "account_objects":
+        req["account"] = GENESIS_ACCOUNT
+        req["limit"] = 10
+    elif command == "ledger":
+        req["ledger_index"] = "validated"
+    elif command == "ledger_data":
+        req["ledger_index"] = "validated"
+        req["limit"] = 5
+    elif command == "tx":
+        # Use a dummy hash — returns "txnNotFound" error but still exercises
+        # the full RPC span pipeline (rpc.ws_message -> rpc.process -> rpc.command.tx).
+        req["transaction"] = "0" * 64
+        req["binary"] = False
+    elif command == "account_tx":
+        req["account"] = GENESIS_ACCOUNT
+        req["ledger_index_min"] = -1
+        req["ledger_index_max"] = -1
+        req["limit"] = 5
+    elif command == "book_offers":
+        req["taker_pays"] = {"currency": "XRP"}
+        req["taker_gets"] = {
+            "currency": "USD",
+            "issuer": GENESIS_ACCOUNT,
+        }
+        req["limit"] = 5
+    elif command == "amm_info":
+        # AMM may not exist — the span is still created on the server side.
+        req["asset"] = {"currency": "XRP"}
+        req["asset2"] = {
+            "currency": "USD",
+            "issuer": GENESIS_ACCOUNT,
+        }
+    elif command == "ripple_path_find":
+        # Self-to-self XRP path search. It returns no usable paths, but the
+        # server still runs the full pathfinding pipeline (pathfind.request ->
+        # pathfind.compute -> pathfind.discover), which is what we trace.
+        req["source_account"] = GENESIS_ACCOUNT
+        req["destination_account"] = GENESIS_ACCOUNT
+        req["destination_amount"] = "1000000"  # 1 XRP in drops
+
+    return req
+
+
+def choose_command(weights: dict[str, int]) -> str:
+    """Select a random RPC command based on configured weights.
+
+    Args:
+        weights: Mapping of command name to relative weight.
+
+    Returns:
+        A command name string.
+    """
+    commands = list(weights.keys())
+    w = [weights[c] for c in commands]
+    return random.choices(commands, weights=w, k=1)[0]
+
+
+# ---------------------------------------------------------------------------
+# WebSocket RPC client
+# ---------------------------------------------------------------------------
+
+
+async def send_rpc(
+    ws: websockets.WebSocketClientProtocol,
+    command: str,
+    stats: LoadStats,
+    inject_traceparent: bool = True,
+) -> None:
+    """Send a single RPC request over WebSocket and record the result.
+
+    Args:
+        ws:                  Open WebSocket connection.
+        command:             RPC command name.
+        stats:               LoadStats instance to record results.
+        inject_traceparent:  If True, add a W3C traceparent header field
+                             to the request for context propagation testing.
+    """
+    request = build_rpc_request(command)
+
+    # Inject W3C traceparent for context propagation testing.
+    # The rippled WebSocket handler extracts this from the JSON body
+    # when present (Phase 2 context propagation).
+    if inject_traceparent:
+        trace_id = uuid.uuid4().hex
+        span_id = uuid.uuid4().hex[:16]
+        request["traceparent"] = f"00-{trace_id}-{span_id}-01"
+
+    t0 = time.monotonic()
+    try:
+        await ws.send(json.dumps(request))
+        raw = await asyncio.wait_for(ws.recv(), timeout=10.0)
+        latency = time.monotonic() - t0
+        response = json.loads(raw)
+        # Native WS responses have {"status": "success", "result": {...}}
+        # or {"status": "error", "error": "...", "error_message": "..."}.
+        success = response.get("status") == "success"
+        stats.record(command, latency, success)
+    except (asyncio.TimeoutError, websockets.exceptions.WebSocketException) as exc:
+        latency = time.monotonic() - t0
+        stats.record(command, latency, False)
+        logger.debug("RPC %s failed: %s", command, exc)
+
+
+async def run_load(
+    endpoints: list[str],
+    rate: float,
+    duration: float,
+    weights: dict[str, int],
+    inject_traceparent: bool,
+) -> LoadStats:
+    """Run the RPC load generator against the given endpoints.
+
+    Distributes requests round-robin across endpoints at the specified
+    rate (requests per second) for the given duration.
+
+    Args:
+        endpoints:          List of WebSocket URLs (ws://host:port).
+        rate:               Target requests per second.
+        duration:           Total run time in seconds.
+        weights:            Command distribution weights.
+        inject_traceparent: Whether to inject W3C traceparent headers.
+
+    Returns:
+        LoadStats with aggregated results.
+    """
+    stats = LoadStats()
+    interval = 1.0 / rate if rate > 0 else 0.1
+
+    # Open persistent connections to all endpoints.
+    connections: list[websockets.WebSocketClientProtocol] = []
+    for ep in endpoints:
+        try:
+            ws = await websockets.connect(ep, ping_interval=20, ping_timeout=10)
+            connections.append(ws)
+            logger.info("Connected to %s", ep)
+        except Exception as exc:
+            logger.error("Failed to connect to %s: %s", ep, exc)
+
+    if not connections:
+        logger.error("No connections established. Aborting.")
+        return stats
+
+    logger.info(
+        "Starting load: rate=%s RPS, duration=%ss, endpoints=%d",
+        rate,
+        duration,
+        len(connections),
+    )
+
+    start = time.monotonic()
+    conn_idx = 0
+
+    try:
+        while (time.monotonic() - start) < duration:
+            command = choose_command(weights)
+            ws = connections[conn_idx % len(connections)]
+            conn_idx += 1
+
+            # Fire-and-forget style with bounded concurrency via sleep.
+            asyncio.create_task(send_rpc(ws, command, stats, inject_traceparent))
+            await asyncio.sleep(interval)
+
+            # Periodic progress log.
+            elapsed = time.monotonic() - start
+            if stats.total_sent % 100 == 0 and stats.total_sent > 0:
+                actual_rps = stats.total_sent / elapsed if elapsed > 0 else 0
+                logger.info(
+                    "Progress: %d sent, %d errors, %.1f RPS (%.0fs elapsed)",
+                    stats.total_sent,
+                    stats.total_errors,
+                    actual_rps,
+                    elapsed,
+                )
+    except asyncio.CancelledError:
+        logger.info("Load generation cancelled.")
+    finally:
+        # Allow in-flight requests to complete.
+        await asyncio.sleep(2)
+        for ws in connections:
+            await ws.close()
+
+    elapsed = time.monotonic() - start
+    logger.info(
+        "Load complete: %d sent, %d success, %d errors in %.1fs (%.1f RPS)",
+        stats.total_sent,
+        stats.total_success,
+        stats.total_errors,
+        elapsed,
+        stats.total_sent / elapsed if elapsed > 0 else 0,
+    )
+
+    return stats
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="RPC Load Generator for rippled telemetry validation",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Basic usage (50 RPS for 2 minutes):
+  python3 rpc_load_generator.py --endpoints ws://localhost:6006 --rate 50 --duration 120
+
+  # Multiple endpoints with custom weights:
+  python3 rpc_load_generator.py \\
+      --endpoints ws://localhost:6006 ws://localhost:6007 \\
+      --rate 100 --duration 300 \\
+      --weights '{"server_info": 80, "account_info": 20}'
+        """,
+    )
+    parser.add_argument(
+        "--endpoints",
+        nargs="+",
+        default=["ws://localhost:6006"],
+        help="WebSocket endpoints (default: ws://localhost:6006)",
+    )
+    parser.add_argument(
+        "--rate",
+        type=float,
+        default=50.0,
+        help="Target requests per second (default: 50)",
+    )
+    parser.add_argument(
+        "--duration",
+        type=float,
+        default=120.0,
+        help="Run duration in seconds (default: 120)",
+    )
+    parser.add_argument(
+        "--weights",
+        type=str,
+        default=None,
+        help="JSON string of command weights (overrides defaults)",
+    )
+    parser.add_argument(
+        "--no-traceparent",
+        action="store_true",
+        help="Disable W3C traceparent injection",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default=None,
+        help="Write JSON summary to this file path",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable debug logging",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Main entry point for the RPC load generator."""
+    args = parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s [%(name)s] %(levelname)s %(message)s",
+    )
+
+    # Parse custom weights if provided.
+    weights = DEFAULT_WEIGHTS.copy()
+    if args.weights:
+        try:
+            custom = json.loads(args.weights)
+            weights = {k: int(v) for k, v in custom.items()}
+            logger.info("Using custom weights: %s", weights)
+        except (json.JSONDecodeError, ValueError) as exc:
+            logger.error("Invalid --weights JSON: %s", exc)
+            sys.exit(1)
+
+    # Run the load generator.
+    stats = asyncio.run(
+        run_load(
+            endpoints=args.endpoints,
+            rate=args.rate,
+            duration=args.duration,
+            weights=weights,
+            inject_traceparent=not args.no_traceparent,
+        )
+    )
+
+    summary = stats.summary()
+    print(json.dumps(summary, indent=2))
+
+    if args.output:
+        with open(args.output, "w") as f:
+            json.dump(summary, f, indent=2)
+        logger.info("Summary written to %s", args.output)
+
+    # Exit with error if error rate exceeds 50%.
+    if summary["error_rate_pct"] > 50:
+        logger.error("High error rate: %.1f%%", summary["error_rate_pct"])
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

docker/telemetry/workload/run-full-validation.sh

@@ -0,0 +1,500 @@
+#!/usr/bin/env bash
+# run-full-validation.sh — Orchestrates the full telemetry validation pipeline.
+#
+# Sequence:
+#   1. Start the observability stack (OTel Collector, Tempo, Prometheus, Loki, Grafana)
+#   2. Start a multi-node rippled cluster with full telemetry enabled
+#   3. Wait for consensus
+#   4. Run workload orchestrator (RPC load, TX submission, propagation wait)
+#   5. Run the telemetry validation suite
+#   6. Capture OTel timings and compare against committed baseline
+#   7. (Optional) Run the performance overhead benchmark
+#
+# Usage:
+#   ./run-full-validation.sh --xrpld /path/to/xrpld
+#   ./run-full-validation.sh --xrpld /path/to/xrpld --with-benchmark
+#   ./run-full-validation.sh --xrpld /path/to/xrpld --skip-regression
+#   ./run-full-validation.sh --cleanup
+#
+# Exit codes:
+#   0 — All validation checks and the regression gate passed
+#   1 — Validation checks failed OR the regression gate detected a regression
+#   2 — Infrastructure error (cluster/stack failed to start, timing capture failed)
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Colored output helpers
+# ---------------------------------------------------------------------------
+log() { printf "\033[1;34m[VALIDATE]\033[0m %s\n" "$*"; }
+ok() { printf "\033[1;32m[VALIDATE]\033[0m %s\n" "$*"; }
+warn() { printf "\033[1;33m[VALIDATE]\033[0m %s\n" "$*"; }
+fail() { printf "\033[1;31m[VALIDATE]\033[0m %s\n" "$*"; }
+die() {
+    printf "\033[1;31m[VALIDATE]\033[0m %s\n" "$*" >&2
+    exit 2
+}
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TELEMETRY_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+REPO_ROOT="$(cd "$TELEMETRY_DIR/../.." && pwd)"
+COMPOSE_FILE="$TELEMETRY_DIR/docker-compose.workload.yaml"
+WORKDIR="/tmp/xrpld-validation"
+
+XRPLD="${XRPLD:-$REPO_ROOT/.build/xrpld}"
+NUM_NODES=5
+RPC_PORT_BASE=5005
+WS_PORT_BASE=6006
+PEER_PORT_BASE=51235
+RPC_RATE=50
+RPC_DURATION=120
+TX_TPS=5
+TX_DURATION=120
+WITH_BENCHMARK=false
+SKIP_LOKI=false
+SKIP_REGRESSION=false
+WORKLOAD_PROFILE="full-validation"
+REPORT_DIR="$WORKDIR/reports"
+# Rate window handed to Prometheus `rate()` when capturing timings. Keep
+# this close to the active workload duration so histogram buckets cover
+# the measurement window; longer windows dilute short-lived regressions.
+REGRESSION_WINDOW="${REGRESSION_WINDOW:-3m}"
+BASELINE_FILE="${BASELINE_FILE:-$SCRIPT_DIR/baselines/baseline-timings.json}"
+THRESHOLDS_FILE="${THRESHOLDS_FILE:-$SCRIPT_DIR/regression-thresholds.json}"
+METRICS_FILE="${METRICS_FILE:-$SCRIPT_DIR/regression-metrics.json}"
+
+GENESIS_ACCOUNT="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"
+GENESIS_SEED="snoPBrXtMeMyMHUVTgbuqAfg1SUTb"
+
+# ---------------------------------------------------------------------------
+# Argument parsing
+# ---------------------------------------------------------------------------
+usage() {
+    echo "Usage: $0 [OPTIONS]"
+    echo ""
+    echo "Options:"
+    echo "  --xrpld PATH         Path to xrpld binary"
+    echo "  --nodes NUM          Number of validator nodes (default: 5)"
+    echo "  --rpc-rate RPS       RPC load rate (default: 50)"
+    echo "  --rpc-duration SECS  RPC load duration (default: 120)"
+    echo "  --tx-tps TPS         Transaction submit rate (default: 5)"
+    echo "  --tx-duration SECS   Transaction submit duration (default: 120)"
+    echo "  --profile NAME       Workload profile (default: full-validation)"
+    echo "  --with-benchmark     Also run performance overhead benchmark (telemetry off vs on)"
+    echo "  --skip-loki          Skip Loki log-trace correlation checks"
+    echo "  --skip-regression    Skip the OTel-baseline regression gate"
+    echo "  --cleanup            Tear down everything and exit"
+    echo "  -h, --help           Show this help"
+    exit 0
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --xrpld)
+            XRPLD="$2"
+            shift 2
+            ;;
+        --nodes)
+            NUM_NODES="$2"
+            shift 2
+            ;;
+        --rpc-rate)
+            RPC_RATE="$2"
+            shift 2
+            ;;
+        --rpc-duration)
+            RPC_DURATION="$2"
+            shift 2
+            ;;
+        --tx-tps)
+            TX_TPS="$2"
+            shift 2
+            ;;
+        --tx-duration)
+            TX_DURATION="$2"
+            shift 2
+            ;;
+        --profile)
+            WORKLOAD_PROFILE="$2"
+            shift 2
+            ;;
+        --with-benchmark)
+            WITH_BENCHMARK=true
+            shift
+            ;;
+        --skip-loki)
+            SKIP_LOKI=true
+            shift
+            ;;
+        --skip-regression)
+            SKIP_REGRESSION=true
+            shift
+            ;;
+        --cleanup) # Cleanup mode
+            log "Cleaning up..."
+            pkill -f "$WORKDIR" 2>/dev/null || true
+            docker compose -f "$COMPOSE_FILE" down 2>/dev/null || true
+            rm -rf "$WORKDIR"
+            ok "Cleanup complete."
+            exit 0
+            ;;
+        -h | --help) usage ;;
+        *) die "Unknown option: $1" ;;
+    esac
+done
+
+# ---------------------------------------------------------------------------
+# Prerequisites
+# ---------------------------------------------------------------------------
+log "Checking prerequisites..."
+[ -x "$XRPLD" ] || die "xrpld binary not found: $XRPLD"
+command -v docker >/dev/null 2>&1 || die "docker not found"
+docker compose version >/dev/null 2>&1 || die "docker compose (v2) not found"
+command -v python3 >/dev/null 2>&1 || die "python3 not found"
+command -v curl >/dev/null 2>&1 || die "curl not found"
+command -v jq >/dev/null 2>&1 || die "jq not found"
+[ -f "$COMPOSE_FILE" ] || die "docker-compose.workload.yaml not found"
+
+# Install Python dependencies.
+log "Installing Python dependencies..."
+pip3 install -q -r "$SCRIPT_DIR/requirements.txt" 2>/dev/null ||
+    pip install -q -r "$SCRIPT_DIR/requirements.txt" 2>/dev/null ||
+    warn "Could not install Python dependencies — they may already be present"
+
+ok "Prerequisites verified."
+
+# ---------------------------------------------------------------------------
+# Cleanup previous run
+# ---------------------------------------------------------------------------
+log "Cleaning up previous run..."
+pkill -f "$WORKDIR" 2>/dev/null || true
+sleep 2
+rm -rf "$WORKDIR"
+mkdir -p "$WORKDIR" "$REPORT_DIR"
+
+# ---------------------------------------------------------------------------
+# Step 1: Start observability stack
+# ---------------------------------------------------------------------------
+log "Step 1: Starting observability stack..."
+# Point the collector's log mount at this run's workdir so the filelog
+# receiver tails the per-node debug.log files generated below.
+XRPLD_LOG_DIR="$WORKDIR" docker compose -f "$COMPOSE_FILE" up -d
+
+log "Waiting for OTel Collector..."
+for attempt in $(seq 1 30); do
+    status=$(curl -so /dev/null -w '%{http_code}' http://localhost:4318/ 2>/dev/null || echo 000)
+    if [ "$status" != "000" ]; then
+        ok "OTel Collector ready (attempt $attempt)"
+        break
+    fi
+    [ "$attempt" -eq 30 ] && die "OTel Collector not ready after 30s"
+    sleep 1
+done
+
+log "Waiting for Tempo..."
+for attempt in $(seq 1 30); do
+    if curl -sf "http://localhost:3200/ready" >/dev/null 2>&1; then
+        ok "Tempo ready (attempt $attempt)"
+        break
+    fi
+    [ "$attempt" -eq 30 ] && die "Tempo not ready after 30s"
+    sleep 1
+done
+
+log "Waiting for Prometheus..."
+for attempt in $(seq 1 30); do
+    if curl -sf "http://localhost:9090/-/healthy" >/dev/null 2>&1; then
+        ok "Prometheus ready (attempt $attempt)"
+        break
+    fi
+    [ "$attempt" -eq 30 ] && die "Prometheus not ready after 30s"
+    sleep 1
+done
+
+# ---------------------------------------------------------------------------
+# Step 2: Generate validator keys and start cluster
+# ---------------------------------------------------------------------------
+log "Step 2: Starting $NUM_NODES-node validator cluster..."
+
+bash "$SCRIPT_DIR/generate-validator-keys.sh" "$XRPLD" "$NUM_NODES" "$WORKDIR"
+
+for i in $(seq 1 "$NUM_NODES"); do
+    NODE_DIR="$WORKDIR/node$i"
+    mkdir -p "$NODE_DIR/nudb" "$NODE_DIR/db"
+
+    RPC_PORT=$((RPC_PORT_BASE + i - 1))
+    WS_PORT=$((WS_PORT_BASE + i - 1))
+    PEER_PORT=$((PEER_PORT_BASE + i - 1))
+    SEED=$(jq -r ".[$((i - 1))].seed" "$WORKDIR/validator-keys.json")
+
+    # Build ips_fixed.
+    IPS_FIXED=""
+    for j in $(seq 1 "$NUM_NODES"); do
+        if [ "$j" -ne "$i" ]; then
+            IPS_FIXED="${IPS_FIXED}127.0.0.1 $((PEER_PORT_BASE + j - 1))
+"
+        fi
+    done
+
+    cat >"$NODE_DIR/xrpld.cfg" <<EOCFG
+[server]
+port_rpc
+port_ws
+port_peer
+
+[port_rpc]
+port = $RPC_PORT
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[port_ws]
+port = $WS_PORT
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = ws
+
+[port_peer]
+port = $PEER_PORT
+ip = 0.0.0.0
+protocol = peer
+
+[node_db]
+type=NuDB
+path=$NODE_DIR/nudb
+online_delete=256
+
+[database_path]
+$NODE_DIR/db
+
+[debug_logfile]
+$NODE_DIR/debug.log
+
+[validation_seed]
+$SEED
+
+[validators_file]
+$WORKDIR/validators.txt
+
+[ips]
+${IPS_FIXED}
+
+[telemetry]
+enabled=1
+service_instance_id=validator-${i}
+endpoint=http://localhost:4318/v1/traces
+exporter=otlp_http
+batch_size=512
+batch_delay_ms=2000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1
+
+[insight]
+# Native OTel metrics via OTLP/HTTP. The collector has no StatsD receiver
+# (metrics pipeline is [otlp, spanmetrics]), so beast::insight must export
+# over OTLP for system metrics to reach Prometheus. prefix=xrpld matches the
+# OTel resource service name and the metric names the dashboards query.
+server=otel
+endpoint=http://localhost:4318/v1/metrics
+prefix=xrpld
+
+[rpc_startup]
+{ "command": "log_level", "severity": "warning" }
+
+[signing_support]
+true
+
+[ssl_verify]
+0
+EOCFG
+
+    "$XRPLD" --conf "$NODE_DIR/xrpld.cfg" --start >"$NODE_DIR/stdout.log" 2>&1 &
+    echo $! >"$NODE_DIR/xrpld.pid"
+    log "  Node $i: RPC=$RPC_PORT WS=$WS_PORT Peer=$PEER_PORT PID=$!"
+done
+
+# ---------------------------------------------------------------------------
+# Step 3: Wait for consensus
+# ---------------------------------------------------------------------------
+log "Step 3: Waiting for consensus..."
+for attempt in $(seq 1 120); do
+    ready=0
+    for i in $(seq 1 "$NUM_NODES"); do
+        port=$((RPC_PORT_BASE + i - 1))
+        state=$(curl -sf "http://localhost:$port" \
+            -d '{"method":"server_info"}' 2>/dev/null |
+            jq -r '.result.info.server_state' 2>/dev/null || echo "")
+        if [ "$state" = "proposing" ]; then
+            ready=$((ready + 1))
+        fi
+    done
+    if [ "$ready" -ge "$NUM_NODES" ]; then
+        ok "All $NUM_NODES nodes proposing (attempt $attempt)"
+        break
+    fi
+    if [ "$attempt" -eq 120 ]; then
+        warn "Consensus timeout — $ready/$NUM_NODES nodes ready"
+    fi
+    printf "\r  %d/%d nodes proposing..." "$ready" "$NUM_NODES"
+    sleep 1
+done
+echo ""
+
+# Wait for first validated ledger.
+log "Waiting for validated ledger..."
+for attempt in $(seq 1 60); do
+    val_seq=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
+        -d '{"method":"server_info"}' 2>/dev/null |
+        jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
+    if [ "$val_seq" -gt 2 ] 2>/dev/null; then
+        ok "Validated ledger: seq $val_seq"
+        break
+    fi
+    [ "$attempt" -eq 60 ] && warn "No validated ledger after 60s"
+    sleep 1
+done
+
+# ---------------------------------------------------------------------------
+# Step 4: Run workload orchestrator
+# ---------------------------------------------------------------------------
+log "Step 4: Running workload orchestrator (profile: $WORKLOAD_PROFILE)..."
+
+WS_ENDPOINTS=""
+for i in $(seq 1 "$NUM_NODES"); do
+    WS_ENDPOINTS="$WS_ENDPOINTS ws://localhost:$((WS_PORT_BASE + i - 1))"
+done
+
+python3 "$SCRIPT_DIR/workload_orchestrator.py" \
+    --profile "$WORKLOAD_PROFILE" \
+    --endpoints $WS_ENDPOINTS \
+    --report "$REPORT_DIR/workload-report.json" \
+    --report-dir "$REPORT_DIR" ||
+    warn "Workload orchestrator returned non-zero exit"
+
+ok "Workload orchestration complete."
+
+# ---------------------------------------------------------------------------
+# Step 5: Run telemetry validation suite
+# ---------------------------------------------------------------------------
+log "Step 5: Running telemetry validation suite..."
+
+VALIDATION_ARGS="--report $REPORT_DIR/validation-report.json"
+if [ "$SKIP_LOKI" = true ]; then
+    VALIDATION_ARGS="$VALIDATION_ARGS --skip-loki"
+fi
+
+VALIDATION_EXIT=0
+python3 "$SCRIPT_DIR/validate_telemetry.py" $VALIDATION_ARGS || VALIDATION_EXIT=$?
+
+if [ "$VALIDATION_EXIT" -eq 0 ]; then
+    ok "All telemetry validation checks passed!"
+else
+    fail "Some telemetry validation checks failed (exit $VALIDATION_EXIT)"
+fi
+
+# ---------------------------------------------------------------------------
+# Step 6: Capture OTel timings and run the regression comparison
+# ---------------------------------------------------------------------------
+# This step ALWAYS captures timings (so CI always has an artifact from which
+# to bootstrap/refresh the committed baseline). The comparator then either:
+#   - prints the paste-me JSON when the baseline is a placeholder, or
+#   - enforces thresholds and fails the run on regression.
+# Use --skip-regression to opt out (e.g. for ad-hoc local exploration).
+TIMINGS_FILE="$REPORT_DIR/timings.json"
+REGRESSION_REPORT="$REPORT_DIR/regression-report.json"
+REGRESSION_EXIT=0
+
+if [ "$SKIP_REGRESSION" != true ]; then
+    log "Step 6: Capturing OTel timings from Prometheus..."
+    if python3 "$SCRIPT_DIR/capture_timings.py" \
+        --prometheus "http://localhost:9090" \
+        --metrics "$METRICS_FILE" \
+        --output "$TIMINGS_FILE" \
+        --window "$REGRESSION_WINDOW" \
+        --profile "$WORKLOAD_PROFILE"; then
+        ok "Timings captured: $TIMINGS_FILE"
+    else
+        fail "Failed to capture timings — skipping regression comparison."
+        REGRESSION_EXIT=2
+        SKIP_REGRESSION=true
+    fi
+fi
+
+if [ "$SKIP_REGRESSION" != true ]; then
+    log "Comparing against baseline $BASELINE_FILE..."
+    python3 "$SCRIPT_DIR/compare_to_baseline.py" \
+        --timings "$TIMINGS_FILE" \
+        --baseline "$BASELINE_FILE" \
+        --thresholds "$THRESHOLDS_FILE" \
+        --report "$REGRESSION_REPORT" || REGRESSION_EXIT=$?
+    if [ "$REGRESSION_EXIT" -eq 0 ]; then
+        ok "Regression gate passed (or baseline placeholder — paste JSON printed above)."
+    elif [ "$REGRESSION_EXIT" -eq 1 ]; then
+        fail "Regression detected — see $REGRESSION_REPORT"
+    else
+        fail "Regression comparator internal error (exit $REGRESSION_EXIT)"
+    fi
+else
+    warn "Regression gate skipped."
+fi
+
+# ---------------------------------------------------------------------------
+# Step 7: (Optional) Run overhead benchmark
+# ---------------------------------------------------------------------------
+if [ "$WITH_BENCHMARK" = true ]; then
+    log "Step 7: Running performance benchmark..."
+    bash "$SCRIPT_DIR/benchmark.sh" \
+        --xrpld "$XRPLD" \
+        --duration 120 \
+        --nodes 3 \
+        --output "$REPORT_DIR" ||
+        warn "Benchmark returned non-zero exit"
+fi
+
+# ---------------------------------------------------------------------------
+# Summary
+# ---------------------------------------------------------------------------
+echo ""
+echo "==========================================================="
+echo "  FULL VALIDATION RESULTS"
+echo "==========================================================="
+echo ""
+echo "  Reports directory: $REPORT_DIR"
+echo ""
+ls -la "$REPORT_DIR/" 2>/dev/null || true
+echo ""
+echo "  Observability stack is running:"
+echo "    Tempo:         http://localhost:3200"
+echo "    Grafana:       http://localhost:3000"
+echo "    Prometheus:    http://localhost:9090"
+echo ""
+echo "  xrpld nodes ($NUM_NODES) are running:"
+for i in $(seq 1 "$NUM_NODES"); do
+    rpc=$((RPC_PORT_BASE + i - 1))
+    ws=$((WS_PORT_BASE + i - 1))
+    pid=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null || echo 'unknown')
+    echo "    Node $i: RPC=$rpc WS=$ws PID=$pid"
+done
+echo ""
+echo "  To tear down:"
+echo "    $0 --cleanup"
+echo ""
+echo "==========================================================="
+
+# Fail the run if EITHER validation or the regression gate failed. The
+# `[ "$VAR" -gt N ]` comparison works here because exit codes are numeric.
+FINAL_EXIT=0
+if [ "$VALIDATION_EXIT" -ne 0 ]; then
+    FINAL_EXIT="$VALIDATION_EXIT"
+fi
+if [ "$REGRESSION_EXIT" -ne 0 ] && [ "$FINAL_EXIT" -eq 0 ]; then
+    FINAL_EXIT="$REGRESSION_EXIT"
+fi
+exit "$FINAL_EXIT"

docker/telemetry/workload/test_accounts.json

@@ -0,0 +1,42 @@
+{
+  "genesis": {
+    "account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh",
+    "seed": "snoPBrXtMeMyMHUVTgbuqAfg1SUTb",
+    "description": "Genesis account with all XRP. Used to fund test accounts."
+  },
+  "test_accounts": [
+    {
+      "name": "alice",
+      "description": "Primary sender for Payment and OfferCreate transactions."
+    },
+    {
+      "name": "bob",
+      "description": "Primary receiver for Payment transactions."
+    },
+    {
+      "name": "carol",
+      "description": "TrustSet and issued currency counterparty."
+    },
+    {
+      "name": "dave",
+      "description": "NFToken operations (mint, offer, accept)."
+    },
+    {
+      "name": "eve",
+      "description": "Escrow operations (create, finish)."
+    },
+    {
+      "name": "frank",
+      "description": "AMM pool operations (create, deposit, withdraw)."
+    },
+    {
+      "name": "grace",
+      "description": "Additional sender for parallel transaction submission."
+    },
+    {
+      "name": "heidi",
+      "description": "Additional receiver for payment diversity."
+    }
+  ],
+  "note": "Test account keypairs are generated dynamically at runtime via wallet_propose RPC. This file defines the logical roles. Actual keys are stored in the workdir during execution."
+}

docker/telemetry/workload/tx_submitter.py

@@ -0,0 +1,848 @@
+#!/usr/bin/env python3
+"""Transaction Submitter for rippled telemetry validation.
+
+Generates diverse transaction types against a rippled cluster to exercise
+the full span and metric surface: tx.process, tx.apply, ledger.build,
+consensus.*, and all associated attributes.
+
+Pre-funds test accounts from the genesis account, then submits a
+configurable mix of transaction types at a target TPS.
+
+Supported transaction types:
+  - Payment (XRP and issued currencies)
+  - OfferCreate / OfferCancel (DEX activity)
+  - TrustSet (trust line creation)
+  - NFTokenMint / NFTokenCreateOffer / NFTokenAcceptOffer
+  - EscrowCreate / EscrowFinish
+  - AMMCreate / AMMDeposit / AMMWithdraw (if amendment enabled)
+
+Usage:
+    python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120
+
+    # Custom transaction mix:
+    python3 tx_submitter.py --endpoint ws://localhost:6006 \\
+        --weights '{"Payment":50,"OfferCreate":20,"TrustSet":10,"NFTokenMint":10,"EscrowCreate":10}'
+"""
+
+import argparse
+import asyncio
+import json
+import logging
+import random
+import sys
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import websockets
+
+logger = logging.getLogger("tx_submitter")
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+GENESIS_ACCOUNT = "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"
+GENESIS_SEED = "snoPBrXtMeMyMHUVTgbuqAfg1SUTb"
+
+# Amount to fund each test account (100,000 XRP in drops).
+FUND_AMOUNT = "100000000000"
+
+# Default transaction mix weights (relative).
+DEFAULT_TX_WEIGHTS: dict[str, int] = {
+    "Payment": 40,
+    "OfferCreate": 15,
+    "OfferCancel": 5,
+    "TrustSet": 10,
+    "NFTokenMint": 10,
+    "NFTokenCreateOffer": 5,
+    "EscrowCreate": 5,
+    "EscrowFinish": 5,
+    "AMMCreate": 3,
+    "AMMDeposit": 2,
+}
+
+# Number of test accounts to create.
+NUM_TEST_ACCOUNTS = 8
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class Account:
+    """Represents a funded XRPL test account.
+
+    Attributes:
+        name:      Human-readable name (e.g., "alice").
+        account:   Classic address (rXXX...).
+        seed:      Secret seed for signing.
+        sequence:  Next available sequence number.
+    """
+
+    name: str
+    account: str
+    seed: str
+    sequence: int = 0
+
+
+@dataclass
+class TxStats:
+    """Tracks transaction submission results.
+
+    Attributes:
+        total_submitted: Total transactions sent to the network.
+        total_success:   Transactions that returned tesSUCCESS or terQUEUED.
+        total_errors:    Transactions that returned an error engine_result.
+        by_type:         Per-transaction-type count of submissions.
+        errors_by_type:  Per-transaction-type count of errors.
+    """
+
+    total_submitted: int = 0
+    total_success: int = 0
+    total_errors: int = 0
+    by_type: dict[str, int] = field(default_factory=dict)
+    errors_by_type: dict[str, int] = field(default_factory=dict)
+
+    def record(self, tx_type: str, success: bool) -> None:
+        """Record the result of a transaction submission."""
+        self.total_submitted += 1
+        self.by_type[tx_type] = self.by_type.get(tx_type, 0) + 1
+        if success:
+            self.total_success += 1
+        else:
+            self.total_errors += 1
+            self.errors_by_type[tx_type] = self.errors_by_type.get(tx_type, 0) + 1
+
+    def summary(self) -> dict[str, Any]:
+        """Return a summary dict suitable for JSON serialization."""
+        return {
+            "total_submitted": self.total_submitted,
+            "total_success": self.total_success,
+            "total_errors": self.total_errors,
+            "success_rate_pct": (
+                round(self.total_success / self.total_submitted * 100, 2)
+                if self.total_submitted
+                else 0
+            ),
+            "by_type": self.by_type,
+            "errors_by_type": self.errors_by_type,
+        }
+
+
+# ---------------------------------------------------------------------------
+# WebSocket RPC helpers
+# ---------------------------------------------------------------------------
+
+
+async def ws_request(
+    ws: websockets.WebSocketClientProtocol,
+    command: str,
+    params: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Send a native WebSocket command and return the result payload.
+
+    Uses rippled's native WebSocket format (``command`` key with flat
+    parameters).  The response has ``status`` at the top level and the
+    actual data payload inside ``result``.  This helper unwraps the
+    ``result`` dict so callers can read fields directly.
+
+    Args:
+        ws:      Open WebSocket connection.
+        command: RPC command name (e.g., ``account_info``, ``submit``).
+        params:  Optional flat parameter dict merged into the request.
+
+    Returns:
+        The inner ``result`` dict from the response.
+
+    Raises:
+        RuntimeError: If the request fails or times out.
+    """
+    request: dict[str, Any] = {"command": command}
+    if params:
+        request.update(params)
+    await ws.send(json.dumps(request))
+    raw = await asyncio.wait_for(ws.recv(), timeout=30.0)
+    resp = json.loads(raw)
+
+    # WS command format: {"status": "success", "result": {...}, "type": "response"}
+    # On error: {"status": "error", "error": "...", "error_message": "..."}
+    if resp.get("status") == "error":
+        logger.warning(
+            "%s error: %s — %s",
+            command,
+            resp.get("error", "unknown"),
+            resp.get("error_message", ""),
+        )
+    return resp.get("result", resp)
+
+
+async def create_account(ws: websockets.WebSocketClientProtocol, name: str) -> Account:
+    """Create a new account via wallet_propose RPC.
+
+    Args:
+        ws:   Open WebSocket connection.
+        name: Human-readable name for the account.
+
+    Returns:
+        An Account instance with the generated keypair.
+    """
+    result = await ws_request(ws, "wallet_propose")
+    if "account_id" not in result:
+        raise RuntimeError(
+            f"wallet_propose failed: {json.dumps(result, indent=None)[:300]}"
+        )
+    return Account(
+        name=name,
+        account=result["account_id"],
+        seed=result["master_seed"],
+    )
+
+
+async def fund_account(
+    ws: websockets.WebSocketClientProtocol,
+    dest: Account,
+    genesis_seq: int,
+) -> tuple[bool, int]:
+    """Fund a test account from genesis.
+
+    Args:
+        ws:          Open WebSocket connection.
+        dest:        Destination account to fund.
+        genesis_seq: Current genesis account sequence number.
+
+    Returns:
+        Tuple of (success: bool, next_sequence: int).
+    """
+    resp = await ws_request(
+        ws,
+        "submit",
+        {
+            "secret": GENESIS_SEED,
+            "tx_json": {
+                "TransactionType": "Payment",
+                "Account": GENESIS_ACCOUNT,
+                "Destination": dest.account,
+                "Amount": FUND_AMOUNT,
+                "Sequence": genesis_seq,
+            },
+        },
+    )
+    engine_result = resp.get("engine_result", "unknown")
+    success = engine_result in ("tesSUCCESS", "terQUEUED")
+    if not success:
+        # Log the full response to help diagnose submit failures in CI.
+        logger.warning(
+            "Fund %s failed: engine_result=%s, full response: %s",
+            dest.name,
+            engine_result,
+            json.dumps(resp, indent=None)[:500],
+        )
+    return success, genesis_seq + 1
+
+
+async def get_account_sequence(
+    ws: websockets.WebSocketClientProtocol, account: str
+) -> int:
+    """Get the current sequence number for an account.
+
+    Args:
+        ws:      Open WebSocket connection.
+        account: Classic address.
+
+    Returns:
+        Current sequence number.
+    """
+    resp = await ws_request(ws, "account_info", {"account": account})
+    if "account_data" not in resp:
+        # Log full response to diagnose WS API format issues.
+        logger.warning(
+            "account_info for %s: no account_data, full response: %s",
+            account[:12],
+            json.dumps(resp, indent=None)[:500],
+        )
+        return 0
+    return resp["account_data"].get("Sequence", 0)
+
+
+# ---------------------------------------------------------------------------
+# Transaction builders
+# ---------------------------------------------------------------------------
+
+
+def build_payment(sender: Account, receiver: Account) -> dict[str, Any]:
+    """Build an XRP Payment transaction.
+
+    Args:
+        sender:   Source account.
+        receiver: Destination account.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    amount = str(random.randint(1000, 1000000))  # 0.001 - 1 XRP
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "Payment",
+            "Account": sender.account,
+            "Destination": receiver.account,
+            "Amount": amount,
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_offer_create(sender: Account) -> dict[str, Any]:
+    """Build an OfferCreate transaction (XRP/USD pair).
+
+    Args:
+        sender: Account placing the offer.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "OfferCreate",
+            "Account": sender.account,
+            "TakerPays": str(random.randint(100000, 10000000)),
+            "TakerGets": {
+                "currency": "USD",
+                "issuer": GENESIS_ACCOUNT,
+                "value": str(round(random.uniform(0.1, 100.0), 2)),
+            },
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_offer_cancel(sender: Account) -> dict[str, Any]:
+    """Build an OfferCancel transaction.
+
+    Uses a non-existent offer sequence — will fail gracefully but still
+    exercises the tx.process span pipeline.
+
+    Args:
+        sender: Account cancelling the offer.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "OfferCancel",
+            "Account": sender.account,
+            "OfferSequence": max(1, sender.sequence - 1),
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_trust_set(sender: Account) -> dict[str, Any]:
+    """Build a TrustSet transaction for a USD trust line.
+
+    Args:
+        sender: Account setting the trust line.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "TrustSet",
+            "Account": sender.account,
+            "LimitAmount": {
+                "currency": "USD",
+                "issuer": GENESIS_ACCOUNT,
+                "value": "1000000",
+            },
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_nftoken_mint(sender: Account) -> dict[str, Any]:
+    """Build an NFTokenMint transaction.
+
+    Args:
+        sender: Account minting the NFT.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "NFTokenMint",
+            "Account": sender.account,
+            "NFTokenTaxon": random.randint(0, 100),
+            "Flags": 8,  # tfTransferable
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_nftoken_create_offer(sender: Account) -> dict[str, Any]:
+    """Build an NFTokenCreateOffer transaction.
+
+    Uses a dummy NFTokenID — will fail but exercises the span pipeline.
+
+    Args:
+        sender: Account creating the NFT offer.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "NFTokenCreateOffer",
+            "Account": sender.account,
+            "NFTokenID": "0" * 64,
+            "Amount": str(random.randint(100000, 1000000)),
+            "Flags": 1,  # tfSellNFToken
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_escrow_create(sender: Account, receiver: Account) -> dict[str, Any]:
+    """Build an EscrowCreate transaction.
+
+    Creates a time-based escrow that finishes 10 seconds from now.
+
+    Args:
+        sender:   Account creating the escrow.
+        receiver: Destination account for escrow funds.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    # Ripple epoch offset: 946684800 seconds from Unix epoch
+    ripple_time = int(time.time()) - 946684800
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "EscrowCreate",
+            "Account": sender.account,
+            "Destination": receiver.account,
+            "Amount": str(random.randint(100000, 1000000)),
+            "FinishAfter": ripple_time + 10,
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_escrow_finish(sender: Account, owner: Account) -> dict[str, Any]:
+    """Build an EscrowFinish transaction.
+
+    Uses a dummy offer sequence — will likely fail but exercises spans.
+
+    Args:
+        sender: Account finishing the escrow.
+        owner:  Account that created the escrow.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "EscrowFinish",
+            "Account": sender.account,
+            "Owner": owner.account,
+            "OfferSequence": max(1, owner.sequence - 2),
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_amm_create(sender: Account) -> dict[str, Any]:
+    """Build an AMMCreate transaction (XRP/USD pool).
+
+    Requires the AMM amendment to be enabled on the network.
+
+    Args:
+        sender: Account creating the AMM pool.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "AMMCreate",
+            "Account": sender.account,
+            "Amount": str(random.randint(10000000, 100000000)),
+            "Amount2": {
+                "currency": "USD",
+                "issuer": GENESIS_ACCOUNT,
+                "value": str(round(random.uniform(10.0, 1000.0), 2)),
+            },
+            "TradingFee": 500,  # 0.5%
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+def build_amm_deposit(sender: Account) -> dict[str, Any]:
+    """Build an AMMDeposit transaction.
+
+    Args:
+        sender: Account depositing into the AMM pool.
+
+    Returns:
+        Transaction JSON and signing secret.
+    """
+    return {
+        "secret": sender.seed,
+        "tx_json": {
+            "TransactionType": "AMMDeposit",
+            "Account": sender.account,
+            "Asset": {"currency": "XRP"},
+            "Asset2": {
+                "currency": "USD",
+                "issuer": GENESIS_ACCOUNT,
+            },
+            "Amount": str(random.randint(1000000, 10000000)),
+            "Flags": 0x00080000,  # tfSingleAsset
+            "Sequence": sender.sequence,
+        },
+    }
+
+
+# Transaction type -> builder function mapping.
+# Each builder takes (accounts: list[Account]) and returns submit params.
+TX_BUILDERS: dict[str, Any] = {
+    "Payment": lambda accts: build_payment(accts[0], accts[1]),
+    "OfferCreate": lambda accts: build_offer_create(accts[0]),
+    "OfferCancel": lambda accts: build_offer_cancel(accts[0]),
+    "TrustSet": lambda accts: build_trust_set(accts[2]),
+    "NFTokenMint": lambda accts: build_nftoken_mint(accts[3]),
+    "NFTokenCreateOffer": lambda accts: build_nftoken_create_offer(accts[3]),
+    "EscrowCreate": lambda accts: build_escrow_create(accts[4], accts[1]),
+    "EscrowFinish": lambda accts: build_escrow_finish(accts[4], accts[4]),
+    "AMMCreate": lambda accts: build_amm_create(accts[5]),
+    "AMMDeposit": lambda accts: build_amm_deposit(accts[5]),
+}
+
+
+# ---------------------------------------------------------------------------
+# Main submission loop
+# ---------------------------------------------------------------------------
+
+
+async def setup_accounts(
+    ws: websockets.WebSocketClientProtocol,
+) -> list[Account]:
+    """Create and fund test accounts from genesis.
+
+    Generates NUM_TEST_ACCOUNTS accounts via wallet_propose, then funds
+    each with FUND_AMOUNT XRP from genesis.
+
+    Args:
+        ws: Open WebSocket connection to a rippled node.
+
+    Returns:
+        List of funded Account instances.
+    """
+    account_names = ["alice", "bob", "carol", "dave", "eve", "frank", "grace", "heidi"]
+
+    logger.info("Creating %d test accounts...", NUM_TEST_ACCOUNTS)
+    accounts: list[Account] = []
+    for name in account_names[:NUM_TEST_ACCOUNTS]:
+        acct = await create_account(ws, name)
+        accounts.append(acct)
+        logger.info("  Created %s: %s", name, acct.account)
+
+    # Get genesis sequence.
+    genesis_seq = await get_account_sequence(ws, GENESIS_ACCOUNT)
+    logger.info("Genesis sequence: %d", genesis_seq)
+
+    # Fund all accounts.
+    logger.info("Funding test accounts...")
+    for acct in accounts:
+        success, genesis_seq = await fund_account(ws, acct, genesis_seq)
+        if success:
+            logger.info("  Funded %s", acct.name)
+        else:
+            logger.warning("  Failed to fund %s", acct.name)
+
+    # Wait for funding transactions to be validated.
+    logger.info("Waiting 10s for funding transactions to validate...")
+    await asyncio.sleep(10)
+
+    # Refresh sequence numbers for all accounts.
+    for acct in accounts:
+        try:
+            acct.sequence = await get_account_sequence(ws, acct.account)
+            logger.info("  %s sequence: %d", acct.name, acct.sequence)
+        except Exception as exc:
+            logger.warning("  Failed to get sequence for %s: %s", acct.name, exc)
+
+    return accounts
+
+
+async def submit_transaction(
+    ws: websockets.WebSocketClientProtocol,
+    tx_type: str,
+    accounts: list[Account],
+    stats: TxStats,
+) -> None:
+    """Submit a single transaction of the given type.
+
+    Selects the appropriate builder, constructs the transaction, submits
+    it via the submit RPC, and records the result.
+
+    Args:
+        ws:       Open WebSocket connection.
+        tx_type:  Transaction type name (e.g., "Payment").
+        accounts: List of funded test accounts.
+        stats:    TxStats instance to record results.
+    """
+    builder = TX_BUILDERS.get(tx_type)
+    if not builder:
+        logger.warning("Unknown transaction type: %s", tx_type)
+        return
+
+    try:
+        params = builder(accounts)
+        # Identify which account is the sender to bump its sequence.
+        sender_addr = params["tx_json"]["Account"]
+        sender = next((a for a in accounts if a.account == sender_addr), None)
+
+        resp = await ws_request(ws, "submit", params)
+        engine_result = resp.get("engine_result", "unknown")
+        success = engine_result in (
+            "tesSUCCESS",
+            "terQUEUED",
+            "tecUNFUNDED_OFFER",
+            "tecNO_DST_INSUF_XRP",
+        )
+        stats.record(tx_type, success)
+
+        if sender:
+            sender.sequence += 1
+
+        if not success:
+            logger.debug(
+                "%s result: %s (%s)",
+                tx_type,
+                engine_result,
+                resp.get("engine_result_message", ""),
+            )
+    except Exception as exc:
+        stats.record(tx_type, False)
+        logger.debug("%s error: %s", tx_type, exc)
+
+
+async def _refresh_sequences(
+    ws: websockets.WebSocketClientProtocol,
+    accounts: list[Account],
+) -> None:
+    """Re-sync account sequences from the validated ledger.
+
+    In a consensus network, other nodes' transactions advance sequences
+    beyond the submitter's local tracking.  Refreshing every ~10 s keeps
+    the local counter close to the ledger and prevents tefPAST_SEQ storms.
+    """
+    for acct in accounts:
+        try:
+            seq = await get_account_sequence(ws, acct.account)
+            if seq > acct.sequence:
+                acct.sequence = seq
+        except Exception:
+            pass
+
+
+async def run_submitter(
+    endpoint: str,
+    tps: float,
+    duration: float,
+    weights: dict[str, int],
+) -> TxStats:
+    """Run the transaction submitter against a single endpoint.
+
+    Args:
+        endpoint: WebSocket URL (ws://host:port).
+        tps:      Target transactions per second.
+        duration: Total run time in seconds.
+        weights:  Transaction type distribution weights.
+
+    Returns:
+        TxStats with aggregated results.
+    """
+    stats = TxStats()
+    interval = 1.0 / tps if tps > 0 else 0.5
+
+    ws = await websockets.connect(endpoint, ping_interval=20, ping_timeout=10)
+    logger.info("Connected to %s", endpoint)
+
+    try:
+        # Setup test accounts.
+        accounts = await setup_accounts(ws)
+        if len(accounts) < 6:
+            logger.error("Need at least 6 funded accounts, got %d", len(accounts))
+            return stats
+
+        # Build weighted command list.
+        tx_types = list(weights.keys())
+        tx_weights = [weights[t] for t in tx_types]
+
+        logger.info(
+            "Starting TX submission: tps=%s, duration=%ss, types=%d",
+            tps,
+            duration,
+            len(tx_types),
+        )
+
+        start = time.monotonic()
+        last_seq_refresh = start
+        seq_refresh_interval = 10.0
+        while (time.monotonic() - start) < duration:
+            # Periodically re-sync account sequences from the ledger so
+            # locally-tracked sequences don't drift behind consensus.
+            if (time.monotonic() - last_seq_refresh) >= seq_refresh_interval:
+                await _refresh_sequences(ws, accounts)
+                last_seq_refresh = time.monotonic()
+
+            tx_type = random.choices(tx_types, weights=tx_weights, k=1)[0]
+            await submit_transaction(ws, tx_type, accounts, stats)
+            await asyncio.sleep(interval)
+
+            # Progress logging every 50 transactions.
+            if stats.total_submitted % 50 == 0 and stats.total_submitted > 0:
+                elapsed = time.monotonic() - start
+                actual_tps = stats.total_submitted / elapsed if elapsed > 0 else 0
+                logger.info(
+                    "Progress: %d submitted, %d success, %d errors, "
+                    "%.1f TPS (%.0fs elapsed)",
+                    stats.total_submitted,
+                    stats.total_success,
+                    stats.total_errors,
+                    actual_tps,
+                    elapsed,
+                )
+
+    finally:
+        await ws.close()
+
+    elapsed = time.monotonic() - start
+    logger.info(
+        "Submission complete: %d submitted, %d success, %d errors "
+        "in %.1fs (%.1f TPS)",
+        stats.total_submitted,
+        stats.total_success,
+        stats.total_errors,
+        elapsed,
+        stats.total_submitted / elapsed if elapsed > 0 else 0,
+    )
+
+    return stats
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Transaction Submitter for rippled telemetry validation",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Basic usage (5 TPS for 2 minutes):
+  python3 tx_submitter.py --endpoint ws://localhost:6006 --tps 5 --duration 120
+
+  # Custom transaction mix:
+  python3 tx_submitter.py --endpoint ws://localhost:6006 \\
+      --weights '{"Payment": 60, "OfferCreate": 20, "TrustSet": 20}'
+        """,
+    )
+    parser.add_argument(
+        "--endpoint",
+        type=str,
+        default="ws://localhost:6006",
+        help="WebSocket endpoint (default: ws://localhost:6006)",
+    )
+    parser.add_argument(
+        "--tps",
+        type=float,
+        default=5.0,
+        help="Target transactions per second (default: 5)",
+    )
+    parser.add_argument(
+        "--duration",
+        type=float,
+        default=120.0,
+        help="Run duration in seconds (default: 120)",
+    )
+    parser.add_argument(
+        "--weights",
+        type=str,
+        default=None,
+        help="JSON string of transaction type weights (overrides defaults)",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default=None,
+        help="Write JSON summary to this file path",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable debug logging",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Main entry point for the transaction submitter."""
+    args = parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s [%(name)s] %(levelname)s %(message)s",
+    )
+
+    # Parse custom weights if provided.
+    weights = DEFAULT_TX_WEIGHTS.copy()
+    if args.weights:
+        try:
+            custom = json.loads(args.weights)
+            weights = {k: int(v) for k, v in custom.items()}
+            logger.info("Using custom weights: %s", weights)
+        except (json.JSONDecodeError, ValueError) as exc:
+            logger.error("Invalid --weights JSON: %s", exc)
+            sys.exit(1)
+
+    # Run the submitter.
+    stats = asyncio.run(
+        run_submitter(
+            endpoint=args.endpoint,
+            tps=args.tps,
+            duration=args.duration,
+            weights=weights,
+        )
+    )
+
+    summary = stats.summary()
+    print(json.dumps(summary, indent=2))
+
+    if args.output:
+        with open(args.output, "w") as f:
+            json.dump(summary, f, indent=2)
+        logger.info("Summary written to %s", args.output)
+
+
+if __name__ == "__main__":
+    main()

docker/telemetry/workload/workload-profiles.json

@@ -0,0 +1,108 @@
+{
+  "profiles": {
+    "full-validation": {
+      "description": "Full 18-dashboard coverage with burst/idle/plateau patterns",
+      "phases": [
+        {
+          "name": "warmup",
+          "description": "Low load to populate baseline gauges and node health metrics",
+          "duration_sec": 30,
+          "rpc": {
+            "rate": 5,
+            "weights": { "server_info": 50, "fee": 30, "ledger": 20 }
+          },
+          "tx": null
+        },
+        {
+          "name": "steady-state",
+          "description": "Medium sustained load — plateau data for all dashboards",
+          "duration_sec": 60,
+          "rpc": { "rate": 30 },
+          "tx": { "tps": 3 }
+        },
+        {
+          "name": "rpc-burst",
+          "description": "Heavy RPC to saturate job queue and spike latency",
+          "duration_sec": 30,
+          "rpc": { "rate": 100 },
+          "tx": null
+        },
+        {
+          "name": "tx-flood",
+          "description": "High TX rate for fee escalation and TxQ pressure",
+          "duration_sec": 30,
+          "rpc": { "rate": 5, "weights": { "server_info": 50, "fee": 50 } },
+          "tx": {
+            "tps": 20,
+            "weights": { "Payment": 70, "OfferCreate": 20, "TrustSet": 10 }
+          }
+        },
+        {
+          "name": "txq-burst",
+          "description": "Single-type Payment burst at high TPS to force open-ledger fee escalation and TxQ queueing, exercising the txq.* spans (txq.enqueue / txq.accept / txq.accept_tx / txq.cleanup)",
+          "duration_sec": 30,
+          "rpc": { "rate": 5, "weights": { "fee": 100 } },
+          "tx": {
+            "tps": 60,
+            "weights": { "Payment": 100 }
+          }
+        },
+        {
+          "name": "mixed-peak",
+          "description": "Realistic peak load — consensus and ledger ops under stress",
+          "duration_sec": 60,
+          "rpc": { "rate": 50 },
+          "tx": { "tps": 10 }
+        },
+        {
+          "name": "cooldown",
+          "description": "Low load for recovery metrics and state transition data",
+          "duration_sec": 30,
+          "rpc": { "rate": 5, "weights": { "server_info": 80, "fee": 20 } },
+          "tx": null
+        }
+      ],
+      "propagation_wait_sec": 60
+    },
+    "quick-smoke": {
+      "description": "Fast smoke test — minimal data for CI quick checks",
+      "phases": [
+        {
+          "name": "smoke",
+          "description": "Single phase covering all generator types",
+          "duration_sec": 30,
+          "rpc": { "rate": 20 },
+          "tx": { "tps": 3 }
+        }
+      ],
+      "propagation_wait_sec": 30
+    },
+    "stress": {
+      "description": "Heavy sustained load for performance benchmarking",
+      "phases": [
+        {
+          "name": "ramp-up",
+          "description": "Gradually increasing load",
+          "duration_sec": 30,
+          "rpc": { "rate": 20 },
+          "tx": { "tps": 5 }
+        },
+        {
+          "name": "peak",
+          "description": "Maximum sustained load",
+          "duration_sec": 120,
+          "rpc": { "rate": 150 },
+          "tx": { "tps": 25 }
+        },
+        {
+          "name": "sustain",
+          "description": "Continued high load for stability check",
+          "duration_sec": 60,
+          "rpc": { "rate": 100 },
+          "tx": { "tps": 15 }
+        }
+      ],
+      "propagation_wait_sec": 60
+    }
+  }
+}

docker/telemetry/workload/workload_orchestrator.py

@@ -0,0 +1,506 @@
+#!/usr/bin/env python3
+"""Workload Orchestrator for rippled telemetry validation.
+
+Reads a named profile from workload-profiles.json and executes sequential
+load phases, each with configurable RPC and TX parameters.  Produces a
+combined report with per-phase results.
+
+Phases run sequentially.  Within each phase, the RPC load generator and
+transaction submitter run concurrently (if both are configured).
+
+Orchestration Flow::
+
+    workload-profiles.json
+           |
+           v
+    workload_orchestrator.py
+           |
+      +----+----+----+----+----+----+
+      | Phase 1 | Phase 2 | ...... | Phase N |
+      +----+----+----+----+----+----+
+           |              |
+      +----+----+    +----+----+
+      | rpc_load |    | tx_sub  |   (concurrent within phase)
+      | _gen.py  |    | mitter  |
+      +----+----+    +----+----+
+           |              |
+           v              v
+      per-phase JSON reports
+           |
+           v
+      combined-report.json
+
+Usage:
+    python3 workload_orchestrator.py --profile full-validation
+    python3 workload_orchestrator.py --profile quick-smoke --endpoints ws://localhost:6006
+    python3 workload_orchestrator.py --profile stress --report /tmp/report.json
+
+Profiles are defined in workload-profiles.json in the same directory.
+"""
+
+import argparse
+import asyncio
+import json
+import logging
+import sys
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("workload_orchestrator")
+
+SCRIPT_DIR = Path(__file__).parent.resolve()
+PROFILES_FILE = SCRIPT_DIR / "workload-profiles.json"
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class PhaseResult:
+    """Result of a single workload phase.
+
+    Attributes:
+        name:         Phase name from the profile.
+        duration_sec: Configured duration.
+        actual_sec:   Actual elapsed time.
+        rpc_summary:  JSON summary from rpc_load_generator, or None.
+        tx_summary:   JSON summary from tx_submitter, or None.
+        errors:       List of error messages from subprocess failures.
+    """
+
+    name: str
+    duration_sec: int
+    actual_sec: float = 0.0
+    rpc_summary: dict[str, Any] | None = None
+    tx_summary: dict[str, Any] | None = None
+    errors: list[str] = field(default_factory=list)
+
+
+# ---------------------------------------------------------------------------
+# Profile loading
+# ---------------------------------------------------------------------------
+
+
+def load_profile(profile_name: str) -> dict[str, Any]:
+    """Load a named profile from workload-profiles.json.
+
+    Args:
+        profile_name: Key in the profiles dict (e.g., "full-validation").
+
+    Returns:
+        The profile dict with phases and propagation_wait_sec.
+
+    Raises:
+        SystemExit: If the profile file or name is not found.
+    """
+    if not PROFILES_FILE.exists():
+        logger.error("Profiles file not found: %s", PROFILES_FILE)
+        sys.exit(2)
+
+    with open(PROFILES_FILE) as f:
+        data = json.load(f)
+
+    profiles = data.get("profiles", {})
+    if profile_name not in profiles:
+        available = ", ".join(profiles.keys())
+        logger.error("Profile '%s' not found. Available: %s", profile_name, available)
+        sys.exit(2)
+
+    profile = profiles[profile_name]
+
+    # Validate profile schema — fail fast on bad config.
+    phases = profile.get("phases", [])
+    if not isinstance(phases, list) or not phases:
+        logger.error("Profile '%s' has no valid phases", profile_name)
+        sys.exit(2)
+    for i, phase in enumerate(phases):
+        if not isinstance(phase.get("name"), str):
+            logger.error("Phase %d missing valid 'name'", i)
+            sys.exit(2)
+        if (
+            not isinstance(phase.get("duration_sec"), (int, float))
+            or phase["duration_sec"] <= 0
+        ):
+            logger.error(
+                "Phase %d '%s' has invalid duration_sec",
+                i,
+                phase.get("name"),
+            )
+            sys.exit(2)
+
+    logger.info(
+        "Loaded profile '%s': %s (%d phases)",
+        profile_name,
+        profile.get("description", ""),
+        len(phases),
+    )
+    return profile
+
+
+# ---------------------------------------------------------------------------
+# Subprocess execution
+# ---------------------------------------------------------------------------
+
+
+async def run_subprocess(cmd: list[str], label: str) -> tuple[int, str, str]:
+    """Run a subprocess and capture its stdout and stderr.
+
+    Args:
+        cmd:   Command and arguments.
+        label: Human-readable label for logging.
+
+    Returns:
+        Tuple of (return_code, stdout_text, stderr_text).
+    """
+    logger.debug("Starting %s: %s", label, " ".join(cmd))
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate()
+    if proc.returncode != 0:
+        logger.warning(
+            "%s exited with code %d: %s",
+            label,
+            proc.returncode,
+            stderr.decode().strip()[-500:],
+        )
+    return proc.returncode, stdout.decode(), stderr.decode()
+
+
+# ---------------------------------------------------------------------------
+# Phase execution
+# ---------------------------------------------------------------------------
+
+
+def _collect_task_result(
+    label: str,
+    returncode: int,
+    stderr: str,
+    report_path: Path,
+    result: PhaseResult,
+) -> None:
+    """Process the result of a completed subprocess task.
+
+    Reads the JSON report file (if it exists) and records any errors.
+
+    Args:
+        label:       "rpc" or "tx".
+        returncode:  Subprocess exit code.
+        stderr:      Captured stderr text.
+        report_path: Path to the JSON report file.
+        result:      PhaseResult to update.
+    """
+    if report_path.exists():
+        try:
+            with open(report_path) as f:
+                summary = json.load(f)
+            if label == "rpc":
+                result.rpc_summary = summary
+            elif label == "tx":
+                result.tx_summary = summary
+        except (json.JSONDecodeError, OSError) as exc:
+            logger.warning("Failed to parse %s report %s: %s", label, report_path, exc)
+            result.errors.append(f"Failed to parse {label} report: {exc}")
+
+    if returncode != 0:
+        snippet = stderr.strip()[-200:] if stderr else ""
+        result.errors.append(
+            f"{label.upper()} generator exited with {returncode}: {snippet}"
+        )
+
+
+def _build_rpc_cmd(
+    endpoints: list[str], rpc_cfg: dict[str, Any], duration: int, output: Path
+) -> list[str]:
+    """Build the command list for the RPC load generator subprocess."""
+    cmd = [
+        sys.executable,
+        str(SCRIPT_DIR / "rpc_load_generator.py"),
+        "--endpoints",
+        *endpoints,
+        "--rate",
+        str(rpc_cfg.get("rate", 50)),
+        "--duration",
+        str(duration),
+        "--output",
+        str(output),
+    ]
+    weights = rpc_cfg.get("weights")
+    if weights:
+        cmd.extend(["--weights", json.dumps(weights)])
+    return cmd
+
+
+def _build_tx_cmd(
+    endpoint: str, tx_cfg: dict[str, Any], duration: int, output: Path
+) -> list[str]:
+    """Build the command list for the TX submitter subprocess."""
+    cmd = [
+        sys.executable,
+        str(SCRIPT_DIR / "tx_submitter.py"),
+        "--endpoint",
+        endpoint,
+        "--tps",
+        str(tx_cfg.get("tps", 5)),
+        "--duration",
+        str(duration),
+        "--output",
+        str(output),
+    ]
+    weights = tx_cfg.get("weights")
+    if weights:
+        cmd.extend(["--weights", json.dumps(weights)])
+    return cmd
+
+
+async def run_phase(
+    phase: dict[str, Any],
+    endpoints: list[str],
+    report_dir: Path,
+    phase_idx: int,
+) -> PhaseResult:
+    """Execute a single workload phase.
+
+    Launches rpc_load_generator.py and/or tx_submitter.py as subprocesses
+    based on the phase configuration.  Both run concurrently if configured.
+
+    Args:
+        phase:      Phase dict from the profile.
+        endpoints:  List of WebSocket endpoint URLs.
+        report_dir: Directory for per-phase JSON reports.
+        phase_idx:  Phase index (for file naming).
+
+    Returns:
+        PhaseResult with subprocess outputs.
+    """
+    name = phase["name"]
+    duration = phase["duration_sec"]
+    result = PhaseResult(name=name, duration_sec=duration)
+    prefix = f"phase{phase_idx + 1}-{name}"
+
+    logger.info(
+        "=== Phase %d: %s (%ds) — %s ===",
+        phase_idx + 1,
+        name,
+        duration,
+        phase.get("description", ""),
+    )
+
+    tasks: list[tuple[str, Path, asyncio.Task]] = []
+    t0 = time.monotonic()
+
+    rpc_cfg = phase.get("rpc")
+    if rpc_cfg:
+        rpc_out = report_dir / f"{prefix}-rpc.json"
+        cmd = _build_rpc_cmd(endpoints, rpc_cfg, duration, rpc_out)
+        tasks.append(
+            ("rpc", rpc_out, asyncio.create_task(run_subprocess(cmd, f"RPC [{name}]")))
+        )
+
+    tx_cfg = phase.get("tx")
+    if tx_cfg:
+        tx_out = report_dir / f"{prefix}-tx.json"
+        cmd = _build_tx_cmd(endpoints[0], tx_cfg, duration, tx_out)
+        tasks.append(
+            ("tx", tx_out, asyncio.create_task(run_subprocess(cmd, f"TX [{name}]")))
+        )
+
+    if not tasks:
+        logger.warning(
+            "Phase %d: %s — no workload configured, skipping", phase_idx + 1, name
+        )
+        return result
+
+    for label, report_path, task in tasks:
+        returncode, _stdout, stderr = await task
+        _collect_task_result(label, returncode, stderr, report_path, result)
+
+    result.actual_sec = time.monotonic() - t0
+    logger.info(
+        "Phase %d complete: %.1fs actual, %d errors",
+        phase_idx + 1,
+        result.actual_sec,
+        len(result.errors),
+    )
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Profile execution
+# ---------------------------------------------------------------------------
+
+
+async def run_profile(
+    profile: dict[str, Any],
+    endpoints: list[str],
+    report_dir: Path,
+) -> dict[str, Any]:
+    """Execute all phases in a profile sequentially.
+
+    Args:
+        profile:    Profile dict with phases and propagation_wait_sec.
+        endpoints:  WebSocket endpoints for the rippled cluster.
+        report_dir: Directory for phase reports.
+
+    Returns:
+        Combined report dict with per-phase results and totals.
+    """
+    phases = profile.get("phases", [])
+    propagation_wait = profile.get("propagation_wait_sec", 60)
+    results: list[PhaseResult] = []
+
+    total_start = time.monotonic()
+
+    for idx, phase in enumerate(phases):
+        result = await run_phase(phase, endpoints, report_dir, idx)
+        results.append(result)
+
+    # Wait for telemetry data to propagate through the collector pipeline.
+    logger.info("Waiting %ds for telemetry data to propagate...", propagation_wait)
+    await asyncio.sleep(propagation_wait)
+
+    total_elapsed = time.monotonic() - total_start
+
+    # Build combined report from all phase results.
+    total_rpc_sent = 0
+    total_rpc_errors = 0
+    total_tx_submitted = 0
+    total_tx_errors = 0
+    phase_reports = []
+
+    for r in results:
+        pr: dict[str, Any] = {
+            "name": r.name,
+            "duration_sec": r.duration_sec,
+            "actual_sec": round(r.actual_sec, 1),
+            "errors": r.errors,
+        }
+        if r.rpc_summary:
+            pr["rpc"] = r.rpc_summary
+            total_rpc_sent += r.rpc_summary.get("total_sent", 0)
+            total_rpc_errors += r.rpc_summary.get("total_errors", 0)
+        if r.tx_summary:
+            pr["tx"] = r.tx_summary
+            total_tx_submitted += r.tx_summary.get("total_submitted", 0)
+            total_tx_errors += r.tx_summary.get("total_errors", 0)
+        phase_reports.append(pr)
+
+    report = {
+        "profile": profile.get("description", ""),
+        "total_elapsed_sec": round(total_elapsed, 1),
+        "phases": phase_reports,
+        "totals": {
+            "rpc_sent": total_rpc_sent,
+            "rpc_errors": total_rpc_errors,
+            "tx_submitted": total_tx_submitted,
+            "tx_errors": total_tx_errors,
+        },
+    }
+
+    return report
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Workload Orchestrator for rippled telemetry validation",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Profiles:
+  full-validation  Full 18-dashboard coverage (~5 min load + 1 min propagation)
+  quick-smoke      Fast CI smoke test (~30s load + 30s propagation)
+  stress           Heavy sustained load for benchmarking (~3.5 min + 1 min)
+
+Examples:
+  python3 workload_orchestrator.py --profile full-validation
+  python3 workload_orchestrator.py --profile quick-smoke --endpoints ws://localhost:6006
+  python3 workload_orchestrator.py --profile stress --report /tmp/report.json
+        """,
+    )
+    parser.add_argument(
+        "--profile",
+        type=str,
+        required=True,
+        help="Named profile from workload-profiles.json",
+    )
+    parser.add_argument(
+        "--endpoints",
+        nargs="+",
+        default=["ws://localhost:6006"],
+        help="WebSocket endpoints (default: ws://localhost:6006)",
+    )
+    parser.add_argument(
+        "--report",
+        type=str,
+        default=None,
+        help="Write combined JSON report to this file",
+    )
+    parser.add_argument(
+        "--report-dir",
+        type=str,
+        default="/tmp/xrpld-validation/reports",
+        help="Directory for per-phase reports",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable debug logging",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Main entry point for the workload orchestrator."""
+    args = parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s [%(name)s] %(levelname)s %(message)s",
+    )
+
+    profile = load_profile(args.profile)
+    report_dir = Path(args.report_dir)
+    report_dir.mkdir(parents=True, exist_ok=True)
+
+    report = asyncio.run(run_profile(profile, args.endpoints, report_dir))
+
+    print(json.dumps(report, indent=2))
+
+    if args.report:
+        with open(args.report, "w") as f:
+            json.dump(report, f, indent=2)
+        logger.info("Combined report written to %s", args.report)
+
+    # Exit with error if either generator had high error rates.
+    totals = report["totals"]
+    rpc_err_rate = (
+        totals["rpc_errors"] / totals["rpc_sent"] * 100 if totals["rpc_sent"] > 0 else 0
+    )
+    tx_err_rate = (
+        totals["tx_errors"] / totals["tx_submitted"] * 100
+        if totals["tx_submitted"] > 0
+        else 0
+    )
+    # TX threshold is higher because short-lived CI test environments lack
+    # pre-funded accounts, causing expected failures for complex transactions
+    # (AMMCreate, EscrowFinish, etc.) that require specific ledger state.
+    if rpc_err_rate > 50 or tx_err_rate > 95:
+        logger.error(
+            "High error rates: RPC=%.1f%%, TX=%.1f%%", rpc_err_rate, tx_err_rate
+        )
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

docker/telemetry/workload/xrpld-validator.cfg.template

@@ -0,0 +1,96 @@
+# xrpld validator node configuration template for workload harness.
+#
+# Placeholders (replaced by docker-compose entrypoint):
+#   {{NODE_INDEX}}        — Node number (1-based)
+#   {{RPC_PORT}}          — HTTP RPC port
+#   {{WS_PORT}}           — WebSocket port
+#   {{PEER_PORT}}         — Peer protocol port
+#   {{DATA_DIR}}          — Node data directory
+#   {{VALIDATION_SEED}}   — Validator seed from key generation
+#   {{VALIDATORS_FILE}}   — Path to shared validators.txt
+#   {{IPS_FIXED}}         — Peer addresses (one per line)
+#   {{OTEL_ENDPOINT}}     — OTel Collector OTLP/HTTP traces endpoint
+#   {{OTEL_METRICS_ENDPOINT}} — OTel Collector OTLP/HTTP metrics endpoint
+#   {{LOG_LEVEL}}         — Log level (debug, info, warning, error)
+
+[server]
+port_rpc
+port_ws
+port_peer
+
+[port_rpc]
+port = {{RPC_PORT}}
+ip = 0.0.0.0
+admin = 0.0.0.0
+protocol = http
+
+[port_ws]
+port = {{WS_PORT}}
+ip = 0.0.0.0
+admin = 0.0.0.0
+protocol = ws
+
+[port_peer]
+port = {{PEER_PORT}}
+ip = 0.0.0.0
+protocol = peer
+
+[node_db]
+type=NuDB
+path={{DATA_DIR}}/nudb
+online_delete=256
+
+[database_path]
+{{DATA_DIR}}/db
+
+[debug_logfile]
+{{DATA_DIR}}/debug.log
+
+[validation_seed]
+{{VALIDATION_SEED}}
+
+[validators_file]
+{{VALIDATORS_FILE}}
+
+[ips_fixed]
+{{IPS_FIXED}}
+
+[peer_private]
+1
+
+# --- OpenTelemetry tracing (all categories enabled) ---
+[telemetry]
+enabled=1
+service_instance_id=validator-{{NODE_INDEX}}
+endpoint={{OTEL_ENDPOINT}}
+exporter=otlp_http
+batch_size=512
+batch_delay_ms=2000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1
+
+# --- Native OTel metrics (beast::insight over OTLP/HTTP) ---
+# The collector has no StatsD receiver (metrics pipeline is [otlp, spanmetrics]),
+# so beast::insight exports natively over OTLP. prefix=xrpld matches the OTel
+# resource service name and the metric names the dashboards query.
+[insight]
+server=otel
+endpoint={{OTEL_METRICS_ENDPOINT}}
+prefix=xrpld
+
+[rpc_startup]
+{ "command": "log_level", "severity": "{{LOG_LEVEL}}" }
+
+[ssl_verify]
+0
+
+# --- Network tuning for local cluster ---
+[network_id]
+0
+
+[sntp_servers]
+time.google.com

docker/telemetry/xrpld-telemetry-mainnet.cfg

@@ -0,0 +1,141 @@
+# xrpld configuration for Mainnet with full OpenTelemetry tracing.
+#
+# Connects to XRP Ledger Mainnet as an observer-only tracking node
+# (no validator credentials) and exercises ALL instrumented workflows:
+# RPC, transactions, consensus, peer overlay, ledger ops, and
+# pathfinding.
+#
+# Usage:
+#   1. Start the observability stack:
+#        docker compose -f docker/telemetry/docker-compose.yml up -d
+#   2. Run xrpld:
+#        ./xrpld --conf docker/telemetry/xrpld-telemetry.cfg
+#   3. Wait for sync (server_state=full), then exercise workflows:
+#        curl -s http://localhost:5005 -d '{"method":"server_info"}'
+#   4. View traces in Grafana Explore -> Tempo: http://localhost:3000
+
+# --- Server ports -----------------------------------------------------------
+
+[server]
+port_rpc_admin_local
+port_ws_admin_local
+port_ws_public
+port_peer
+
+[port_rpc_admin_local]
+port = 5005
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[port_ws_admin_local]
+port = 6006
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = ws
+
+[port_ws_public]
+port = 6005
+ip = 0.0.0.0
+protocol = ws
+
+[port_peer]
+port = 51235
+ip = 0.0.0.0
+protocol = peer
+
+# --- Network ----------------------------------------------------------------
+
+[network_id]
+main
+
+# [ips] omitted on purpose. xrpld uses its built-in mainnet hub list
+# (r.ripple.com, sahyadri.isrdc.in, hubs.xrpkuwait.com,
+# hub.xrpl-commons.org) for peer discovery.
+
+[validators_file]
+validators-mainnet.txt
+
+[peer_private]
+0
+
+[peers_max]
+21
+
+# --- Pathfinding (exercises ripple_path_find / path_find workflows) ---------
+
+[path_search]
+7
+
+[path_search_fast]
+2
+
+[path_search_max]
+10
+
+# --- Signing (allows sign/sign_for RPC for test tx submission) --------------
+
+[signing_support]
+true
+
+# --- Database ---------------------------------------------------------------
+
+[node_db]
+type=NuDB
+path=docker/telemetry/data/nudb
+online_delete=2000
+advisory_delete=0
+
+[database_path]
+docker/telemetry/data
+
+[ledger_history]
+1000
+
+# --- Logging ----------------------------------------------------------------
+
+# Path is resolved relative to this config file's directory (docker/telemetry),
+# so this writes to docker/telemetry/data/logs/mainnet/debug.log — the same
+# dir the compose stack bind-mounts into the collector as /var/log/xrpld.
+[debug_logfile]
+data/logs/mainnet/debug.log
+
+[rpc_startup]
+{ "command": "log_level", "severity": "warning" }
+
+# --- SSL --------------------------------------------------------------------
+
+[ssl_verify]
+0
+
+# --- Insight (native OTel metrics via beast::insight) -----------------------
+
+[insight]
+server=otel
+endpoint=http://localhost:4318/v1/metrics
+prefix=xrpld
+# Sets the OTel service.instance.id resource attribute, which Prometheus
+# exposes as the `service_instance_id` label. Dashboards filter on it via the
+# $node template variable, so without this every insight-backed panel is
+# empty. Matches [telemetry] service_instance_id for a single node identity.
+service_instance_id=xrpld-mainnet
+
+# --- OpenTelemetry tracing --------------------------------------------------
+
+[telemetry]
+enabled=1
+service_instance_id=xrpld-mainnet
+endpoint=http://localhost:4318/v1/traces
+metrics_endpoint=http://localhost:4318/v1/metrics
+exporter=otlp_http
+# Mainnet has high span throughput across peer/ledger/consensus. Head
+# sampling is fixed at 1.0 (sample everything) and not configurable;
+# reduce Tempo/collector load with collector-side tail sampling.
+batch_size=512
+batch_delay_ms=5000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1

docker/telemetry/xrpld-telemetry.cfg

@@ -0,0 +1,136 @@
+# xrpld configuration for Devnet with full OpenTelemetry tracing.
+#
+# Connects to the XRP Ledger Devnet and exercises ALL instrumented
+# workflows: RPC, transactions, consensus, peer overlay, ledger ops,
+# and pathfinding.
+#
+# Usage:
+#   1. Start the observability stack:
+#        docker compose -f docker/telemetry/docker-compose.yml up -d
+#   2. Run xrpld:
+#        ./xrpld --conf docker/telemetry/xrpld-telemetry.cfg
+#   3. Wait for sync (server_state=full), then exercise workflows:
+#        curl -s http://localhost:5005 -d '{"method":"server_info"}'
+#   4. View traces in Grafana Explore -> Tempo: http://localhost:3000
+
+# --- Server ports -----------------------------------------------------------
+
+[server]
+port_rpc_admin_local
+port_ws_admin_local
+port_ws_public
+port_peer
+
+[port_rpc_admin_local]
+port = 5005
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = http
+
+[port_ws_admin_local]
+port = 6006
+ip = 127.0.0.1
+admin = 127.0.0.1
+protocol = ws
+
+[port_ws_public]
+port = 6005
+ip = 0.0.0.0
+protocol = ws
+
+[port_peer]
+port = 51235
+ip = 0.0.0.0
+protocol = peer
+
+# --- Network ----------------------------------------------------------------
+
+[network_id]
+devnet
+
+[ips]
+s.devnet.rippletest.net 51235
+
+[validators_file]
+validators-devnet.txt
+
+[peer_private]
+0
+
+[peers_max]
+21
+
+# --- Pathfinding (exercises ripple_path_find / path_find workflows) ---------
+
+[path_search]
+7
+
+[path_search_fast]
+2
+
+[path_search_max]
+10
+
+# --- Signing (allows sign/sign_for RPC for test tx submission) --------------
+
+[signing_support]
+true
+
+# --- Database ---------------------------------------------------------------
+
+[node_db]
+type=NuDB
+path=docker/telemetry/data/nudb
+online_delete=2000
+advisory_delete=0
+
+[database_path]
+docker/telemetry/data
+
+[ledger_history]
+1000
+
+# --- Logging ----------------------------------------------------------------
+
+# Path is resolved relative to this config file's directory (docker/telemetry),
+# so this writes to docker/telemetry/data/logs/devnet/debug.log — the same
+# dir the compose stack bind-mounts into the collector as /var/log/xrpld.
+[debug_logfile]
+data/logs/devnet/debug.log
+
+[rpc_startup]
+{ "command": "log_level", "severity": "debug" }
+
+# --- SSL --------------------------------------------------------------------
+
+[ssl_verify]
+0
+
+# --- Insight (native OTel metrics via beast::insight) -----------------------
+
+[insight]
+server=otel
+endpoint=http://localhost:4318/v1/metrics
+prefix=xrpld
+# Sets the OTel service.instance.id resource attribute, which Prometheus
+# exposes as the `service_instance_id` label. Dashboards filter on it via the
+# $node template variable, so without this every insight-backed panel is
+# empty. Matches [telemetry] service_instance_id for a single node identity.
+service_instance_id=xrpld-devnet
+
+# --- OpenTelemetry tracing --------------------------------------------------
+
+[telemetry]
+enabled=1
+service_instance_id=xrpld-devnet
+endpoint=http://localhost:4318/v1/traces
+metrics_endpoint=http://localhost:4318/v1/metrics
+exporter=otlp_http
+batch_size=512
+batch_delay_ms=5000
+max_queue_size=2048
+trace_rpc=1
+trace_transactions=1
+trace_consensus=1
+trace_peer=1
+trace_ledger=1

include/xrpl/beast/insight/OTelCollector.h

@@ -0,0 +1,135 @@
+#pragma once
+
+/**
+ * @file OTelCollector.h
+ * @brief OpenTelemetry-based implementation of the beast::insight::Collector
+ *        interface for native OTLP metric export.
+ *
+ * When XRPL_ENABLE_TELEMETRY is defined, OTelCollector maps each
+ * beast::insight instrument type (Counter, Gauge, Event, Meter, Hook) to
+ * the corresponding OpenTelemetry Metrics SDK instrument and exports
+ * them via OTLP/HTTP to an OpenTelemetry Collector.
+ *
+ * When XRPL_ENABLE_TELEMETRY is NOT defined, OTelCollector::New() returns
+ * a NullCollector so the binary compiles without OTel dependencies.
+ *
+ * Dependency diagram:
+ *
+ *   +-----------------+     +-------------------+
+ *   | Collector (ABC) |<----| OTelCollector     |
+ *   +-----------------+     | (public header)   |
+ *         ^                 +-------------------+
+ *         |                          |
+ *   +-----------------+     +-------------------+
+ *   | NullCollector   |     | OTelCollectorImp  |
+ *   | (fallback when  |     | (impl in .cpp,    |
+ *   |  no telemetry)  |     |  uses OTel SDK)   |
+ *   +-----------------+     +-------------------+
+ *                                    |
+ *                           +-------------------+
+ *                           | OTel Metrics SDK  |
+ *                           | MeterProvider     |
+ *                           | OTLP HTTP Metric  |
+ *                           | Exporter          |
+ *                           +-------------------+
+ */
+
+#include <xrpl/beast/insight/Collector.h>
+#include <xrpl/beast/utility/Journal.h>
+
+#include <memory>
+#include <string>
+
+namespace beast::insight {
+
+/**
+ * @brief A Collector that exports metrics via OpenTelemetry OTLP/HTTP.
+ *
+ * Replaces StatsD-based metric collection with native OTel Metrics SDK
+ * instruments. Each beast::insight instrument maps to an OTel equivalent:
+ *
+ *   - Counter  -> OTel Counter<uint64_t>
+ *   - Gauge    -> OTel ObservableGauge<int64_t> (async callback)
+ *   - Event    -> OTel Histogram<double> (duration in milliseconds)
+ *   - Meter    -> OTel Counter<uint64_t> (monotonic, unsigned)
+ *   - Hook     -> Called by PeriodicMetricReader at collection time
+ *
+ * Example — primary use (create the collector and record a metric):
+ * @code
+ *   auto collector = beast::insight::OTelCollector::New(
+ *       "http://localhost:4318/v1/metrics",  // OTLP/HTTP endpoint
+ *       "xrpld",                              // metric name prefix
+ *       "node-1",                             // service.instance.id
+ *       "xrpld",                              // service.name
+ *       "mainnet",                            // xrpl.network.type
+ *       journal);
+ *
+ *   auto counter = collector->makeCounter("ledgers", "closed");
+ *   ++counter;  // exported on the next PeriodicMetricReader tick
+ * @endcode
+ *
+ * Example — edge case (telemetry disabled at compile time): New()
+ * returns a NullCollector, so callers need no #ifdef guard. The same
+ * instruments compile and run, but recording is a no-op.
+ * @code
+ *   auto collector = beast::insight::OTelCollector::New(
+ *       endpoint, prefix, instanceId, serviceName, networkType, journal);
+ *   auto gauge = collector->makeGauge("peers", "count");
+ *   gauge = 42;  // silently discarded when XRPL_ENABLE_TELEMETRY is off
+ * @endcode
+ *
+ * @note Thread safety: instrument recording (Counter::Add,
+ *       Histogram::Record, and atomic gauge writes) is thread-safe and
+ *       may be called concurrently. Instrument *creation* (make_counter,
+ *       make_gauge, etc.) is serialized by an internal mutex. Hook and
+ *       observable-gauge callbacks run on the SDK's collection thread, so
+ *       any state they read must itself be thread-safe.
+ * @note Limitations: metrics export over OTLP/HTTP only (no gRPC); the
+ *       PeriodicMetricReader interval is fixed at 1s; and gauge values
+ *       are stored as int64_t, so fractional gauges are truncated.
+ *
+ * @see StatsDCollector for the StatsD-based alternative.
+ * @see NullCollector for the no-op fallback.
+ */
+class OTelCollector : public Collector
+{
+public:
+    explicit OTelCollector() = default;
+
+    /**
+     * @brief Factory method to create an OTelCollector instance.
+     *
+     * When XRPL_ENABLE_TELEMETRY is defined, creates a real OTel-backed
+     * collector that exports metrics via OTLP/HTTP. When telemetry is
+     * disabled at compile time, returns a NullCollector.
+     *
+     * @param endpoint    OTLP/HTTP metrics endpoint URL
+     *                    (e.g. "http://localhost:4318/v1/metrics").
+     * @param prefix      Prefix prepended to all metric names
+     *                    (e.g. "xrpld").
+     * @param instanceId  Unique identifier for this node instance,
+     *                    emitted as the `service.instance.id` OTel
+     *                    resource attribute. Defaults to empty string
+     *                    (attribute omitted when empty).
+     * @param serviceName Value for the `service.name` OTel resource
+     *                    attribute. When empty, defaults to "xrpld".
+     *                    Matches the trace exporter's service.name so
+     *                    metrics and traces share one service identity.
+     * @param networkType Value for the `xrpl.network.type` OTel resource
+     *                    attribute (e.g. "mainnet"). When empty, the
+     *                    attribute is omitted. Supplied by the node so
+     *                    metrics carry the same network label as traces.
+     * @param journal     Journal for logging.
+     * @return Shared pointer to the created Collector.
+     */
+    static std::shared_ptr<Collector>
+    // NOLINTNEXTLINE(readability-identifier-naming)
+    New(std::string const& endpoint,
+        std::string const& prefix,
+        std::string const& instanceId,
+        std::string const& serviceName,
+        std::string const& networkType,
+        Journal journal);
+};
+
+}  // namespace beast::insight

include/xrpl/core/ServiceRegistry.h

@@ -26,6 +26,7 @@ class PerfLog;
 }  // namespace perf
 namespace telemetry {
 class Telemetry;
+class MetricsRegistry;
 }  // namespace telemetry
 
 // This is temporary until we migrate all code to use ServiceRegistry.
@@ -230,6 +231,12 @@ public:
     virtual telemetry::Telemetry&
     getTelemetry() = 0;
 
+    /** Return the MetricsRegistry, or nullptr if telemetry is disabled.
+        Used by PerfLog and other hot paths to record OTel metrics.
+    */
+    virtual telemetry::MetricsRegistry*
+    getMetricsRegistry() = 0;
+
     // Configuration and state
     [[nodiscard]] virtual bool
     isStopping() const = 0;

include/xrpl/proto/xrpl.proto

@@ -102,6 +102,10 @@ message TraceContext {
   optional bytes trace_id = 1;      // 16-byte trace identifier
   optional bytes span_id = 2;       // 8-byte parent span identifier
   optional uint32 trace_flags = 3;  // bit 0 = sampled
+  // TODO: trace_state is reserved for W3C tracestate vendor-specific
+  // key-value pairs but is not yet read or written by
+  // TraceContextPropagator. Wire it when cross-vendor trace
+  // propagation is needed.
   optional string trace_state = 4;  // RESERVED — see TraceContext header note
 }
 

include/xrpl/server/NetworkOPs.h

@@ -101,6 +101,13 @@ public:
 
     [[nodiscard]] virtual OperatingMode
     getOperatingMode() const = 0;
+    /** Time spent in the current operating mode so far, in microseconds.
+        Same value as `server_state_duration_us` in server_info; exposed as a
+        lightweight accessor so metrics can read it without building the full
+        server_info JSON on every collection tick.
+    */
+    [[nodiscard]] virtual std::chrono::microseconds
+    getServerStateDurationUs() const = 0;
     [[nodiscard]] virtual std::string
     strOperatingMode(OperatingMode const mode, bool const admin = false) const = 0;
     [[nodiscard]] virtual std::string

include/xrpl/telemetry/Telemetry.h

@@ -98,6 +98,7 @@
 
 #ifdef XRPL_ENABLE_TELEMETRY
 #include <opentelemetry/context/context.h>
+#include <opentelemetry/metrics/meter.h>
 #include <opentelemetry/nostd/shared_ptr.h>
 #include <opentelemetry/trace/span.h>
 #include <opentelemetry/trace/span_metadata.h>
@@ -111,6 +112,13 @@ namespace xrpl::telemetry {
     source of spans; distinct from the `service.name` resource attribute
     (Setup::serviceName), which is config-overridable. */
 inline constexpr std::string_view kTracerName{"xrpld"};
+
+/** OTel instrumentation scope (meter) name. Identifies this library as the
+    source of metrics; symmetric with kTracerName for the tracing side. */
+inline constexpr std::string_view kMeterName{"xrpld"};
+
+/** OTel instrumentation scope version reported for the meter. */
+inline constexpr std::string_view kMeterVersion{"1.0.0"};
 #endif
 
 class Telemetry
@@ -302,6 +310,20 @@ public:
     virtual opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer>
     getTracer(std::string_view name = kTracerName) = 0;
 
+    /** Get or create a named meter instance.
+
+        Returns the raw OTel Meter, giving developers direct access to the
+        full metrics API. From the returned Meter any instrument type can be
+        created: Counter, UpDownCounter, Gauge, and Histogram (synchronous),
+        plus their observable/async variants (ObservableCounter,
+        ObservableUpDownCounter, ObservableGauge).
+
+        @param name  Meter name used to identify the instrumentation scope.
+        @return A shared pointer to the Meter.
+    */
+    virtual opentelemetry::nostd::shared_ptr<opentelemetry::metrics::Meter>
+    getMeter(std::string_view name = kMeterName) = 0;
+
     /** Start a new span on the current thread's context.
 
         The span becomes a child of the current active span (if any) via
@@ -366,4 +388,15 @@ makeTelemetrySetup(
     std::string const& version,
     std::uint32_t networkId);
 
+/** Derive a human-readable network type label from the numeric network ID.
+
+    Shared by the trace and metric export paths so both stamp the same
+    `xrpl.network.type` resource attribute value.
+
+    @param networkId  The network identifier from [network_id] config.
+    @return "mainnet" (0), "testnet" (1), "devnet" (2), or "unknown".
+*/
+std::string
+networkTypeFromId(std::uint32_t networkId);
+
 }  // namespace xrpl::telemetry

sanitizers/suppressions/ubsan.supp

@@ -102,6 +102,10 @@ undefined:nudb
 # Snappy compression library intentional overflows
 unsigned-integer-overflow:snappy.cc
 
+# OpenTelemetry SDK intentional overflows in hash functions
+unsigned-integer-overflow:opentelemetry/sdk/common/attributemap_hash.h
+unsigned-integer-overflow:opentelemetry
+
 # Abseil intentional overflows in hashing, RNG and time arithmetic.
 # Matched at library scope (like boost above): the wraparound is by design
 # across many absl files (hash mixing, raw_hash_set probing, duration math,

src/libxrpl/basics/Log.cpp

@@ -7,6 +7,15 @@
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/filesystem/path.hpp>
 
+#ifdef XRPL_ENABLE_TELEMETRY
+#include <opentelemetry/context/runtime_context.h>
+#include <opentelemetry/nostd/shared_ptr.h>
+#include <opentelemetry/nostd/span.h>
+#include <opentelemetry/nostd/variant.h>
+#include <opentelemetry/trace/span.h>
+#include <opentelemetry/trace/span_metadata.h>
+#endif  // XRPL_ENABLE_TELEMETRY
+
 #include <chrono>
 #include <cstring>
 #include <fstream>
@@ -291,6 +300,34 @@ Logs::format(
             break;
     }
 
+#ifdef XRPL_ENABLE_TELEMETRY
+    // Inject OTel trace context when an active span exists on this thread.
+    // Checks the thread-local context value directly to avoid the heap
+    // allocation that GetSpan() performs on the no-span path.
+    {
+        auto context = opentelemetry::context::RuntimeContext::GetCurrent();
+        auto spanValue = context.GetValue(opentelemetry::trace::kSpanKey);
+        if (opentelemetry::nostd::holds_alternative<
+                opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>>(spanValue))
+        {
+            auto span = opentelemetry::nostd::get<
+                opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>>(spanValue);
+            auto spanCtx = span->GetContext();
+            if (spanCtx.IsValid())
+            {
+                char traceId[32], spanId[16];
+                spanCtx.trace_id().ToLowerBase16(opentelemetry::nostd::span<char, 32>{traceId});
+                spanCtx.span_id().ToLowerBase16(opentelemetry::nostd::span<char, 16>{spanId});
+                output += "trace_id=";
+                output.append(traceId, 32);
+                output += " span_id=";
+                output.append(spanId, 16);
+                output += ' ';
+            }
+        }
+    }
+#endif  // XRPL_ENABLE_TELEMETRY
+
     output += message;
 
     // Limit the maximum length of the output

src/libxrpl/beast/insight/OTelCollector.cpp

@@ -0,0 +1,877 @@
+/**
+ * @file OTelCollector.cpp
+ * @brief OpenTelemetry Metrics SDK implementation of beast::insight::Collector.
+ *
+ * Compiled only when XRPL_ENABLE_TELEMETRY is defined (via CMake
+ * telemetry=ON). Maps beast::insight instruments to OTel SDK instruments
+ * created on the GLOBAL Meter published by the telemetry module. This class
+ * is a legacy shim: it no longer owns an export pipeline. The MeterProvider,
+ * PeriodicExportingMetricReader, OTLP exporter and histogram view all live in
+ * xrpl::telemetry::Telemetry.
+ *
+ * When XRPL_ENABLE_TELEMETRY is not defined, OTelCollector::New() returns
+ * a NullCollector so the build succeeds without OTel dependencies.
+ *
+ * Data flow:
+ *
+ *   beast::insight callers
+ *       |
+ *       v
+ *   OTelCounterImpl / OTelGaugeImpl / OTelEventImpl / OTelMeterImpl
+ *       |                    |                |              |
+ *       v                    v                v              v
+ *   Counter<uint64_t>  ObservableGauge  Histogram<double>  Counter<uint64_t>
+ *       |                    |                |              |
+ *       +--------------------+----------------+--------------+
+ *       |
+ *       v
+ *   GLOBAL Meter (from metrics::Provider::GetMeterProvider())
+ *       |
+ *       v
+ *   telemetry-owned PeriodicMetricReader (1s) -> OTLP exporter -> Prometheus
+ */
+
+#ifdef XRPL_ENABLE_TELEMETRY
+#include <xrpl/beast/insight/OTelCollector.h>
+
+#include <xrpl/beast/insight/Collector.h>
+#include <xrpl/beast/insight/CounterImpl.h>
+#include <xrpl/beast/insight/EventImpl.h>
+#include <xrpl/beast/insight/GaugeImpl.h>
+#include <xrpl/beast/insight/Hook.h>
+#include <xrpl/beast/insight/HookImpl.h>
+#include <xrpl/beast/insight/MeterImpl.h>
+#include <xrpl/beast/utility/Journal.h>
+
+#include <opentelemetry/metrics/async_instruments.h>
+#include <opentelemetry/metrics/meter.h>
+#include <opentelemetry/metrics/meter_provider.h>
+#include <opentelemetry/metrics/observer_result.h>
+#include <opentelemetry/metrics/provider.h>
+#include <opentelemetry/metrics/sync_instruments.h>
+#include <opentelemetry/nostd/shared_ptr.h>
+#include <opentelemetry/nostd/unique_ptr.h>
+#include <opentelemetry/nostd/variant.h>
+
+#include <algorithm>
+#include <atomic>
+#include <cctype>
+#include <chrono>
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace beast::insight {
+
+namespace detail {
+
+namespace metrics_api = opentelemetry::metrics;
+
+class OTelCollectorImp;
+
+//------------------------------------------------------------------------------
+
+/**
+ * @brief OTel-backed implementation of beast::insight::HookImpl.
+ *
+ * Stores a handler function that is invoked during each periodic
+ * metric collection cycle. This mirrors the StatsDHookImpl pattern
+ * where hooks are called at each 1-second timer tick, but here the
+ * invocation is triggered by the OTel PeriodicMetricReader's
+ * observable callback mechanism.
+ */
+class OTelHookImpl : public HookImpl
+{
+public:
+    /**
+     * @param handler  Callback invoked at each collection interval.
+     * @param impl     Owning collector (prevents premature destruction).
+     */
+    OTelHookImpl(HandlerType handler, std::shared_ptr<OTelCollectorImp> impl);
+
+    ~OTelHookImpl() override;
+
+    OTelHookImpl&
+    operator=(OTelHookImpl const&) = delete;
+
+    /**
+     * @brief Invoke the stored handler.
+     *
+     * Called by the collector during observable gauge callbacks to give
+     * metric producers a chance to update gauge values before export.
+     */
+    void
+    callHandler();
+
+private:
+    /** Owning collector. Prevents collector destruction while hook alive. */
+    std::shared_ptr<OTelCollectorImp> impl_;
+
+    /** User-supplied handler called at each collection interval. */
+    HandlerType handler_;
+};
+
+//------------------------------------------------------------------------------
+
+/**
+ * @brief OTel-backed implementation of beast::insight::CounterImpl.
+ *
+ * Wraps an OTel Counter<uint64_t> instrument. Each increment() call
+ * is forwarded directly to the OTel counter's Add() method. The
+ * PeriodicMetricReader collects and exports the accumulated delta.
+ *
+ * Thread safety: OTel Counter::Add() is thread-safe by specification.
+ */
+class OTelCounterImpl : public CounterImpl
+{
+public:
+    /**
+     * @param name   Export-ready metric name, already run through
+     *               formatName() by the collector: prefix prepended and
+     *               dots replaced with underscores (e.g. "rpc_size").
+     * @param meter  OTel Meter used to create the counter instrument.
+     */
+    OTelCounterImpl(
+        std::string const& name,
+        opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter);
+
+    ~OTelCounterImpl() override = default;
+
+    OTelCounterImpl&
+    operator=(OTelCounterImpl const&) = delete;
+
+    /**
+     * @brief Add amount to the counter.
+     * @param amount  Value to add (must be non-negative for OTel counters).
+     */
+    void
+    increment(value_type amount) override;
+
+private:
+    /** OTel synchronous counter instrument. */
+    opentelemetry::nostd::unique_ptr<metrics_api::Counter<uint64_t>> counter_;
+};
+
+//------------------------------------------------------------------------------
+
+/**
+ * @brief OTel-backed implementation of beast::insight::EventImpl.
+ *
+ * Wraps an OTel Histogram<double> instrument. Each notify() call
+ * records the duration in milliseconds. Uses explicit bucket boundaries
+ * matching the SpanMetrics connector configuration:
+ *   [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms
+ *
+ * Thread safety: OTel Histogram::Record() is thread-safe by specification.
+ */
+class OTelEventImpl : public EventImpl
+{
+public:
+    /**
+     * @param name   Export-ready metric name, already run through
+     *               formatName() by the collector: prefix prepended and
+     *               dots replaced with underscores (e.g. "rpc_size").
+     * @param meter  OTel Meter used to create the histogram instrument.
+     */
+    OTelEventImpl(
+        std::string const& name,
+        opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter);
+
+    ~OTelEventImpl() override = default;
+
+    OTelEventImpl&
+    operator=(OTelEventImpl const&) = delete;
+
+    /**
+     * @brief Record a duration measurement.
+     * @param value  Duration in milliseconds.
+     */
+    void
+    notify(value_type const& value) override;
+
+private:
+    /** OTel histogram instrument for recording durations. */
+    opentelemetry::nostd::unique_ptr<metrics_api::Histogram<double>> histogram_;
+};
+
+//------------------------------------------------------------------------------
+
+/**
+ * @brief OTel-backed implementation of beast::insight::GaugeImpl.
+ *
+ * Uses an atomic int64_t to store the current gauge value. The OTel SDK
+ * reads this value via an ObservableGauge async callback during each
+ * collection cycle. The set() and increment() methods update the
+ * atomic value without blocking the collection thread.
+ *
+ * Design note: OTel gauges are asynchronous (observable) instruments.
+ * The SDK calls a registered callback to read the value rather than
+ * accepting push-style updates. We bridge the beast::insight push-style
+ * API to OTel's pull-style API via the atomic variable.
+ *
+ * Thread safety: std::atomic operations are lock-free on all platforms.
+ */
+class OTelGaugeImpl : public GaugeImpl
+{
+public:
+    /**
+     * @param name       Export-ready metric name, already run through
+     *                   formatName() by the collector: prefix prepended
+     *                   and dots replaced with underscores.
+     * @param meter      OTel Meter used to create the observable gauge.
+     * @param collector  Owning collector, used to invoke hooks before reads.
+     */
+    OTelGaugeImpl(
+        std::string const& name,
+        opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter,
+        std::shared_ptr<OTelCollectorImp> const& collector);
+
+    ~OTelGaugeImpl() override;
+
+    /**
+     * @brief Set the gauge to an absolute value.
+     * @param value  New gauge value.
+     */
+    void
+    set(value_type value) override;
+
+    /**
+     * @brief Increment (or decrement) the gauge by a signed amount.
+     *
+     * Clamps the result to [0, INT64_MAX] to match StatsDGaugeImpl
+     * behavior.
+     *
+     * @param amount  Signed amount to add to the current value.
+     */
+    void
+    increment(difference_type amount) override;
+
+    /**
+     * @brief Return the current gauge value for the OTel callback.
+     * @return The most recently set/incremented value.
+     */
+    int64_t
+    currentValue() const;
+
+    OTelGaugeImpl&
+    operator=(OTelGaugeImpl const&) = delete;
+
+    /** Static callback registered with the OTel SDK observable gauge. */
+    static void
+    gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state);
+
+private:
+    /** Current gauge value, updated atomically by set()/increment(). */
+    std::atomic<int64_t> value_{0};
+
+    /** OTel observable gauge handle (prevents deregistration). */
+    opentelemetry::nostd::shared_ptr<metrics_api::ObservableInstrument> gauge_;
+
+    /** Owning collector, used to invoke hooks before reading gauge values. */
+    std::shared_ptr<OTelCollectorImp> collector_;
+};
+
+//------------------------------------------------------------------------------
+
+/**
+ * @brief OTel-backed implementation of beast::insight::MeterImpl.
+ *
+ * Wraps an OTel Counter<uint64_t> instrument. Semantically identical
+ * to Counter but uses unsigned values. The OTel SDK accumulates deltas
+ * and exports them via the PeriodicMetricReader.
+ *
+ * Note: In StatsD, Meter used the non-standard "|m" type which was
+ * silently dropped by the OTel StatsD receiver. With native OTel,
+ * Meter values are properly captured as counter deltas.
+ *
+ * Thread safety: OTel Counter::Add() is thread-safe by specification.
+ */
+class OTelMeterImpl : public MeterImpl
+{
+public:
+    /**
+     * @param name   Export-ready metric name, already run through
+     *               formatName() by the collector: prefix prepended and
+     *               dots replaced with underscores (e.g. "rpc_size").
+     * @param meter  OTel Meter used to create the counter instrument.
+     */
+    OTelMeterImpl(
+        std::string const& name,
+        opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter);
+
+    ~OTelMeterImpl() override = default;
+
+    OTelMeterImpl&
+    operator=(OTelMeterImpl const&) = delete;
+
+    /**
+     * @brief Add amount to the meter.
+     * @param amount  Value to add (unsigned).
+     */
+    void
+    increment(value_type amount) override;
+
+private:
+    /** OTel synchronous counter instrument (unsigned). */
+    opentelemetry::nostd::unique_ptr<metrics_api::Counter<uint64_t>> counter_;
+};
+
+//------------------------------------------------------------------------------
+
+/**
+ * @brief Main OTel Collector implementation (legacy shim).
+ *
+ * Obtains its Meter from the GLOBAL MeterProvider owned and published by the
+ * telemetry module (xrpl::telemetry::Telemetry), rather than building its own
+ * export pipeline. Implements all Collector::make_*() factory methods to
+ * create OTel-backed instrument wrappers on that shared Meter.
+ *
+ * The metrics pipeline (MeterProvider + PeriodicExportingMetricReader + OTLP
+ * HTTP exporter + histogram view) lives in the telemetry module. This class is
+ * a thin adapter kept for beast::insight callers during deprecation.
+ *
+ * Class diagram:
+ *
+ *   +------------------+      +------------------+
+ *   | Collector (ABC)  |<-----| OTelCollector    |
+ *   +------------------+      | (public header)  |
+ *          ^                  +------------------+
+ *          |                          ^
+ *   +------------------+             |
+ *   | OTelCollectorImp |-------------+
+ *   +------------------+
+ *   | - journal_       |
+ *   | - prefix_        |     +--------------------------+
+ *   | - otelMeter_     |---->| GLOBAL MeterProvider     |
+ *   | - hooks_[]       |     | (owned by Telemetry)     |
+ *   | - gauges_[]      |     | + PeriodicReader         |
+ *   +------------------+     | + OtlpHttpExporter       |
+ *                            +--------------------------+
+ *
+ * Lifecycle:
+ *   1. Constructor fetches the Meter from the global MeterProvider.
+ *   2. make_*() methods create instruments on that shared Meter.
+ *   3. The telemetry-owned PeriodicMetricReader collects every 1s, calling
+ *      observable callbacks.
+ *   4. Observable callbacks invoke hooks, read gauge atomics.
+ *   5. Destructor only logs; the telemetry module owns pipeline teardown.
+ *
+ * Caveats:
+ *   - Observable gauge callbacks run on the SDK's internal thread. Hook
+ *     handlers must be thread-safe.
+ *   - Metric names are formed as "prefix_name" with dots replaced by
+ *     underscores to match StatsD->Prometheus naming conventions.
+ *   - The OTel Prometheus exporter appends "_total" to counters. The
+ *     metric names we register do NOT include this suffix — Prometheus
+ *     adds it automatically.
+ *
+ * Example usage:
+ * @code
+ *   auto collector = OTelCollector::New(
+ *       "http://localhost:4318/v1/metrics", "xrpld",
+ *       "node-1", "xrpld", "mainnet", journal);
+ *   auto counter = collector->makeCounter("rpc.requests");
+ *   counter.increment(1);
+ *   // Metric "rpc_requests" exported via OTLP every 1s.
+ * @endcode
+ */
+class OTelCollectorImp : public OTelCollector, public std::enable_shared_from_this<OTelCollectorImp>
+{
+public:
+    /**
+     * @brief Construct the OTel collector over the global MeterProvider.
+     *
+     * @param endpoint    OTLP/HTTP metrics endpoint URL. Informational only:
+     *                    the global telemetry pipeline is authoritative for
+     *                    the actual export endpoint. Retained for logging and
+     *                    back-compat with the New() signature.
+     * @param prefix      Prefix for all metric names.
+     * @param instanceId  Value for the service.instance.id resource attribute.
+     *                    When empty, the attribute is omitted.
+     * @param serviceName Value for the service.name resource attribute.
+     *                    When empty, defaults to "xrpld".
+     * @param networkType Value for the xrpl.network.type resource attribute.
+     *                    When empty, the attribute is omitted.
+     * @param journal     Journal for logging.
+     */
+    OTelCollectorImp(
+        std::string const& endpoint,
+        std::string prefix,
+        std::string const& instanceId,
+        std::string const& serviceName,
+        std::string const& networkType,
+        Journal journal);
+
+    /**
+     * @brief Shut down the MeterProvider, flushing any pending exports.
+     */
+    ~OTelCollectorImp() override;
+
+    /** @name Collector interface implementation */
+    /** @{ */
+    Hook
+    makeHook(HookImpl::HandlerType const& handler) override;
+
+    Counter
+    makeCounter(std::string const& name) override;
+
+    Event
+    makeEvent(std::string const& name) override;
+
+    Gauge
+    makeGauge(std::string const& name) override;
+
+    Meter
+    makeMeter(std::string const& name) override;
+    /** @} */
+
+    /** @name Hook management for observable callbacks */
+    /** @{ */
+
+    /**
+     * @brief Register a hook for periodic invocation.
+     * @param hook  Pointer to the hook to register.
+     */
+    void
+    addHook(OTelHookImpl* hook);
+
+    /**
+     * @brief Unregister a hook.
+     * @param hook  Pointer to the hook to unregister.
+     */
+    void
+    removeHook(OTelHookImpl* hook);
+
+    /**
+     * @brief Invoke all registered hooks.
+     *
+     * Called from observable gauge callbacks before reading gauge values,
+     * so that hook handlers have a chance to update metrics.
+     */
+    void
+    callHooks();
+    /** @} */
+
+    /** @name Gauge registration for observable callbacks */
+    /** @{ */
+
+    /**
+     * @brief Register a gauge for observable callback reading.
+     * @param gauge  Pointer to the gauge to register.
+     */
+    void
+    addGauge(OTelGaugeImpl* gauge);
+
+    /**
+     * @brief Unregister a gauge.
+     * @param gauge  Pointer to the gauge to unregister.
+     */
+    void
+    removeGauge(OTelGaugeImpl* gauge);
+    /** @} */
+
+    /**
+     * @brief Format a metric name with the configured prefix.
+     *
+     * Replaces dots with underscores to match StatsD->Prometheus naming.
+     * Example: prefix="xrpld", name="LedgerMaster.Validated_Ledger_Age"
+     *   -> "ledgermaster_validated_ledger_age"
+     *
+     * @param name  Raw metric name from beast::insight callers.
+     * @return Fully-qualified metric name.
+     */
+    static std::string
+    formatName(std::string const& name);
+
+private:
+    /** Journal for log output. */
+    Journal journal_;
+
+    /** Prefix for all metric names (e.g., "xrpld"). */
+    std::string prefix_;
+
+    /** OTel Meter used to create all instruments. */
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter> otelMeter_;
+
+    /** Mutex protecting hook and gauge registration lists. */
+    std::mutex mutex_;
+
+    /** Registered hooks called during observable callbacks. */
+    std::vector<OTelHookImpl*> hooks_;
+
+    /** Registered gauges read during observable callbacks. */
+    std::vector<OTelGaugeImpl*> gauges_;
+
+    /**
+     * @brief Debounce timestamp for callHooks().
+     *
+     * Multiple gauge callbacks fire during the same collection cycle.
+     * This atomic tracks the last time hooks were invoked (ms since epoch).
+     * Hooks are called at most once per 500ms window to avoid redundant
+     * invocations while still ensuring fresh values each collection cycle.
+     */
+    std::atomic<int64_t> lastHookCallMs_{0};
+};
+
+//==============================================================================
+// Implementation
+//==============================================================================
+
+//------------------------------------------------------------------------------
+// OTelHookImpl
+//------------------------------------------------------------------------------
+
+OTelHookImpl::OTelHookImpl(HandlerType handler, std::shared_ptr<OTelCollectorImp> impl)
+    : impl_(std::move(impl)), handler_(std::move(handler))
+{
+    impl_->addHook(this);
+}
+
+OTelHookImpl::~OTelHookImpl()
+{
+    impl_->removeHook(this);
+}
+
+void
+OTelHookImpl::callHandler()
+{
+    handler_();
+}
+
+//------------------------------------------------------------------------------
+// OTelCounterImpl
+//------------------------------------------------------------------------------
+
+OTelCounterImpl::OTelCounterImpl(
+    std::string const& name,
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter)
+    : counter_(meter->CreateUInt64Counter(name))
+{
+}
+
+void
+OTelCounterImpl::increment(value_type amount)
+{
+    // OTel counters require non-negative values. beast::insight CounterImpl
+    // uses int64_t, so clamp negative values to 0 and cast to uint64_t.
+    if (amount > 0)
+        counter_->Add(static_cast<uint64_t>(amount));
+}
+
+//------------------------------------------------------------------------------
+// OTelEventImpl
+//------------------------------------------------------------------------------
+
+OTelEventImpl::OTelEventImpl(
+    std::string const& name,
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter)
+    : histogram_(meter->CreateDoubleHistogram(name, "Duration in ms", "ms"))
+{
+}
+
+void
+OTelEventImpl::notify(value_type const& value)
+{
+    histogram_->Record(static_cast<double>(value.count()), opentelemetry::context::Context{});
+}
+
+//------------------------------------------------------------------------------
+// OTelGaugeImpl
+//------------------------------------------------------------------------------
+
+OTelGaugeImpl::OTelGaugeImpl(
+    std::string const& name,
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter,
+    std::shared_ptr<OTelCollectorImp> const& collector)
+    : gauge_(meter->CreateInt64ObservableGauge(name)), collector_(collector)
+{
+    collector_->addGauge(this);
+    gauge_->AddCallback(gaugeCallback, this);
+}
+
+void
+OTelGaugeImpl::gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state)
+{
+    auto* self = static_cast<OTelGaugeImpl*>(state);
+    self->collector_->callHooks();
+    if (auto intResult = opentelemetry::nostd::get_if<
+            opentelemetry::nostd::shared_ptr<opentelemetry::metrics::ObserverResultT<int64_t>>>(
+            &result))
+    {
+        (*intResult)->Observe(self->currentValue());
+    }
+}
+
+OTelGaugeImpl::~OTelGaugeImpl()
+{
+    // RemoveCallback must run before this object is destroyed so the SDK
+    // collection thread cannot invoke gaugeCallback on a dangling `this`.
+    // The SDK's ObservableRegistry guards its callback list and the Observe()
+    // pass with the same mutex, so RemoveCallback cannot return while a
+    // callback for this instrument is in flight — removal is synchronous.
+    gauge_->RemoveCallback(gaugeCallback, this);
+    collector_->removeGauge(this);
+}
+
+void
+OTelGaugeImpl::set(value_type value)
+{
+    value_.store(static_cast<int64_t>(value), std::memory_order_relaxed);
+}
+
+void
+OTelGaugeImpl::increment(difference_type amount)
+{
+    // Use compare-exchange loop to safely clamp to [0, MAX].
+    int64_t current = value_.load(std::memory_order_relaxed);
+    int64_t desired = 0;
+    do
+    {
+        desired = current + amount;
+        // Clamp to 0 on underflow.
+        desired = std::max(desired, int64_t{0});
+    } while (!value_.compare_exchange_weak(current, desired, std::memory_order_relaxed));
+}
+
+int64_t
+OTelGaugeImpl::currentValue() const
+{
+    return value_.load(std::memory_order_relaxed);
+}
+
+//------------------------------------------------------------------------------
+// OTelMeterImpl
+//------------------------------------------------------------------------------
+
+OTelMeterImpl::OTelMeterImpl(
+    std::string const& name,
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter)
+    : counter_(meter->CreateUInt64Counter(name))
+{
+}
+
+void
+OTelMeterImpl::increment(value_type amount)
+{
+    counter_->Add(amount);
+}
+
+//------------------------------------------------------------------------------
+// OTelCollectorImp
+//------------------------------------------------------------------------------
+
+OTelCollectorImp::OTelCollectorImp(
+    std::string const& endpoint,
+    std::string prefix,
+    std::string const& instanceId,
+    std::string const& serviceName,
+    std::string const& networkType,
+    Journal journal)
+    : journal_(journal), prefix_(std::move(prefix))
+{
+    // instanceId/serviceName/networkType are retained on the New() signature
+    // for back-compat but no longer used here: the telemetry module owns the
+    // resource attributes for the shared metrics pipeline.
+    (void)instanceId;
+    (void)serviceName;
+    (void)networkType;
+
+    if (journal_.info())
+    {
+        // endpoint is informational: the global telemetry pipeline owns the
+        // real exporter. It is logged here for back-compat and diagnostics.
+        journal_.info() << "OTelCollector starting: endpoint=" << endpoint << " prefix=" << prefix_;
+    }
+
+    // Fetch the Meter from the GLOBAL MeterProvider. The telemetry module
+    // (xrpl::telemetry::Telemetry) builds the metrics pipeline (exporter,
+    // periodic reader, histogram view, resource attributes) and registers it
+    // via metrics::Provider::SetMeterProvider() during start(). beast metrics
+    // ride that shared pipeline, so both direct-API and beast-sourced metrics
+    // export under one resource identity.
+    //
+    // The name/version literals MUST match the telemetry module's kMeterName
+    // ("xrpld") and kMeterVersion ("1.0.0"). They are written as literals (not
+    // referenced from Telemetry.h) because beast/insight sits below the
+    // telemetry module in the layering and cannot include its header.
+    otelMeter_ = metrics_api::Provider::GetMeterProvider()->GetMeter(
+        std::string{"xrpld"}, std::string{"1.0.0"});
+
+    if (journal_.info())
+    {
+        journal_.info() << "OTelCollector started successfully";
+    }
+}
+
+OTelCollectorImp::~OTelCollectorImp()
+{
+    if (journal_.info())
+    {
+        journal_.info() << "OTelCollector shutting down";
+    }
+    // No pipeline teardown here: the telemetry module owns the global
+    // MeterProvider lifecycle (ForceFlush/Shutdown happen in Telemetry::stop()).
+    if (journal_.info())
+    {
+        journal_.info() << "OTelCollector stopped";
+    }
+}
+
+Hook
+OTelCollectorImp::makeHook(HookImpl::HandlerType const& handler)
+{
+    return Hook(std::make_shared<OTelHookImpl>(handler, shared_from_this()));
+}
+
+Counter
+OTelCollectorImp::makeCounter(std::string const& name)
+{
+    return Counter(std::make_shared<OTelCounterImpl>(formatName(name), otelMeter_));
+}
+
+Event
+OTelCollectorImp::makeEvent(std::string const& name)
+{
+    return Event(std::make_shared<OTelEventImpl>(formatName(name), otelMeter_));
+}
+
+Gauge
+OTelCollectorImp::makeGauge(std::string const& name)
+{
+    return Gauge(std::make_shared<OTelGaugeImpl>(formatName(name), otelMeter_, shared_from_this()));
+}
+
+Meter
+OTelCollectorImp::makeMeter(std::string const& name)
+{
+    return Meter(std::make_shared<OTelMeterImpl>(formatName(name), otelMeter_));
+}
+
+void
+OTelCollectorImp::addHook(OTelHookImpl* hook)
+{
+    std::scoped_lock const lock(mutex_);
+    hooks_.push_back(hook);
+}
+
+void
+OTelCollectorImp::removeHook(OTelHookImpl* hook)
+{
+    std::scoped_lock const lock(mutex_);
+    std::erase(hooks_, hook);
+}
+
+void
+OTelCollectorImp::callHooks()
+{
+    // Debounce: hooks run at most once per 500ms. Multiple gauge callbacks
+    // fire during the same collection cycle — only the first one triggers
+    // hooks. Subsequent callbacks within the window read already-updated
+    // gauge values.
+    auto now = std::chrono::duration_cast<std::chrono::milliseconds>(
+                   std::chrono::steady_clock::now().time_since_epoch())
+                   .count();
+    auto last = lastHookCallMs_.load(std::memory_order_acquire);
+    if (now - last < 500)
+        return;
+    if (!lastHookCallMs_.compare_exchange_strong(last, now, std::memory_order_acq_rel))
+        return;  // Another thread won the race.
+
+    // Copy the hook list under the lock, then invoke handlers outside it.
+    // A handler may drop the last reference to an OTelHookImpl, whose
+    // destructor calls removeHook() and re-acquires mutex_; invoking
+    // handlers while holding the (non-recursive) lock would deadlock.
+    std::vector<OTelHookImpl*> hooks;
+    {
+        std::scoped_lock const lock(mutex_);
+        hooks = hooks_;
+    }
+    for (auto* hook : hooks)
+        hook->callHandler();
+}
+
+void
+OTelCollectorImp::addGauge(OTelGaugeImpl* gauge)
+{
+    std::scoped_lock const lock(mutex_);
+    gauges_.push_back(gauge);
+}
+
+void
+OTelCollectorImp::removeGauge(OTelGaugeImpl* gauge)
+{
+    std::scoped_lock const lock(mutex_);
+    std::erase(gauges_, gauge);
+}
+
+std::string
+OTelCollectorImp::formatName(std::string const& name)
+{
+    // Produce a clean, lowercase, Prometheus-compatible metric name.
+    // No prefix — the OTel resource (service.name) identifies the service.
+    // Dots and spaces become underscores; everything lowercased.
+    std::string result;
+    result.reserve(name.size());
+    for (char const c : name)
+    {
+        if (c == '.' || c == ' ')
+        {
+            result += '_';
+        }
+        else
+        {
+            result += static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
+        }
+    }
+    return result;
+}
+
+}  // namespace detail
+
+//------------------------------------------------------------------------------
+
+std::shared_ptr<Collector>
+OTelCollector::New(
+    std::string const& endpoint,
+    std::string const& prefix,
+    std::string const& instanceId,
+    std::string const& serviceName,
+    std::string const& networkType,
+    Journal journal)
+{
+    return std::make_shared<detail::OTelCollectorImp>(
+        endpoint, prefix, instanceId, serviceName, networkType, journal);
+}
+
+}  // namespace beast::insight
+
+#else  // !XRPL_ENABLE_TELEMETRY
+
+// When telemetry is disabled at compile time, OTelCollector::New()
+// returns a NullCollector so callers do not need conditional logic.
+
+#include <xrpl/beast/insight/Collector.h>
+#include <xrpl/beast/insight/NullCollector.h>
+#include <xrpl/beast/insight/OTelCollector.h>
+#include <xrpl/beast/utility/Journal.h>
+
+namespace beast::insight {
+
+std::shared_ptr<Collector>
+OTelCollector::New(
+    std::string const& /* endpoint */,
+    std::string const& /* prefix */,
+    std::string const& /* instanceId */,
+    std::string const& /* serviceName */,
+    std::string const& /* networkType */,
+    Journal /* journal */)
+{
+    return NullCollector::make();
+}
+
+}  // namespace beast::insight
+
+#endif  // XRPL_ENABLE_TELEMETRY

src/libxrpl/beast/insight/StatsDCollector.cpp

@@ -166,7 +166,7 @@ private:
     std::string name_;
     GaugeImpl::value_type lastValue_{0};
     GaugeImpl::value_type value_{0};
-    bool dirty_{false};
+    bool dirty_{true};
 };
 
 //------------------------------------------------------------------------------
@@ -572,6 +572,9 @@ StatsDEventImpl::doNotify(EventImpl::value_type const& value)
 StatsDGaugeImpl::StatsDGaugeImpl(std::string name, std::shared_ptr<StatsDCollectorImp> impl)
     : impl_(std::move(impl)), name_(std::move(name))
 {
+    // Start dirty so the initial value (0) is emitted on the first flush.
+    // Without this, gauges whose value never changes from 0 would never
+    // appear in downstream metric stores (e.g. Prometheus via StatsD).
     impl_->add(*this);
 }
 

src/libxrpl/telemetry/SpanGuard.cpp

@@ -472,6 +472,7 @@ SpanGuard::addEvent(std::string_view name, std::initializer_list<EventAttribute>
     // std::vector<std::pair<...>> doesn't satisfy is_key_value_iterable.
     // Wrap in nostd::span over the vector's storage so the SDK accepts it.
     std::vector<std::pair<opentelemetry::nostd::string_view, opentelemetry::common::AttributeValue>>
+
         otelAttrs;
     otelAttrs.reserve(attrs.size());
     for (auto const& [k, v] : attrs)

src/libxrpl/telemetry/Telemetry.cpp

@@ -25,7 +25,23 @@
 #include <opentelemetry/context/context.h>
 #include <opentelemetry/exporters/otlp/otlp_http_exporter_factory.h>
 #include <opentelemetry/exporters/otlp/otlp_http_exporter_options.h>
+#include <opentelemetry/exporters/otlp/otlp_http_metric_exporter_factory.h>
+#include <opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h>
+#include <opentelemetry/metrics/meter.h>
+#include <opentelemetry/metrics/meter_provider.h>
+#include <opentelemetry/metrics/noop.h>
+#include <opentelemetry/metrics/provider.h>
 #include <opentelemetry/nostd/shared_ptr.h>
+#include <opentelemetry/sdk/metrics/aggregation/aggregation_config.h>
+#include <opentelemetry/sdk/metrics/export/periodic_exporting_metric_reader_factory.h>
+#include <opentelemetry/sdk/metrics/export/periodic_exporting_metric_reader_options.h>
+#include <opentelemetry/sdk/metrics/instruments.h>
+#include <opentelemetry/sdk/metrics/meter_provider.h>
+#include <opentelemetry/sdk/metrics/meter_provider_factory.h>
+#include <opentelemetry/sdk/metrics/view/instrument_selector_factory.h>
+#include <opentelemetry/sdk/metrics/view/meter_selector_factory.h>
+#include <opentelemetry/sdk/metrics/view/view_factory.h>
+#include <opentelemetry/sdk/metrics/view/view_registry.h>
 #include <opentelemetry/sdk/resource/resource.h>
 #include <opentelemetry/sdk/trace/batch_span_processor_factory.h>
 #include <opentelemetry/sdk/trace/batch_span_processor_options.h>
@@ -50,6 +66,7 @@
 #include <string>
 #include <string_view>
 #include <utility>
+#include <vector>
 
 namespace xrpl::telemetry {
 
@@ -57,6 +74,8 @@ namespace {
 
 namespace trace_api = opentelemetry::trace;
 namespace trace_sdk = opentelemetry::sdk::trace;
+namespace metrics_api = opentelemetry::metrics;
+namespace metrics_sdk = opentelemetry::sdk::metrics;
 namespace otlp_http = opentelemetry::exporter::otlp;
 namespace resource = opentelemetry::sdk::resource;
 
@@ -222,6 +241,16 @@ public:
         return noopTracer;
     }
 
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter>
+    getMeter(std::string_view name) override
+    {
+        // Serve a meter from a process-wide noop provider, mirroring the
+        // noop tracer above. Instruments created from it are inert.
+        static auto noopProvider = opentelemetry::nostd::shared_ptr<metrics_api::MeterProvider>(
+            new metrics_api::NoopMeterProvider());
+        return noopProvider->GetMeter(std::string(name), std::string(kMeterVersion));
+    }
+
     opentelemetry::nostd::shared_ptr<trace_api::Span>
     startSpan(std::string_view, trace_api::SpanKind) override
     {
@@ -259,9 +288,32 @@ class TelemetryImpl : public Telemetry
     */
     std::shared_ptr<trace_sdk::TracerProvider> sdkProvider_;
 
+    /** The SDK MeterProvider that owns the metric export pipeline.
+
+        Symmetric with sdkProvider_ on the tracing side. Held as
+        std::shared_ptr so we can ForceFlush() on shutdown; wrapped in a
+        nostd::shared_ptr when registered as the global meter provider.
+    */
+    std::shared_ptr<metrics_sdk::MeterProvider> meterProvider_;
+
 public:
     TelemetryImpl(Setup setup, beast::Journal journal) : setup_(std::move(setup)), journal_(journal)
     {
+        // Build the metrics pipeline NOW, in the constructor, so the global
+        // MeterProvider is published before any subsystem is constructed.
+        // beast::insight instruments are created eagerly in subsystem
+        // constructors (e.g. LedgerMaster, NetworkOPs, ServerHandler), which
+        // run during ApplicationImp's member-init list — long before start().
+        // opentelemetry-cpp has no proxy MeterProvider, so an instrument
+        // created before SetMeterProvider() binds to the noop provider forever.
+        // Tracing does not have this problem because getTracer() is called
+        // fresh at each span creation (runtime, after start()).
+        //
+        // The metrics resource uses setup_.serviceInstanceId as provided by
+        // config. A later setServiceInstanceId() (node-key fallback) cannot
+        // change this immutable resource, so operators relying on the node-key
+        // identity should set [telemetry] service_instance_id explicitly.
+        initMetrics();
     }
 
     void
@@ -331,6 +383,10 @@ public:
         trace_api::Provider::SetTracerProvider(
             opentelemetry::nostd::shared_ptr<trace_api::TracerProvider>(sdkProvider_));
 
+        // The metrics pipeline (meterProvider_) was already built and published
+        // in the constructor via initMetrics(), before any subsystem could
+        // create a beast::insight instrument. See initMetrics().
+
         // Register as the global Telemetry instance so SpanGuard factory
         // methods can access it without callers passing a reference.
         Telemetry::setInstance(this);
@@ -338,6 +394,99 @@ public:
         JLOG(journal_.info()) << "Telemetry started successfully";
     }
 
+    /** Build and publish the metrics pipeline (MeterProvider + periodic
+        reader + OTLP exporter + histogram view).
+
+        Called from the constructor, NOT start(), so the global MeterProvider
+        exists before subsystems construct their beast::insight instruments
+        during ApplicationImp's member-init list. The metrics resource uses
+        setup_.serviceInstanceId from config; it is immutable once the provider
+        is built, so a later node-key setServiceInstanceId() does not affect it.
+    */
+    void
+    initMetrics()
+    {
+        // Derive the metrics endpoint from the trace endpoint by swapping
+        // the trailing "/v1/traces" path for "/v1/metrics". Any other URL
+        // shape is used as-is.
+        std::string metricsEndpoint = setup_.exporterEndpoint;
+        constexpr std::string_view tracesPath{"/v1/traces"};
+        if (metricsEndpoint.ends_with(tracesPath))
+        {
+            metricsEndpoint.replace(
+                metricsEndpoint.size() - tracesPath.size(), tracesPath.size(), "/v1/metrics");
+        }
+
+        // Configure OTLP HTTP metric exporter, honoring the same TLS
+        // options as the trace exporter.
+        otlp_http::OtlpHttpMetricExporterOptions metricExporterOpts;
+        metricExporterOpts.url = metricsEndpoint;
+        if (setup_.useTls)
+        {
+            metricExporterOpts.ssl_ca_cert_path = setup_.tlsCertPath;
+            metricExporterOpts.ssl_client_cert_path = setup_.tlsClientCertPath;
+            metricExporterOpts.ssl_client_key_path = setup_.tlsClientKeyPath;
+        }
+
+        auto metricExporter = otlp_http::OtlpHttpMetricExporterFactory::Create(metricExporterOpts);
+
+        // Configure periodic metric reader (1-second export interval,
+        // matching the beast OTelCollector path).
+        metrics_sdk::PeriodicExportingMetricReaderOptions readerOpts;
+        readerOpts.export_interval_millis = std::chrono::milliseconds(1000);
+        readerOpts.export_timeout_millis = std::chrono::milliseconds(500);
+
+        auto reader = metrics_sdk::PeriodicExportingMetricReaderFactory::Create(
+            std::move(metricExporter), readerOpts);
+
+        // Metrics resource: same attributes as the tracer resource so metrics
+        // and traces share one identity. Built here (not shared with start())
+        // because start() runs later; serviceInstanceId comes from config.
+        auto resourceAttrs = resource::Resource::Create({
+            {opentelemetry::semconv::service::kServiceName, setup_.serviceName},
+            {opentelemetry::semconv::service::kServiceVersion, setup_.serviceVersion},
+            {opentelemetry::semconv::service::kServiceInstanceId, setup_.serviceInstanceId},
+            {std::string(attr::networkId), static_cast<int64_t>(setup_.networkId)},
+            {std::string(attr::networkType), setup_.networkType},
+        });
+
+        // Create MeterProvider with the shared resource, then attach reader.
+        meterProvider_ = metrics_sdk::MeterProviderFactory::Create(
+            std::make_unique<metrics_sdk::ViewRegistry>(), resourceAttrs);
+        meterProvider_->AddMetricReader(std::move(reader));
+
+        // Histogram view: SpanMetrics-compatible bucket boundaries (ms) so
+        // histogram instruments align with the collector's SpanMetrics. The
+        // view is created with an EMPTY name so it applies the buckets WITHOUT
+        // renaming instruments — a non-empty view name would collapse every
+        // matching histogram (ios_latency, rpc_size, rpc_time, pathfind_*)
+        // into a single series under that one name.
+        auto histogramSelector = metrics_sdk::InstrumentSelectorFactory::Create(
+            metrics_sdk::InstrumentType::kHistogram, "*", "ms");
+        // Meter selector MUST match the meter name used by getMeter() and the
+        // beast OTelCollector (kMeterName = "xrpld"); otherwise this histogram
+        // view never applies and duration histograms fall back to the SDK
+        // default boundaries instead of these SpanMetrics-aligned buckets.
+        auto meterSelector =
+            metrics_sdk::MeterSelectorFactory::Create(std::string(kMeterName), "", "");
+        auto histogramConfig = std::make_shared<metrics_sdk::HistogramAggregationConfig>();
+        histogramConfig->boundaries_ =
+            std::vector<double>{1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 5000.0};
+        auto histogramView = metrics_sdk::ViewFactory::Create(
+            "",  // empty name: keep each instrument's own name, only set buckets
+            "SpanMetrics-compatible histogram buckets",
+            metrics_sdk::AggregationType::kHistogram,
+            histogramConfig);
+
+        meterProvider_->AddView(
+            std::move(histogramSelector), std::move(meterSelector), std::move(histogramView));
+
+        // Publish as the global meter provider so developers (and the beast
+        // OTelCollector shim) reach the same pipeline.
+        metrics_api::Provider::SetMeterProvider(
+            opentelemetry::nostd::shared_ptr<metrics_api::MeterProvider>(meterProvider_));
+    }
+
     void
     stop() override
     {
@@ -363,6 +512,18 @@ public:
                 opentelemetry::nostd::shared_ptr<trace_api::TracerProvider>(
                     new trace_api::NoopTracerProvider()));
         }
+
+        if (meterProvider_)
+        {
+            // Mirror the tracer teardown: bounded flush, drop our provider,
+            // then restore a noop global provider. Same shutdown-order
+            // caveat as sdkProvider_ above applies to getMeter() callers.
+            meterProvider_->ForceFlush(std::chrono::milliseconds(5000));
+            meterProvider_.reset();
+            metrics_api::Provider::SetMeterProvider(
+                opentelemetry::nostd::shared_ptr<metrics_api::MeterProvider>(
+                    new metrics_api::NoopMeterProvider()));
+        }
         JLOG(journal_.info()) << "Telemetry stopped";
     }
 
@@ -418,6 +579,17 @@ public:
         return sdkProvider_->GetTracer(std::string(name));
     }
 
+    opentelemetry::nostd::shared_ptr<metrics_api::Meter>
+    getMeter(std::string_view name = kMeterName) override
+    {
+        if (!meterProvider_)
+        {
+            return metrics_api::Provider::GetMeterProvider()->GetMeter(
+                std::string(name), std::string(kMeterVersion));
+        }
+        return meterProvider_->GetMeter(std::string(name), std::string(kMeterVersion));
+    }
+
     opentelemetry::nostd::shared_ptr<trace_api::Span>
     startSpan(std::string_view name, trace_api::SpanKind kind) override
     {