rippled/.github/workflows/telemetry-validation.yml

# Telemetry Validation CI Workflow
#
# Builds rippled with telemetry enabled, runs the multi-node workload
# harness, validates all telemetry data, and runs performance benchmarks.
#
# This is a separate workflow from the main CI. It runs:
#   - On manual dispatch (workflow_dispatch)
#   - On pushes to telemetry-related branches
#
# The workflow is intentionally heavyweight (builds rippled, starts Docker
# services, runs a multi-node cluster) — it validates the full telemetry
# stack end-to-end rather than individual unit tests.
#
# Architecture: three jobs to leverage cached dependencies:
#   0. linux-image-tag — reads the CI image tag from the build matrix so this
#      workflow cannot drift onto a different compiler than the main CI.
#   1. build-xrpld — runs on a self-hosted runner inside the same container
#      image the main CI uses. This ensures Conan packages are fetched from
#      the XRPLF remote instead of built from source, and ccache hits the
#      remote cache.
#   2. validate-telemetry — runs on ubuntu-latest (which has Docker) to
#      launch the telemetry stack (OTel collector, Prometheus, Tempo, etc.)
#      and validate the full pipeline end-to-end.

name: Telemetry Validation

on:
  workflow_dispatch:
    # NOTE: rpc_rate / rpc_duration / tx_tps / tx_duration have NO effect.
    # They are forwarded to run-full-validation.sh, which parses them into
    # shell variables and never reads them again — load shape comes entirely
    # from --profile and docker/telemetry/workload/workload-profiles.json.
    # They are kept (and labelled) rather than removed so existing dispatch
    # bookmarks and any saved input sets do not break. To change the load,
    # edit or add a profile in workload-profiles.json.
    inputs:
      rpc_rate:
        description: "UNUSED — has no effect. Load shape comes from the workload profile."
        required: false
        default: "50"
      rpc_duration:
        description: "UNUSED — has no effect. Load shape comes from the workload profile."
        required: false
        default: "120"
      tx_tps:
        description: "UNUSED — has no effect. Load shape comes from the workload profile."
        required: false
        default: "5"
      tx_duration:
        description: "UNUSED — has no effect. Load shape comes from the workload profile."
        required: false
        default: "120"
      run_benchmark:
        description: "Run performance benchmarks (the only input that changes behaviour)"
        required: false
        type: boolean
        default: false

  push:
    # No branches filter, deliberately. Branch names are not something this
    # repository controls, so gating on one decides whether telemetry gets
    # validated by what a branch is CALLED rather than by what it CHANGED. The
    # previous list ("pratik/otel-phase*", "feature/otel-*",
    # "feature/telemetry-*") silently excluded every other name, and because
    # GitHub ANDs the branch and path filters the effect was total: pushes to
    # pratik/otel-sync-diagnostics matched the paths below but not the branch
    # glob, so this workflow was never dispatched there at all -- not queued,
    # not skipped, no run to look at. Two rounds of harness fixes on that branch
    # produced no signal before anyone noticed. The paths below already express
    # the real question, which is whether a change can affect telemetry.
    #
    # Keep these globs pointing at paths that actually exist. Two earlier
    # entries (include/xrpl/basics/Telemetry*.h, src/xrpld/app/misc/Telemetry*)
    # matched zero tracked files, so a pure C++ telemetry change never
    # triggered this workflow on push — only edits under docker/telemetry/**
    # or to this file did.
    paths:
      # This workflow, and the harness it runs.
      - ".github/workflows/telemetry-validation.yml"
      - "docker/telemetry/**"
      # The telemetry modules themselves.
      - "include/xrpl/telemetry/**"
      - "src/libxrpl/telemetry/**"
      - "src/xrpld/telemetry/**"
      # beast::insight, whose gauges and counters this workflow asserts on.
      # Both halves: the interface headers decide what the OTel collector can
      # publish, so editing one moves the metric surface as surely as editing
      # the implementation does.
      - "include/xrpl/beast/insight/**"
      - "src/libxrpl/beast/insight/**"
      # The span-name and metric-name constants. These are the wire contract the
      # harness asserts against by literal string, so a rename here compiles
      # cleanly and silently empties the assertions. Matched by filename rather
      # than directory because the convention colocates each header with the
      # class it serves: they live under consensus/, overlay/, app/ledger/,
      # app/main/, app/misc/, rpc/ and tx/, not under a telemetry/ directory.
      - "**/*SpanNames.h"
      - "**/*MetricNames.h"
      # The GTests that pin those same constants.
      - "src/tests/libxrpl/telemetry/**"
      # The checkers that gate this surface. otel-naming derives the legal
      # attribute and metric names from the headers above; the telemetry scripts
      # enforce the histogram-bucket parity and regression-bound rules the
      # harness depends on.
      - ".github/scripts/otel-naming/**"
      - ".github/scripts/telemetry/**"

concurrency:
  group: telemetry-validation-${{ github.ref }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

env:
  BUILD_DIR: build

jobs:
  # ── Job 0: Resolve the CI image tag ────────────────────────────────
  # The tag is pinned once, alongside the build matrix, in linux.json. Reading
  # it here rather than hardcoding a second copy means this workflow always
  # builds in the same image (and therefore the same compiler) as the main CI.
  # A hardcoded copy silently went stale and left this job on gcc 13 after the
  # rest of CI moved to gcc 15, which broke the build on code the main CI
  # compiled fine.
  linux-image-tag:
    runs-on: ubuntu-latest
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Read nix image tag
        id: tag
        run: echo "tag=$(jq -r .image_tag .github/scripts/strategy-matrix/linux.json)" >>"${GITHUB_OUTPUT}"

  # ── Job 1: Build xrpld in the same container the main CI uses ──────
  # This ensures Conan binary packages are fetched from the XRPLF remote
  # (matching package IDs) and ccache hits the remote compilation cache.
  build-xrpld:
    name: Build xrpld
    needs: linux-image-tag
    runs-on: [self-hosted, Linux, X64, heavy]
    container: ghcr.io/xrplf/xrpld/nix-debian:${{ needs.linux-image-tag.outputs.tag }}
    timeout-minutes: 60
    env:
      CCACHE_NAMESPACE: telemetry-validation
      CCACHE_REMOTE_ONLY: true
      CCACHE_REMOTE_STORAGE: http://cache.dev.ripplex.io:8080|layout=bazel
      CCACHE_SLOPPINESS: include_file_ctime,include_file_mtime
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Prepare runner
        uses: XRPLF/actions/prepare-runner@c00c22ada3bd6bcda48fcb0d62fbbab49fec8a0f
        with:
          enable_ccache: ${{ github.repository_owner == 'XRPLF' }}

      - name: Print build environment
        uses: XRPLF/actions/print-build-env@59dec886e4afb05a1724443af08baccbc045b574

      - name: Get number of processors
        uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf
        id: nproc
        with:
          subtract: 2

      # The nix image ships several toolchains, so CC/CXX must be set
      # explicitly for Conan to detect the intended one. gcc matches the
      # debian gcc-release config the main CI builds.
      - name: Set compiler environment
        uses: ./.github/actions/set-compiler-env
        with:
          compiler: gcc

      - name: Setup Conan
        uses: ./.github/actions/setup-conan

      - name: Build dependencies
        uses: ./.github/actions/build-deps
        with:
          build_nproc: ${{ steps.nproc.outputs.nproc }}
          build_type: Release
          log_verbosity: verbose

      # telemetry is passed explicitly even though the CMake option and the
      # Conan recipe both default it on. The whole point of this workflow is to
      # exercise telemetry, so it should not silently depend on a default it
      # does not control: if that default ever flips, every span and metric
      # assertion would fail for a reason no log names. Stated here, a build
      # without the dependency fails loudly instead, because CMakeLists.txt
      # does find_package(opentelemetry-cpp CONFIG REQUIRED) under this option.
      - name: Configure CMake
        working-directory: ${{ env.BUILD_DIR }}
        run: |
          cmake \
              -G Ninja \
              -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
              -DCMAKE_BUILD_TYPE=Release \
              -Dtelemetry=ON \
              ..

      - name: Build xrpld
        working-directory: ${{ env.BUILD_DIR }}
        env:
          BUILD_NPROC: ${{ steps.nproc.outputs.nproc }}
        run: |
          cmake \
              --build . \
              --config Release \
              --parallel "${BUILD_NPROC}" \
              --target xrpld

      - name: Show ccache statistics
        if: ${{ github.repository_owner == 'XRPLF' }}
        run: ccache --show-stats -vv

      - name: Upload xrpld binary
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: xrpld-telemetry
          path: ${{ env.BUILD_DIR }}/xrpld
          retention-days: 1
          if-no-files-found: error

  # ── Job 2: Run telemetry validation on ubuntu-latest (has Docker) ──
  validate-telemetry:
    name: Telemetry Stack Validation
    needs: build-xrpld
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Python dependencies
        run: pip3 install -r docker/telemetry/workload/requirements.txt

      - name: Download xrpld binary
        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
        with:
          name: xrpld-telemetry
          path: ${{ env.BUILD_DIR }}

      - name: Make binaries and scripts executable
        run: |
          chmod +x ${{ env.BUILD_DIR }}/xrpld
          chmod +x docker/telemetry/workload/*.sh

      - name: Run full telemetry validation
        id: validation
        env:
          RPC_RATE: ${{ github.event.inputs.rpc_rate || '50' }}
          RPC_DURATION: ${{ github.event.inputs.rpc_duration || '120' }}
          TX_TPS: ${{ github.event.inputs.tx_tps || '5' }}
          TX_DURATION: ${{ github.event.inputs.tx_duration || '120' }}
          RUN_BENCHMARK: ${{ github.event.inputs.run_benchmark }}
        run: |
          # The four rate/duration flags below are inert (see the
          # workflow_dispatch inputs note): run-full-validation.sh parses them
          # and never reads them. Load shape comes from the default
          # --profile full-validation. They are still passed so the flags stay
          # exercised if they are ever wired up.
          ARGS="--xrpld ${{ env.BUILD_DIR }}/xrpld"
          ARGS="$ARGS --rpc-rate $RPC_RATE"
          ARGS="$ARGS --rpc-duration $RPC_DURATION"
          ARGS="$ARGS --tx-tps $TX_TPS"
          ARGS="$ARGS --tx-duration $TX_DURATION"
          if [ "$RUN_BENCHMARK" = "true" ]; then
              ARGS="$ARGS --with-benchmark"
          fi
          docker/telemetry/workload/run-full-validation.sh $ARGS
        # continue-on-error allows subsequent steps (artifact upload,
        # summary printing) to run even if validation fails. The final
        # "Check validation result" step re-checks steps.validation.outcome
        # (the pre-continue-on-error result) and fails the job properly.
        continue-on-error: true

      - name: Upload validation reports
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-validation-reports
          path: /tmp/xrpld-validation/reports/
          retention-days: 30

      # Keyed on the validation step's own outcome, not job status. The step
      # above sets continue-on-error, so the job is not failing at this point
      # and `if: failure()` never fires -- which silently skipped these logs on
      # every failed run, and they are the only record of why a node did not
      # reach consensus.
      #
      # stdout.log matters as much as debug.log: a node that dies before its
      # log sink opens writes no debug.log at all, so stdout is the only place
      # its reason survives. A run that timed out at 4/5 nodes was left
      # undiagnosable because that file was not collected.
      - name: Upload node logs
        if: always() && steps.validation.outcome != 'success'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: xrpld-node-logs
          path: |
            /tmp/xrpld-validation/node*/debug.log
            /tmp/xrpld-validation/node*/stdout.log
            /tmp/xrpld-validation/*.log
          retention-days: 7
          if-no-files-found: warn

      - name: Print validation summary
        if: always()
        run: |
          REPORT="/tmp/xrpld-validation/reports/validation-report.json"
          if [ -f "$REPORT" ]; then
              echo "## Telemetry Validation Results" >>"$GITHUB_STEP_SUMMARY"
              echo "" >>"$GITHUB_STEP_SUMMARY"
              TOTAL=$(jq '.summary.total' "$REPORT")
              PASSED=$(jq '.summary.passed' "$REPORT")
              FAILED=$(jq '.summary.failed' "$REPORT")
              echo "| Metric | Value |" >>"$GITHUB_STEP_SUMMARY"
              echo "|--------|-------|" >>"$GITHUB_STEP_SUMMARY"
              echo "| Total Checks | $TOTAL |" >>"$GITHUB_STEP_SUMMARY"
              echo "| Passed | $PASSED |" >>"$GITHUB_STEP_SUMMARY"
              echo "| Failed | $FAILED |" >>"$GITHUB_STEP_SUMMARY"
              echo "" >>"$GITHUB_STEP_SUMMARY"
              if [ "$FAILED" -gt 0 ]; then
                  echo "### Failed Checks" >>"$GITHUB_STEP_SUMMARY"
                  echo "" >>"$GITHUB_STEP_SUMMARY"
                  jq -r '.checks[] | select(.passed == false) | "- **\(.name)**: \(.message)"' "$REPORT" >>"$GITHUB_STEP_SUMMARY"
              fi
          fi

      # Publishes captured OTel timings + regression report to the Step Summary.
      # When the committed baseline is a placeholder, emits a fenced JSON block
      # that can be copy-pasted directly into baselines/baseline-timings.json.
      # When the baseline is populated, summarises the top regressions so the
      # PR author sees the failure reason without downloading artifacts.
      - name: Print regression summary
        if: always()
        run: |
          set -euo pipefail
          TIMINGS="/tmp/xrpld-validation/reports/timings.json"
          REGRESSION="/tmp/xrpld-validation/reports/regression-report.json"
          BASELINE="docker/telemetry/workload/baselines/baseline-timings.json"

          if [ ! -f "$TIMINGS" ]; then
              echo "## Regression Gate: no timings captured" >>"$GITHUB_STEP_SUMMARY"
              echo "::warning::capture_timings.py did not produce timings.json — regression gate was not evaluated."
              exit 0
          fi

          if [ ! -f "$BASELINE" ]; then
              echo "## Regression Gate: baseline file missing" >>"$GITHUB_STEP_SUMMARY"
              echo "::error::baselines/baseline-timings.json not found in checkout"
              exit 1
          fi

          # NOTE: do NOT use `jq -e` here. With -e, jq exits non-zero when the
          # filter's result is boolean false — which is the normal case for a
          # populated (non-placeholder) baseline — and that would be
          # misreported as a parse failure. Plain `jq -r` exits 0 on any valid
          # JSON, so a real non-zero exit genuinely means malformed JSON.
          IS_PLACEHOLDER=$(jq -r '.placeholder == true or (.metrics | length == 0)' "$BASELINE") || {
              echo "::error::Failed to parse baseline JSON"
              exit 1
          }

          echo "## OTel Timings Regression Gate" >>"$GITHUB_STEP_SUMMARY"
          echo "" >>"$GITHUB_STEP_SUMMARY"

          if [ "$IS_PLACEHOLDER" = "true" ]; then
              echo "### Paste into \`baselines/baseline-timings.json\`" >>"$GITHUB_STEP_SUMMARY"
              echo "" >>"$GITHUB_STEP_SUMMARY"
              echo "The committed baseline is a placeholder. Open a PR replacing" \
                  "its contents with the JSON block below to activate the" \
                  "regression gate." >>"$GITHUB_STEP_SUMMARY"
              echo "" >>"$GITHUB_STEP_SUMMARY"
              echo '```json' >>"$GITHUB_STEP_SUMMARY"
              cat "$TIMINGS" >>"$GITHUB_STEP_SUMMARY"
              echo '```' >>"$GITHUB_STEP_SUMMARY"
          elif [ -f "$REGRESSION" ]; then
              REGR_COUNT=$(jq -e '.summary.regressions' "$REGRESSION") || REGR_COUNT=0
              IMPR_COUNT=$(jq -e '.summary.improvements' "$REGRESSION") || IMPR_COUNT=0
              TOTAL=$(jq -e '.summary.total' "$REGRESSION") || TOTAL=0
              echo "| Stat | Count |" >>"$GITHUB_STEP_SUMMARY"
              echo "|------|-------|" >>"$GITHUB_STEP_SUMMARY"
              echo "| Metrics compared | $TOTAL |" >>"$GITHUB_STEP_SUMMARY"
              echo "| Regressions | $REGR_COUNT |" >>"$GITHUB_STEP_SUMMARY"
              echo "| Improvements | $IMPR_COUNT |" >>"$GITHUB_STEP_SUMMARY"
              echo "" >>"$GITHUB_STEP_SUMMARY"
              if [ "$REGR_COUNT" -gt 0 ]; then
                  echo "### Regressions" >>"$GITHUB_STEP_SUMMARY"
                  echo "" >>"$GITHUB_STEP_SUMMARY"
                  echo "| Metric | Baseline | Current | Δ | % | Unit |" >>"$GITHUB_STEP_SUMMARY"
                  echo "|--------|---------:|--------:|--:|--:|------|" >>"$GITHUB_STEP_SUMMARY"
                  jq -r '.metrics[] | select(.regressed) | "| \(.key) | \(.baseline) | \(.current) | \(.delta) | \(.pct_change)% | \(.unit) |"' \
                      "$REGRESSION" >>"$GITHUB_STEP_SUMMARY"
              fi
          fi

      - name: Cleanup
        if: always()
        run: |
          docker/telemetry/workload/run-full-validation.sh --cleanup 2>/dev/null || true

      - name: Check validation result
        if: steps.validation.outcome == 'failure'
        run: |
          echo "Telemetry validation failed. Check the uploaded reports for details."
          exit 1