mirror of
https://github.com/XRPLF/rippled.git
synced 2026-07-30 18:40:28 +00:00
817c773162
Two gaps meant the last failure produced no evidence of its cause. The node-log upload was gated on `if: failure()`, but the validation step sets continue-on-error, so the job is not failing at that point and the condition never fired. Every failed run silently skipped the one artifact that records why a node did not reach consensus. It now keys on the validation step's own outcome, and also collects the harness logs. Transaction failures were logged at DEBUG, which CI does not enable, so a run where all 3052 submissions failed on a refused connection reported nothing about it. The first occurrence of each distinct failure kind is now a warning and repeats stay at DEBUG, so one refused connection says so once instead of 3052 times.
349 lines
14 KiB
YAML
349 lines
14 KiB
YAML
# Telemetry Validation CI Workflow
|
|
#
|
|
# Builds rippled with telemetry enabled, runs the multi-node workload
|
|
# harness, validates all telemetry data, and runs performance benchmarks.
|
|
#
|
|
# This is a separate workflow from the main CI. It runs:
|
|
# - On manual dispatch (workflow_dispatch)
|
|
# - On pushes to telemetry-related branches
|
|
#
|
|
# The workflow is intentionally heavyweight (builds rippled, starts Docker
|
|
# services, runs a multi-node cluster) — it validates the full telemetry
|
|
# stack end-to-end rather than individual unit tests.
|
|
#
|
|
# Architecture: three jobs to leverage cached dependencies:
|
|
# 0. linux-image-tag — reads the CI image tag from the build matrix so this
|
|
# workflow cannot drift onto a different compiler than the main CI.
|
|
# 1. build-xrpld — runs on a self-hosted runner inside the same container
|
|
# image the main CI uses. This ensures Conan packages are fetched from
|
|
# the XRPLF remote instead of built from source, and ccache hits the
|
|
# remote cache.
|
|
# 2. validate-telemetry — runs on ubuntu-latest (which has Docker) to
|
|
# launch the telemetry stack (OTel collector, Prometheus, Tempo, etc.)
|
|
# and validate the full pipeline end-to-end.
|
|
|
|
name: Telemetry Validation
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
rpc_rate:
|
|
description: "RPC load rate (requests per second)"
|
|
required: false
|
|
default: "50"
|
|
rpc_duration:
|
|
description: "RPC load duration (seconds)"
|
|
required: false
|
|
default: "120"
|
|
tx_tps:
|
|
description: "Transaction submit rate (TPS)"
|
|
required: false
|
|
default: "5"
|
|
tx_duration:
|
|
description: "Transaction submit duration (seconds)"
|
|
required: false
|
|
default: "120"
|
|
run_benchmark:
|
|
description: "Run performance benchmarks"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
|
|
push:
|
|
branches:
|
|
- "pratik/otel-phase*"
|
|
- "feature/otel-*"
|
|
- "feature/telemetry-*"
|
|
paths:
|
|
- ".github/workflows/telemetry-validation.yml"
|
|
- "docker/telemetry/**"
|
|
- "include/xrpl/basics/Telemetry*.h"
|
|
- "src/xrpld/app/misc/Telemetry*"
|
|
|
|
concurrency:
|
|
group: telemetry-validation-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
env:
|
|
BUILD_DIR: build
|
|
|
|
jobs:
|
|
# ── Job 0: Resolve the CI image tag ────────────────────────────────
|
|
# The tag is pinned once, alongside the build matrix, in linux.json. Reading
|
|
# it here rather than hardcoding a second copy means this workflow always
|
|
# builds in the same image (and therefore the same compiler) as the main CI.
|
|
# A hardcoded copy silently went stale and left this job on gcc 13 after the
|
|
# rest of CI moved to gcc 15, which broke the build on code the main CI
|
|
# compiled fine.
|
|
linux-image-tag:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
tag: ${{ steps.tag.outputs.tag }}
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Read nix image tag
|
|
id: tag
|
|
run: echo "tag=$(jq -r .image_tag .github/scripts/strategy-matrix/linux.json)" >>"${GITHUB_OUTPUT}"
|
|
|
|
# ── Job 1: Build xrpld in the same container the main CI uses ──────
|
|
# This ensures Conan binary packages are fetched from the XRPLF remote
|
|
# (matching package IDs) and ccache hits the remote compilation cache.
|
|
build-xrpld:
|
|
name: Build xrpld
|
|
needs: linux-image-tag
|
|
runs-on: [self-hosted, Linux, X64, heavy]
|
|
container: ghcr.io/xrplf/xrpld/nix-debian:${{ needs.linux-image-tag.outputs.tag }}
|
|
timeout-minutes: 60
|
|
env:
|
|
CCACHE_NAMESPACE: telemetry-validation
|
|
CCACHE_REMOTE_ONLY: true
|
|
CCACHE_REMOTE_STORAGE: http://cache.dev.ripplex.io:8080|layout=bazel
|
|
CCACHE_SLOPPINESS: include_file_ctime,include_file_mtime
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Prepare runner
|
|
uses: XRPLF/actions/prepare-runner@c00c22ada3bd6bcda48fcb0d62fbbab49fec8a0f
|
|
with:
|
|
enable_ccache: ${{ github.repository_owner == 'XRPLF' }}
|
|
|
|
- name: Print build environment
|
|
uses: XRPLF/actions/print-build-env@59dec886e4afb05a1724443af08baccbc045b574
|
|
|
|
- name: Get number of processors
|
|
uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf
|
|
id: nproc
|
|
with:
|
|
subtract: 2
|
|
|
|
# The nix image ships several toolchains, so CC/CXX must be set
|
|
# explicitly for Conan to detect the intended one. gcc matches the
|
|
# debian gcc-release config the main CI builds.
|
|
- name: Set compiler environment
|
|
uses: ./.github/actions/set-compiler-env
|
|
with:
|
|
compiler: gcc
|
|
|
|
- name: Setup Conan
|
|
uses: ./.github/actions/setup-conan
|
|
|
|
- name: Build dependencies
|
|
uses: ./.github/actions/build-deps
|
|
with:
|
|
build_nproc: ${{ steps.nproc.outputs.nproc }}
|
|
build_type: Release
|
|
log_verbosity: verbose
|
|
|
|
- name: Configure CMake
|
|
working-directory: ${{ env.BUILD_DIR }}
|
|
run: |
|
|
cmake \
|
|
-G Ninja \
|
|
-DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
|
|
-DCMAKE_BUILD_TYPE=Release \
|
|
..
|
|
|
|
- name: Build xrpld
|
|
working-directory: ${{ env.BUILD_DIR }}
|
|
env:
|
|
BUILD_NPROC: ${{ steps.nproc.outputs.nproc }}
|
|
run: |
|
|
cmake \
|
|
--build . \
|
|
--config Release \
|
|
--parallel "${BUILD_NPROC}" \
|
|
--target xrpld
|
|
|
|
- name: Show ccache statistics
|
|
if: ${{ github.repository_owner == 'XRPLF' }}
|
|
run: ccache --show-stats -vv
|
|
|
|
- name: Upload xrpld binary
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: xrpld-telemetry
|
|
path: ${{ env.BUILD_DIR }}/xrpld
|
|
retention-days: 1
|
|
if-no-files-found: error
|
|
|
|
# ── Job 2: Run telemetry validation on ubuntu-latest (has Docker) ──
|
|
validate-telemetry:
|
|
name: Telemetry Stack Validation
|
|
needs: build-xrpld
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Install Python dependencies
|
|
run: pip3 install -r docker/telemetry/workload/requirements.txt
|
|
|
|
- name: Download xrpld binary
|
|
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
|
|
with:
|
|
name: xrpld-telemetry
|
|
path: ${{ env.BUILD_DIR }}
|
|
|
|
- name: Make binaries and scripts executable
|
|
run: |
|
|
chmod +x ${{ env.BUILD_DIR }}/xrpld
|
|
chmod +x docker/telemetry/workload/*.sh
|
|
|
|
- name: Run full telemetry validation
|
|
id: validation
|
|
env:
|
|
RPC_RATE: ${{ github.event.inputs.rpc_rate || '50' }}
|
|
RPC_DURATION: ${{ github.event.inputs.rpc_duration || '120' }}
|
|
TX_TPS: ${{ github.event.inputs.tx_tps || '5' }}
|
|
TX_DURATION: ${{ github.event.inputs.tx_duration || '120' }}
|
|
RUN_BENCHMARK: ${{ github.event.inputs.run_benchmark }}
|
|
run: |
|
|
ARGS="--xrpld ${{ env.BUILD_DIR }}/xrpld --skip-loki"
|
|
ARGS="$ARGS --rpc-rate $RPC_RATE"
|
|
ARGS="$ARGS --rpc-duration $RPC_DURATION"
|
|
ARGS="$ARGS --tx-tps $TX_TPS"
|
|
ARGS="$ARGS --tx-duration $TX_DURATION"
|
|
if [ "$RUN_BENCHMARK" = "true" ]; then
|
|
ARGS="$ARGS --with-benchmark"
|
|
fi
|
|
docker/telemetry/workload/run-full-validation.sh $ARGS
|
|
# continue-on-error allows subsequent steps (artifact upload,
|
|
# summary printing) to run even if validation fails. The final
|
|
# "Check validation result" step re-checks steps.validation.outcome
|
|
# (the pre-continue-on-error result) and fails the job properly.
|
|
continue-on-error: true
|
|
|
|
- name: Upload validation reports
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: telemetry-validation-reports
|
|
path: /tmp/xrpld-validation/reports/
|
|
retention-days: 30
|
|
|
|
# Keyed on the validation step's own outcome, not job status. The step
|
|
# above sets continue-on-error, so the job is not failing at this point
|
|
# and `if: failure()` never fires -- which silently skipped these logs on
|
|
# every failed run, and they are the only record of why a node did not
|
|
# reach consensus.
|
|
- name: Upload node logs
|
|
if: always() && steps.validation.outcome != 'success'
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: xrpld-node-logs
|
|
path: |
|
|
/tmp/xrpld-validation/node*/debug.log
|
|
/tmp/xrpld-validation/*.log
|
|
retention-days: 7
|
|
if-no-files-found: warn
|
|
|
|
- name: Print validation summary
|
|
if: always()
|
|
run: |
|
|
REPORT="/tmp/xrpld-validation/reports/validation-report.json"
|
|
if [ -f "$REPORT" ]; then
|
|
echo "## Telemetry Validation Results" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
TOTAL=$(jq '.summary.total' "$REPORT")
|
|
PASSED=$(jq '.summary.passed' "$REPORT")
|
|
FAILED=$(jq '.summary.failed' "$REPORT")
|
|
echo "| Metric | Value |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "|--------|-------|" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Total Checks | $TOTAL |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Passed | $PASSED |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Failed | $FAILED |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
if [ "$FAILED" -gt 0 ]; then
|
|
echo "### Failed Checks" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
jq -r '.checks[] | select(.passed == false) | "- **\(.name)**: \(.message)"' "$REPORT" >>"$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
fi
|
|
|
|
# Publishes captured OTel timings + regression report to the Step Summary.
|
|
# When the committed baseline is a placeholder, emits a fenced JSON block
|
|
# that can be copy-pasted directly into baselines/baseline-timings.json.
|
|
# When the baseline is populated, summarises the top regressions so the
|
|
# PR author sees the failure reason without downloading artifacts.
|
|
- name: Print regression summary
|
|
if: always()
|
|
run: |
|
|
set -euo pipefail
|
|
TIMINGS="/tmp/xrpld-validation/reports/timings.json"
|
|
REGRESSION="/tmp/xrpld-validation/reports/regression-report.json"
|
|
BASELINE="docker/telemetry/workload/baselines/baseline-timings.json"
|
|
|
|
if [ ! -f "$TIMINGS" ]; then
|
|
echo "## Regression Gate: no timings captured" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "::warning::capture_timings.py did not produce timings.json — regression gate was not evaluated."
|
|
exit 0
|
|
fi
|
|
|
|
if [ ! -f "$BASELINE" ]; then
|
|
echo "## Regression Gate: baseline file missing" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "::error::baselines/baseline-timings.json not found in checkout"
|
|
exit 1
|
|
fi
|
|
|
|
# NOTE: do NOT use `jq -e` here. With -e, jq exits non-zero when the
|
|
# filter's result is boolean false — which is the normal case for a
|
|
# populated (non-placeholder) baseline — and that would be
|
|
# misreported as a parse failure. Plain `jq -r` exits 0 on any valid
|
|
# JSON, so a real non-zero exit genuinely means malformed JSON.
|
|
IS_PLACEHOLDER=$(jq -r '.placeholder == true or (.metrics | length == 0)' "$BASELINE") || {
|
|
echo "::error::Failed to parse baseline JSON"
|
|
exit 1
|
|
}
|
|
|
|
echo "## OTel Timings Regression Gate" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
|
|
if [ "$IS_PLACEHOLDER" = "true" ]; then
|
|
echo "### Paste into \`baselines/baseline-timings.json\`" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "The committed baseline is a placeholder. Open a PR replacing" \
|
|
"its contents with the JSON block below to activate the" \
|
|
"regression gate." >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
echo '```json' >>"$GITHUB_STEP_SUMMARY"
|
|
cat "$TIMINGS" >>"$GITHUB_STEP_SUMMARY"
|
|
echo '```' >>"$GITHUB_STEP_SUMMARY"
|
|
elif [ -f "$REGRESSION" ]; then
|
|
REGR_COUNT=$(jq -e '.summary.regressions' "$REGRESSION") || REGR_COUNT=0
|
|
IMPR_COUNT=$(jq -e '.summary.improvements' "$REGRESSION") || IMPR_COUNT=0
|
|
TOTAL=$(jq -e '.summary.total' "$REGRESSION") || TOTAL=0
|
|
echo "| Stat | Count |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "|------|-------|" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Metrics compared | $TOTAL |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Regressions | $REGR_COUNT |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Improvements | $IMPR_COUNT |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
if [ "$REGR_COUNT" -gt 0 ]; then
|
|
echo "### Regressions" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Metric | Baseline | Current | Δ | % | Unit |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "|--------|---------:|--------:|--:|--:|------|" >>"$GITHUB_STEP_SUMMARY"
|
|
jq -r '.metrics[] | select(.regressed) | "| \(.key) | \(.baseline) | \(.current) | \(.delta) | \(.pct_change)% | \(.unit) |"' \
|
|
"$REGRESSION" >>"$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
fi
|
|
|
|
- name: Cleanup
|
|
if: always()
|
|
run: |
|
|
docker/telemetry/workload/run-full-validation.sh --cleanup 2>/dev/null || true
|
|
|
|
- name: Check validation result
|
|
if: steps.validation.outcome == 'failure'
|
|
run: |
|
|
echo "Telemetry validation failed. Check the uploaded reports for details."
|
|
exit 1
|