mirror of
https://github.com/XRPLF/rippled.git
synced 2026-09-30 16:58:30 +00:00
8521b96d85
The regression baseline is bootstrapped by copying a CI artifact. The workflow
tested only that timings.json existed, then printed it verbatim under a heading
inviting the reader to paste it in as the new baseline.
capture_timings.py writes that file and only then enforces --min-capture-ratio,
so an incomplete capture leaves a file that exists but covers fewer keys than
the contract declares. The verdict lived in CAPTURE_EXIT, a shell variable local
to run-full-validation.sh that no other program could read. So on a placeholder
baseline plus a thin capture, CI offered an incomplete artifact as the next
baseline, and pasting it narrowed the gate with nothing reporting that it had.
That is the failure shape this harness keeps producing: a degraded result that
looks exactly like a good one.
The artifact now carries its own completeness, next to metrics:
"capture": { "declared": 20, "captured": 20, "min_ratio": 0.5, "complete": true }
complete is the same condition the producer exits 0 on, computed once with the
exit code read off it, so the flag and the status cannot drift apart. Any
consumer can now tell a complete capture from a thin one, not just CI.
Both paste-me paths refuse rather than warn: the workflow prints the counts and
an error annotation with no JSON, and the comparator explains on stderr while
leaving stdout empty, so a redirect cannot produce a plausible-looking file. A
warning above a copyable block is still a copyable block, and a reader who has
just hit a red gate is already predisposed to re-baseline. A missing capture
block fails closed.
Refusal is scoped to bootstrapping a baseline, not to comparing against one, so
artifacts captured before this change still replay: verified against the run the
current baseline came from, which carries no capture block and still reports 0
regressions. An injected regression is still caught, and the gated surface is
unchanged at 20 keys with 5 excluded.
462 lines
22 KiB
YAML
462 lines
22 KiB
YAML
# Telemetry Validation CI Workflow
|
|
#
|
|
# Builds rippled with telemetry enabled, runs the multi-node workload
|
|
# harness, validates all telemetry data, and runs performance benchmarks.
|
|
#
|
|
# This is a separate workflow from the main CI. It runs:
|
|
# - On manual dispatch (workflow_dispatch)
|
|
# - On pushes to telemetry-related branches
|
|
#
|
|
# The workflow is intentionally heavyweight (builds rippled, starts Docker
|
|
# services, runs a multi-node cluster) — it validates the full telemetry
|
|
# stack end-to-end rather than individual unit tests.
|
|
#
|
|
# Architecture: three jobs to leverage cached dependencies:
|
|
# 0. linux-image-tag — reads the CI image tag from the build matrix so this
|
|
# workflow cannot drift onto a different compiler than the main CI.
|
|
# 1. build-xrpld — runs on a self-hosted runner inside the same container
|
|
# image the main CI uses. This ensures Conan packages are fetched from
|
|
# the XRPLF remote instead of built from source, and ccache hits the
|
|
# remote cache.
|
|
# 2. validate-telemetry — runs on ubuntu-latest (which has Docker) to
|
|
# launch the telemetry stack (OTel collector, Prometheus, Tempo, etc.)
|
|
# and validate the full pipeline end-to-end.
|
|
|
|
name: Telemetry Validation
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
# NOTE: rpc_rate / rpc_duration / tx_tps / tx_duration have NO effect.
|
|
# They are forwarded to run-full-validation.sh, which parses them into
|
|
# shell variables and never reads them again — load shape comes entirely
|
|
# from --profile and docker/telemetry/workload/workload-profiles.json.
|
|
# They are kept (and labelled) rather than removed so existing dispatch
|
|
# bookmarks and any saved input sets do not break. To change the load,
|
|
# edit or add a profile in workload-profiles.json.
|
|
inputs:
|
|
rpc_rate:
|
|
description: "UNUSED — has no effect. Load shape comes from the workload profile."
|
|
required: false
|
|
default: "50"
|
|
rpc_duration:
|
|
description: "UNUSED — has no effect. Load shape comes from the workload profile."
|
|
required: false
|
|
default: "120"
|
|
tx_tps:
|
|
description: "UNUSED — has no effect. Load shape comes from the workload profile."
|
|
required: false
|
|
default: "5"
|
|
tx_duration:
|
|
description: "UNUSED — has no effect. Load shape comes from the workload profile."
|
|
required: false
|
|
default: "120"
|
|
run_benchmark:
|
|
description: "Run performance benchmarks (the only input that changes behaviour)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
|
|
push:
|
|
# No branches filter, deliberately. Branch names are not something this
|
|
# repository controls, so gating on one decides whether telemetry gets
|
|
# validated by what a branch is CALLED rather than by what it CHANGED. The
|
|
# previous list ("pratik/otel-phase*", "feature/otel-*",
|
|
# "feature/telemetry-*") silently excluded every other name, and because
|
|
# GitHub ANDs the branch and path filters the effect was total: pushes to
|
|
# pratik/otel-sync-diagnostics matched the paths below but not the branch
|
|
# glob, so this workflow was never dispatched there at all -- not queued,
|
|
# not skipped, no run to look at. Two rounds of harness fixes on that branch
|
|
# produced no signal before anyone noticed. The paths below already express
|
|
# the real question, which is whether a change can affect telemetry.
|
|
#
|
|
# Keep these globs pointing at paths that actually exist. Two earlier
|
|
# entries (include/xrpl/basics/Telemetry*.h, src/xrpld/app/misc/Telemetry*)
|
|
# matched zero tracked files, so a pure C++ telemetry change never
|
|
# triggered this workflow on push — only edits under docker/telemetry/**
|
|
# or to this file did.
|
|
paths:
|
|
# This workflow, and the harness it runs.
|
|
- ".github/workflows/telemetry-validation.yml"
|
|
- "docker/telemetry/**"
|
|
# The telemetry modules themselves.
|
|
- "include/xrpl/telemetry/**"
|
|
- "src/libxrpl/telemetry/**"
|
|
- "src/xrpld/telemetry/**"
|
|
# beast::insight, whose gauges and counters this workflow asserts on.
|
|
# Both halves: the interface headers decide what the OTel collector can
|
|
# publish, so editing one moves the metric surface as surely as editing
|
|
# the implementation does.
|
|
- "include/xrpl/beast/insight/**"
|
|
- "src/libxrpl/beast/insight/**"
|
|
# The span-name and metric-name constants. These are the wire contract the
|
|
# harness asserts against by literal string, so a rename here compiles
|
|
# cleanly and silently empties the assertions. Matched by filename rather
|
|
# than directory because the convention colocates each header with the
|
|
# class it serves: they live under consensus/, overlay/, app/ledger/,
|
|
# app/main/, app/misc/, rpc/ and tx/, not under a telemetry/ directory.
|
|
- "**/*SpanNames.h"
|
|
- "**/*MetricNames.h"
|
|
# The GTests that pin those same constants.
|
|
- "src/tests/libxrpl/telemetry/**"
|
|
# The checkers that gate this surface. otel-naming derives the legal
|
|
# attribute and metric names from the headers above; the telemetry scripts
|
|
# enforce the histogram-bucket parity and regression-bound rules the
|
|
# harness depends on.
|
|
- ".github/scripts/otel-naming/**"
|
|
- ".github/scripts/telemetry/**"
|
|
|
|
concurrency:
|
|
group: telemetry-validation-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
env:
|
|
BUILD_DIR: build
|
|
|
|
jobs:
|
|
# ── Job 0: Resolve the CI image tag ────────────────────────────────
|
|
# The tag is pinned once, alongside the build matrix, in linux.json. Reading
|
|
# it here rather than hardcoding a second copy means this workflow always
|
|
# builds in the same image (and therefore the same compiler) as the main CI.
|
|
# A hardcoded copy silently went stale and left this job on gcc 13 after the
|
|
# rest of CI moved to gcc 15, which broke the build on code the main CI
|
|
# compiled fine.
|
|
linux-image-tag:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
tag: ${{ steps.tag.outputs.tag }}
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Read nix image tag
|
|
id: tag
|
|
run: echo "tag=$(jq -r .image_tag .github/scripts/strategy-matrix/linux.json)" >>"${GITHUB_OUTPUT}"
|
|
|
|
# ── Job 1: Build xrpld in the same container the main CI uses ──────
|
|
# This ensures Conan binary packages are fetched from the XRPLF remote
|
|
# (matching package IDs) and ccache hits the remote compilation cache.
|
|
build-xrpld:
|
|
name: Build xrpld
|
|
needs: linux-image-tag
|
|
runs-on: [self-hosted, Linux, X64, heavy]
|
|
container: ghcr.io/xrplf/xrpld/nix-debian:${{ needs.linux-image-tag.outputs.tag }}
|
|
timeout-minutes: 60
|
|
env:
|
|
CCACHE_NAMESPACE: telemetry-validation
|
|
CCACHE_REMOTE_ONLY: true
|
|
CCACHE_REMOTE_STORAGE: http://cache.dev.ripplex.io:8080|layout=bazel
|
|
CCACHE_SLOPPINESS: include_file_ctime,include_file_mtime
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Prepare runner
|
|
uses: XRPLF/actions/prepare-runner@c00c22ada3bd6bcda48fcb0d62fbbab49fec8a0f
|
|
with:
|
|
enable_ccache: ${{ github.repository_owner == 'XRPLF' }}
|
|
|
|
- name: Print build environment
|
|
uses: XRPLF/actions/print-build-env@59dec886e4afb05a1724443af08baccbc045b574
|
|
|
|
- name: Get number of processors
|
|
uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf
|
|
id: nproc
|
|
with:
|
|
subtract: 2
|
|
|
|
# The nix image ships several toolchains, so CC/CXX must be set
|
|
# explicitly for Conan to detect the intended one. gcc matches the
|
|
# debian gcc-release config the main CI builds.
|
|
- name: Set compiler environment
|
|
uses: ./.github/actions/set-compiler-env
|
|
with:
|
|
compiler: gcc
|
|
|
|
- name: Setup Conan
|
|
uses: ./.github/actions/setup-conan
|
|
|
|
- name: Build dependencies
|
|
uses: ./.github/actions/build-deps
|
|
with:
|
|
build_nproc: ${{ steps.nproc.outputs.nproc }}
|
|
build_type: Release
|
|
log_verbosity: verbose
|
|
|
|
# telemetry is passed explicitly even though the CMake option and the
|
|
# Conan recipe both default it on. The whole point of this workflow is to
|
|
# exercise telemetry, so it should not silently depend on a default it
|
|
# does not control: if that default ever flips, every span and metric
|
|
# assertion would fail for a reason no log names. Stated here, a build
|
|
# without the dependency fails loudly instead, because CMakeLists.txt
|
|
# does find_package(opentelemetry-cpp CONFIG REQUIRED) under this option.
|
|
- name: Configure CMake
|
|
working-directory: ${{ env.BUILD_DIR }}
|
|
run: |
|
|
cmake \
|
|
-G Ninja \
|
|
-DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
|
|
-DCMAKE_BUILD_TYPE=Release \
|
|
-Dtelemetry=ON \
|
|
..
|
|
|
|
- name: Build xrpld
|
|
working-directory: ${{ env.BUILD_DIR }}
|
|
env:
|
|
BUILD_NPROC: ${{ steps.nproc.outputs.nproc }}
|
|
run: |
|
|
cmake \
|
|
--build . \
|
|
--config Release \
|
|
--parallel "${BUILD_NPROC}" \
|
|
--target xrpld
|
|
|
|
- name: Show ccache statistics
|
|
if: ${{ github.repository_owner == 'XRPLF' }}
|
|
run: ccache --show-stats -vv
|
|
|
|
- name: Upload xrpld binary
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: xrpld-telemetry
|
|
path: ${{ env.BUILD_DIR }}/xrpld
|
|
retention-days: 1
|
|
if-no-files-found: error
|
|
|
|
# ── Job 2: Run telemetry validation on ubuntu-latest (has Docker) ──
|
|
validate-telemetry:
|
|
name: Telemetry Stack Validation
|
|
needs: build-xrpld
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Install Python dependencies
|
|
run: pip3 install -r docker/telemetry/workload/requirements.txt
|
|
|
|
- name: Download xrpld binary
|
|
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
|
|
with:
|
|
name: xrpld-telemetry
|
|
path: ${{ env.BUILD_DIR }}
|
|
|
|
- name: Make binaries and scripts executable
|
|
run: |
|
|
chmod +x ${{ env.BUILD_DIR }}/xrpld
|
|
chmod +x docker/telemetry/workload/*.sh
|
|
|
|
- name: Run full telemetry validation
|
|
id: validation
|
|
env:
|
|
RPC_RATE: ${{ github.event.inputs.rpc_rate || '50' }}
|
|
RPC_DURATION: ${{ github.event.inputs.rpc_duration || '120' }}
|
|
TX_TPS: ${{ github.event.inputs.tx_tps || '5' }}
|
|
TX_DURATION: ${{ github.event.inputs.tx_duration || '120' }}
|
|
RUN_BENCHMARK: ${{ github.event.inputs.run_benchmark }}
|
|
run: |
|
|
# The four rate/duration flags below are inert (see the
|
|
# workflow_dispatch inputs note): run-full-validation.sh parses them
|
|
# and never reads them. Load shape comes from the default
|
|
# --profile full-validation. They are still passed so the flags stay
|
|
# exercised if they are ever wired up.
|
|
ARGS="--xrpld ${{ env.BUILD_DIR }}/xrpld"
|
|
ARGS="$ARGS --rpc-rate $RPC_RATE"
|
|
ARGS="$ARGS --rpc-duration $RPC_DURATION"
|
|
ARGS="$ARGS --tx-tps $TX_TPS"
|
|
ARGS="$ARGS --tx-duration $TX_DURATION"
|
|
if [ "$RUN_BENCHMARK" = "true" ]; then
|
|
ARGS="$ARGS --with-benchmark"
|
|
fi
|
|
docker/telemetry/workload/run-full-validation.sh $ARGS
|
|
# continue-on-error allows subsequent steps (artifact upload,
|
|
# summary printing) to run even if validation fails. The final
|
|
# "Check validation result" step re-checks steps.validation.outcome
|
|
# (the pre-continue-on-error result) and fails the job properly.
|
|
continue-on-error: true
|
|
|
|
- name: Upload validation reports
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: telemetry-validation-reports
|
|
path: /tmp/xrpld-validation/reports/
|
|
retention-days: 30
|
|
|
|
# Keyed on the validation step's own outcome, not job status. The step
|
|
# above sets continue-on-error, so the job is not failing at this point
|
|
# and `if: failure()` never fires -- which silently skipped these logs on
|
|
# every failed run, and they are the only record of why a node did not
|
|
# reach consensus.
|
|
#
|
|
# stdout.log matters as much as debug.log: a node that dies before its
|
|
# log sink opens writes no debug.log at all, so stdout is the only place
|
|
# its reason survives. A run that timed out at 4/5 nodes was left
|
|
# undiagnosable because that file was not collected.
|
|
- name: Upload node logs
|
|
if: always() && steps.validation.outcome != 'success'
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: xrpld-node-logs
|
|
path: |
|
|
/tmp/xrpld-validation/node*/debug.log
|
|
/tmp/xrpld-validation/node*/stdout.log
|
|
/tmp/xrpld-validation/*.log
|
|
retention-days: 7
|
|
if-no-files-found: warn
|
|
|
|
- name: Print validation summary
|
|
if: always()
|
|
run: |
|
|
REPORT="/tmp/xrpld-validation/reports/validation-report.json"
|
|
if [ -f "$REPORT" ]; then
|
|
echo "## Telemetry Validation Results" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
TOTAL=$(jq '.summary.total' "$REPORT")
|
|
PASSED=$(jq '.summary.passed' "$REPORT")
|
|
FAILED=$(jq '.summary.failed' "$REPORT")
|
|
echo "| Metric | Value |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "|--------|-------|" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Total Checks | $TOTAL |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Passed | $PASSED |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Failed | $FAILED |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
if [ "$FAILED" -gt 0 ]; then
|
|
echo "### Failed Checks" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
jq -r '.checks[] | select(.passed == false) | "- **\(.name)**: \(.message)"' "$REPORT" >>"$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
fi
|
|
|
|
# Publishes captured OTel timings + regression report to the Step Summary.
|
|
# When the committed baseline is a placeholder AND the capture is
|
|
# complete, emits a fenced JSON block that can be copy-pasted directly
|
|
# into baselines/baseline-timings.json. An incomplete capture is named as
|
|
# such and its JSON withheld — see the comment on that branch below.
|
|
# When the baseline is populated, summarises the top regressions so the
|
|
# PR author sees the failure reason without downloading artifacts.
|
|
- name: Print regression summary
|
|
if: always()
|
|
run: |
|
|
set -euo pipefail
|
|
TIMINGS="/tmp/xrpld-validation/reports/timings.json"
|
|
REGRESSION="/tmp/xrpld-validation/reports/regression-report.json"
|
|
BASELINE="docker/telemetry/workload/baselines/baseline-timings.json"
|
|
|
|
if [ ! -f "$TIMINGS" ]; then
|
|
echo "## Regression Gate: no timings captured" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "::warning::capture_timings.py did not produce timings.json — regression gate was not evaluated."
|
|
exit 0
|
|
fi
|
|
|
|
if [ ! -f "$BASELINE" ]; then
|
|
echo "## Regression Gate: baseline file missing" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "::error::baselines/baseline-timings.json not found in checkout"
|
|
exit 1
|
|
fi
|
|
|
|
# NOTE: do NOT use `jq -e` here. With -e, jq exits non-zero when the
|
|
# filter's result is boolean false — which is the normal case for a
|
|
# populated (non-placeholder) baseline — and that would be
|
|
# misreported as a parse failure. Plain `jq -r` exits 0 on any valid
|
|
# JSON, so a real non-zero exit genuinely means malformed JSON.
|
|
IS_PLACEHOLDER=$(jq -r '.placeholder == true or (.metrics | length == 0)' "$BASELINE") || {
|
|
echo "::error::Failed to parse baseline JSON"
|
|
exit 1
|
|
}
|
|
|
|
# Whether the capture is usable as baseline material is the capture's
|
|
# own verdict, carried in the artifact by capture_timings.py, which
|
|
# computes it against --min-capture-ratio. It is READ here, never
|
|
# re-derived: a second copy of the ratio rule in shell would be a
|
|
# second source of truth and would drift from the producer.
|
|
#
|
|
# `// false` covers both an artifact written before the block existed
|
|
# and a truncated one. Neither can prove it is complete, so neither is
|
|
# offered — the whole point is that a degraded capture must not look
|
|
# like a good one. Same `jq -r` reasoning as the baseline parse above.
|
|
CAPTURE_COMPLETE=$(jq -r '.capture.complete // false' "$TIMINGS") || {
|
|
echo "::error::Failed to parse timings JSON"
|
|
exit 1
|
|
}
|
|
if [ "$(jq -r 'has("capture")' "$TIMINGS")" = "true" ]; then
|
|
CAPTURE_COUNTS=$(jq -r '"\(.capture.captured)/\(.capture.declared)"' "$TIMINGS")
|
|
CAPTURE_SHORTFALL="only **$CAPTURE_COUNTS** declared metrics came back, below the capture's own minimum ratio"
|
|
else
|
|
CAPTURE_COUNTS="unknown"
|
|
CAPTURE_SHORTFALL="the artifact carries no \`capture\` block, so it predates completeness reporting and cannot state what it captured"
|
|
fi
|
|
|
|
echo "## OTel Timings Regression Gate" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
|
|
if [ "$IS_PLACEHOLDER" = "true" ] && [ "$CAPTURE_COMPLETE" != "true" ]; then
|
|
# The placeholder path is the ONLY route to a committed baseline,
|
|
# which makes it the one place an incomplete capture does lasting
|
|
# damage: pasted in, it silently narrows the gate to the keys that
|
|
# happened to come back. So the JSON is withheld rather than
|
|
# printed with a caveat — a warning above a copyable block is
|
|
# still a copyable block. The counts are shown so the reader knows
|
|
# how thin it was, and the artifact is still uploaded for anyone
|
|
# who needs to inspect it deliberately.
|
|
echo "### Baseline NOT refreshable from this run" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "The committed baseline is a placeholder, so this run would" \
|
|
"normally print a block to paste into" \
|
|
"\`baselines/baseline-timings.json\`. It is withheld because" \
|
|
"$CAPTURE_SHORTFALL, so the JSON may describe metrics that" \
|
|
"were never measured. Pasting it would narrow the gate to" \
|
|
"whichever keys were captured, with nothing reporting that" \
|
|
"it had narrowed." >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "Fix the capture first — the usual cause is Prometheus not" \
|
|
"being scraped for long enough, or nodes not reaching" \
|
|
"consensus — then re-run. See the \`timings.json\` artifact's" \
|
|
"\`capture\` block for the exact counts." >>"$GITHUB_STEP_SUMMARY"
|
|
echo "::error::Timing capture is incomplete ($CAPTURE_COUNTS metrics) — no baseline block printed. Do not refresh the baseline from this run."
|
|
elif [ "$IS_PLACEHOLDER" = "true" ]; then
|
|
echo "### Paste into \`baselines/baseline-timings.json\`" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "The committed baseline is a placeholder. Open a PR replacing" \
|
|
"its contents with the JSON block below to activate the" \
|
|
"regression gate. The capture is complete" \
|
|
"($CAPTURE_COUNTS declared metrics)." >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
echo '```json' >>"$GITHUB_STEP_SUMMARY"
|
|
cat "$TIMINGS" >>"$GITHUB_STEP_SUMMARY"
|
|
echo '```' >>"$GITHUB_STEP_SUMMARY"
|
|
elif [ -f "$REGRESSION" ]; then
|
|
REGR_COUNT=$(jq -e '.summary.regressions' "$REGRESSION") || REGR_COUNT=0
|
|
IMPR_COUNT=$(jq -e '.summary.improvements' "$REGRESSION") || IMPR_COUNT=0
|
|
TOTAL=$(jq -e '.summary.total' "$REGRESSION") || TOTAL=0
|
|
echo "| Stat | Count |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "|------|-------|" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Metrics compared | $TOTAL |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Regressions | $REGR_COUNT |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Improvements | $IMPR_COUNT |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
if [ "$REGR_COUNT" -gt 0 ]; then
|
|
echo "### Regressions" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "| Metric | Baseline | Current | Δ | % | Unit |" >>"$GITHUB_STEP_SUMMARY"
|
|
echo "|--------|---------:|--------:|--:|--:|------|" >>"$GITHUB_STEP_SUMMARY"
|
|
jq -r '.metrics[] | select(.regressed) | "| \(.key) | \(.baseline) | \(.current) | \(.delta) | \(.pct_change)% | \(.unit) |"' \
|
|
"$REGRESSION" >>"$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
fi
|
|
|
|
- name: Cleanup
|
|
if: always()
|
|
run: |
|
|
docker/telemetry/workload/run-full-validation.sh --cleanup 2>/dev/null || true
|
|
|
|
- name: Check validation result
|
|
if: steps.validation.outcome == 'failure'
|
|
run: |
|
|
echo "Telemetry validation failed. Check the uploaded reports for details."
|
|
exit 1
|