ci: Patch binaries in nix-based images and test in every distro (#7376)

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

docker/check-tool-versions.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Verify that every tool expected in the Nix CI env is present and runnable.
+set -euo pipefail
+
+ccache --version
+clang --version
+clang++ --version
+clang-format --version
+cmake --version
+conan --version
+g++ --version
+gcc --version
+gcovr --version
+git --version
+less --version
+make --version
+mold --version
+ninja --version
+perl --version
+pkg-config --version
+pre-commit --version
+python3 --version
+run-clang-tidy --help
+vim --version

docker/install-sanitizer-libs.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+# Install sanitizer runtime libraries required to run binaries compiled with:
+#   -fsanitize=address   → libasan.so.8
+#   -fsanitize=thread    → libtsan.so.2
+#   -fsanitize=undefined → libubsan.so.1
+#
+# The exact SONAMEs required depend on the compiler toolchain used to build the
+# test binaries (see nix/ci-env.nix). If the toolchain is bumped and SONAMEs
+# change, update the list below (or detect them from the binaries).
+#
+# Supported base images:
+#   debian:bookworm
+#   ubuntu:20.04
+#   rhel:9
+#   nixos/nix        — tests are skipped; this script is not called
+
+set -euo pipefail
+
+if [ ! -f /etc/os-release ]; then
+    echo "ERROR: /etc/os-release not found; cannot detect OS" >&2
+    exit 1
+fi
+
+# shellcheck source=/dev/null
+. /etc/os-release
+
+echo "Detected OS: ${ID} ${VERSION_ID:-}"
+
+case "${ID}" in
+    debian)
+        apt-get update -y
+        apt-get install -y --no-install-recommends \
+            libasan8 \
+            libtsan2 \
+            libubsan1
+
+        apt-get clean
+        rm -rf /var/lib/apt/lists/*
+        ;;
+
+    ubuntu)
+        apt-get update -y
+        apt-get install -y --no-install-recommends \
+            gnupg \
+            software-properties-common
+        add-apt-repository -y ppa:ubuntu-toolchain-r/test
+        apt-get update -y
+        apt-get install -y --no-install-recommends \
+            libasan8 \
+            libtsan2 \
+            libubsan1
+
+        apt-get clean
+        rm -rf /var/lib/apt/lists/*
+        ;;
+
+    rhel | centos | rocky | almalinux)
+        dnf install -y \
+            libasan8 \
+            libtsan2 \
+            libubsan
+
+        dnf clean -y all
+        rm -rf /var/cache/dnf/*
+        ;;
+
+    *)
+        echo "ERROR: unsupported OS '${ID}'. Supported: debian, ubuntu, rhel-family" >&2
+        exit 1
+        ;;
+esac
+
+# Verify that every expected library is now resolvable by the dynamic linker.
+missing=0
+for lib in libasan.so.8 libtsan.so.2 libubsan.so.1; do
+    if ldconfig -p | grep -q "${lib}"; then
+        echo "OK: ${lib} found"
+    else
+        echo "ERROR: ${lib} not found after installation" >&2
+        missing=$((missing + 1))
+    fi
+done
+
+if [ "${missing}" -ne 0 ]; then
+    echo "ERROR: ${missing} library/libraries missing" >&2
+    exit 1
+fi
+
+echo "All sanitizer runtime libraries installed successfully."

docker/nix.Dockerfile

@@ -32,7 +32,7 @@ FROM ${BASE_IMAGE} AS final
 ARG BASE_IMAGE
 
 # bash is not located at /bin/bash in nixos/nix, so we need to create a symlink to it.
-RUN if [ -d /nix ]; then \
+RUN if echo "${BASE_IMAGE}" | grep -qiE 'nixos'; then \
         ln -s /root/.nix-profile/bin/bash /bin/bash; \
     fi
 
@@ -65,38 +65,44 @@ if [ ! -e "${target}" ]; then
 fi
 EOF
 
-RUN <<EOF
-ccache --version
-clang --version
-clang++ --version
-clang-format --version
-cmake --version
-conan --version
-g++ --version
-gcc --version
-gcovr --version
-git --version
-make --version
-mold --version
-ninja --version
-perl --version
-pkg-config --version
-pre-commit --version
-python3 --version
-run-clang-tidy --help
-vim --version
-EOF
+COPY docker/check-tool-versions.sh /tmp/check-tool-versions.sh
+RUN /tmp/check-tool-versions.sh
 
-# Sanity-check that the sanitizer runtimes shipped with g++/clang++ are able to build binaries
+# Sanity-check that the g++/clang++ are able to build binaries, including sanitizer-instrumented ones.
 COPY docker/test_files/cpp_sources/ /tmp/cpp_sources/
 COPY docker/test_files/compile-cpp-sources.sh /tmp/compile-cpp-sources.sh
 RUN /tmp/compile-cpp-sources.sh /tmp/cpp_sources /tmp/bins
 
-# Sanity-check that the built binaries are able to run.
-# We only support running the test binaries on Ubuntu and NixOS right now (will be fixed in the future)
-#
-# When build and test images will be separate, we will be to run on vanilla images.
-COPY docker/test_files/run-test-binaries.sh /tmp/run-test-binaries.sh
-RUN if echo "${BASE_IMAGE}" | grep -qiE '(ubuntu|nixos)'; then \
-        /tmp/run-test-binaries.sh /tmp/bins; \
+# Tester: start from a clean BASE_IMAGE, install sanitizer runtime libraries,
+# and run the compiled test binaries to verify they execute correctly.
+FROM ${BASE_IMAGE} AS tester
+
+ARG BASE_IMAGE
+
+# bash is not located at /bin/bash in nixos/nix, so we need to create a symlink to it.
+RUN if echo "${BASE_IMAGE}" | grep -qiE 'nixos'; then \
+        ln -s /root/.nix-profile/bin/bash /bin/bash; \
     fi
+
+SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]
+
+# Sanity-check that the built binaries run correctly in the vanilla base image, with the necessary sanitizer runtime libraries installed.
+COPY docker/install-sanitizer-libs.sh /tmp/install-sanitizer-libs.sh
+COPY docker/test_files/run-test-binaries.sh /tmp/run-test-binaries.sh
+COPY --from=final /tmp/bins /tmp/bins
+
+RUN <<EOF
+if echo "${BASE_IMAGE}" | grep -qiE 'nixos'; then
+    echo "Skipping runnning binaries on NixOS."
+else
+    /tmp/install-sanitizer-libs.sh
+    /tmp/run-test-binaries.sh /tmp/bins
+fi
+touch /tmp/tests-passed
+EOF
+
+# Output: the final image, gated on a successful test run in the tester stage.
+# Copying the sentinel from tester creates a hard build dependency: if the test
+# run above failed, this stage — and the overall build — fails too.
+FROM final
+COPY --from=tester /tmp/tests-passed /tmp/tests-passed

docker/test_files/compile-cpp-sources.sh

@@ -20,14 +20,21 @@ function compile() {
     local src="${src_dir}/${name}.cpp"
     local binary="${dst_dir}/${name}-${compiler}"
 
-    echo "=== Compile ${name} with ${compiler} ==="
-    cmd="${compiler} -std=c++23 -O1 -g \
+    echo "=== Compiling ${name} with ${compiler} ==="
+    # Always statically link libstdc++ so the test binary does not depend on
+    # the host's libstdc++.so.6 version.
+    local compile_cmd="${compiler} -std=c++23 -O1 -g \
         -pthread \
-        -Wl,--dynamic-linker=${loader} \
+        -static-libstdc++ \
         ${san_flag} \
         ${src} -o ${binary}"
-    echo "Command: ${cmd}"
-    eval "${cmd}"
+    echo "Compile cmd: ${compile_cmd}"
+    eval "${compile_cmd}"
+
+    echo "=== Patching ${binary} to use ${loader} as PT_INTERP ==="
+    local patch_cmd="patchelf --set-interpreter ${loader} --remove-rpath ${binary}"
+    echo "Patch cmd: ${patch_cmd}"
+    eval "${patch_cmd}"
 }
 
 declare -A sanitize=(

docker/test_files/run-test-binaries.sh

@@ -7,6 +7,8 @@ set -eo pipefail
 
 bins_dir="${1:?usage: $0 <bins_dir>}"
 
+failed_binaries=()
+
 # Run a binary and verify its exit code and output.
 # Usage: run <binary> <expected_output> <expected_rc>
 function run() {
@@ -18,27 +20,34 @@ function run() {
     out_file="$(mktemp)"
 
     echo "=== Run ${binary} ==="
-    local rc=0
-    "${binary}" >"${out_file}" 2>&1 || rc=$?
+    set +e
+    "${binary}" >"${out_file}" 2>&1
+    local rc=$?
+    set -e
 
     cat "${out_file}"
 
+    local failed=0
     if [ "${expected_rc}" = "nonzero" ]; then
         if [ "${rc}" -eq 0 ]; then
             echo "ERROR: expected non-zero exit code from ${binary}, got ${rc}" >&2
-            exit 1
+            failed=1
         fi
     elif [ "${rc}" -ne "${expected_rc}" ]; then
         echo "ERROR: expected exit code ${expected_rc} from ${binary}, got ${rc}" >&2
-        exit 1
+        failed=1
     fi
 
-    grep -q "${expected_output}" "${out_file}" ||
-        {
-            echo "ERROR: expected '${expected_output}' from ${binary}" >&2
-            exit 1
-        }
-    echo "OK: '${expected_output}' detected"
+    if ! grep -q "${expected_output}" "${out_file}"; then
+        echo "ERROR: expected '${expected_output}' from ${binary}" >&2
+        failed=1
+    fi
+
+    if [ "${failed}" -eq 0 ]; then
+        echo "OK: '${expected_output}' detected"
+    else
+        failed_binaries+=("${binary}")
+    fi
 }
 
 declare -A expect=(
@@ -52,6 +61,15 @@ declare -A expect=(
 for compiler in g++ clang++; do
     for name in regular asan tsan ubsan; do
         binary="${bins_dir}/${name}-${compiler}"
+
+        if [ "${name}" = "tsan" ] && [ "${compiler}" = "g++" ] &&
+            grep -qi 'debian' /etc/os-release 2>/dev/null &&
+            [ "$(uname -m)" = "aarch64" ]; then
+            echo "=== Skipping ${binary} (tsan-g++ unsupported on Debian ARM64) ==="
+            echo "    NOTE: to enable it, add --security-opt seccomp=unconfined to your docker run command"
+            continue
+        fi
+
         if [ "${name}" = "regular" ]; then
             expected_rc=0
         else
@@ -60,3 +78,9 @@ for compiler in g++ clang++; do
         run "${binary}" "${expect[$name]}" "${expected_rc}"
     done
 done
+
+if [ "${#failed_binaries[@]}" -gt 0 ]; then
+    echo "ERROR: the following binaries failed:" >&2
+    printf '  %s\n' "${failed_binaries[@]}" >&2
+    exit 1
+fi

nix/packages.nix

@@ -15,6 +15,7 @@ in
     git
     gnumake
     llvmPackages_22.clang-tools
+    less # needed for git diff
     mold
     ninja
     patchelf