ci: Add clang to nix images (#7308)

Co-authored-by: semgrep-companion-app[bot] <218312740+semgrep-companion-app[bot]@users.noreply.github.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

.github/workflows/build-nix-image.yml

@@ -6,14 +6,16 @@ on:
       - develop
     paths:
       - ".github/workflows/build-nix-image.yml"
-      - "docker/nix.Dockerfile"
+      - ".github/workflows/reusable-build-docker-image.yml"
+      - "docker/**"
       - "flake.nix"
       - "flake.lock"
       - "nix/**"
   pull_request:
     paths:
       - ".github/workflows/build-nix-image.yml"
-      - "docker/nix.Dockerfile"
+      - ".github/workflows/reusable-build-docker-image.yml"
+      - "docker/**"
       - "flake.nix"
       - "flake.lock"
       - "nix/**"
@@ -27,75 +29,81 @@ defaults:
   run:
     shell: bash
 
-env:
-  UBUNTU_VERSION: "20.04"
-  RHEL_VERSION: "9"
-  DEBIAN_VERSION: "bookworm"
-
 jobs:
   build:
-    name: Build and push Nix image (${{ matrix.distro }})
+    name: Build ${{ matrix.distro.name }} (${{ matrix.target.platform }})
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        # The base images are the oldest supported version of each distro
+        # that we want to build images for.
+        distro:
+          - name: nixos
+            base_image: nixos/nix:latest
+          - name: ubuntu
+            base_image: ubuntu:20.04
+          - name: rhel
+            base_image: registry.access.redhat.com/ubi9/ubi:latest
+          - name: debian
+            base_image: debian:bookworm
+        target:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+    uses: ./.github/workflows/reusable-build-docker-image.yml
+    with:
+      image_name: ghcr.io/xrplf/xrpld/nix-${{ matrix.distro.name }}
+      dockerfile: docker/nix.Dockerfile
+      base_image: ${{ matrix.distro.base_image }}
+      platform: ${{ matrix.target.platform }}
+      runner: ${{ matrix.target.runner }}
+      push: ${{ github.event_name == 'push' }}
+
+  merge:
+    name: Merge ${{ matrix.distro }} manifest
+    needs: build
+    if: github.event_name == 'push'
     runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
     strategy:
+      fail-fast: false
       matrix:
-        include:
-          - distro: nixos
-          - distro: ubuntu
-          - distro: rhel
-          - distro: debian
+        distro: [nixos, ubuntu, rhel, debian]
+    env:
+      IMAGE_NAME: ghcr.io/xrplf/xrpld/nix-${{ matrix.distro }}
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Determine base image
-        id: vars
-        run: |
-          case "${{ matrix.distro }}" in
-            nixos)
-              echo "base_image=nixos/nix:latest" >> $GITHUB_OUTPUT
-              ;;
-            ubuntu)
-              echo "base_image=ubuntu:${UBUNTU_VERSION}" >> $GITHUB_OUTPUT
-              ;;
-            rhel)
-              echo "base_image=registry.access.redhat.com/ubi${RHEL_VERSION}/ubi:latest" >> $GITHUB_OUTPUT
-              ;;
-            debian)
-              echo "base_image=debian:${DEBIAN_VERSION}" >> $GITHUB_OUTPUT
-              ;;
-          esac
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
 
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
+        with:
+          images: ${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,prefix=sha-,format=short
+            type=raw,value=latest
+
       - name: Login to GitHub Container Registry
-        if: github.event_name == 'push'
         uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Docker metadata
-        id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
-        with:
-          images: ghcr.io/xrplf/ci/nix-${{ matrix.distro }}
-          tags: |
-            type=sha,prefix=sha-,format=short
-            type=raw,value=latest
+      - name: Create multi-arch manifests
+        run: |
+          for tag in $(jq -cr '.tags[]' <<< "$DOCKER_METADATA_OUTPUT_JSON"); do
+            docker buildx imagetools create -t "$tag" "${tag}-amd64" "${tag}-arm64"
+          done
 
-      - name: Build and push
-        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
-        with:
-          context: .
-          file: docker/nix.Dockerfile
-          platforms: linux/amd64
-          push: ${{ github.event_name == 'push' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: BASE_IMAGE=${{ steps.vars.outputs.base_image }}
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect "${IMAGE_NAME}:${{ steps.meta.outputs.version }}"

.github/workflows/reusable-build-docker-image.yml

@@ -0,0 +1,89 @@
+# Build a single-platform Docker image. On push, the image is pushed to
+# GHCR with arch-suffixed tags (e.g. `:latest-amd64`, `:sha-abc-amd64`)
+# so the calling workflow can stitch per-arch builds into a multi-arch
+# manifest without needing to pass digests around.
+name: Reusable build Docker image (single platform)
+
+on:
+  workflow_call:
+    inputs:
+      image_name:
+        description: "Full image name without tag (e.g. 'ghcr.io/xrplf/xrpld/nix-ubuntu')"
+        required: true
+        type: string
+      dockerfile:
+        description: "Path to the Dockerfile, relative to the repository root"
+        required: true
+        type: string
+      base_image:
+        description: "Value passed to the Dockerfile as the BASE_IMAGE build arg"
+        required: true
+        type: string
+      platform:
+        description: "Docker platform string, e.g. linux/amd64"
+        required: true
+        type: string
+      runner:
+        description: "GitHub Actions runner label to build on"
+        required: true
+        type: string
+      push:
+        description: "Whether to push the image to GHCR"
+        required: true
+        type: boolean
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build:
+    name: Build (${{ inputs.platform }})
+    runs-on: ${{ inputs.runner }}
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Determine arch
+        id: vars
+        env:
+          PLATFORM: ${{ inputs.platform }}
+        run: |
+          echo "arch=${PLATFORM##*/}" >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+
+      - name: Login to GitHub Container Registry
+        if: inputs.push
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
+        with:
+          images: ${{ inputs.image_name }}
+          tags: |
+            type=sha,prefix=sha-,format=short
+            type=raw,value=latest
+          flavor: |
+            suffix=-${{ steps.vars.outputs.arch }},onlatest=true
+
+      - name: Build and push
+        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
+        with:
+          context: .
+          file: ${{ inputs.dockerfile }}
+          platforms: ${{ inputs.platform }}
+          push: ${{ inputs.push }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: BASE_IMAGE=${{ inputs.base_image }}

cspell.config.yaml

@@ -199,11 +199,13 @@ words:
   - nonxrp
   - noreplace
   - noripple
+  - nostdinc
   - notifempty
   - nudb
   - nullptr
   - nunl
   - Nyffenegger
+  - onlatest
   - ostr
   - pargs
   - partitioner
@@ -298,6 +300,7 @@ words:
   - unauthorizing
   - unergonomic
   - unfetched
+  - unfindable
   - unflatten
   - unfund
   - unimpair

docker/check-sanitizers.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Sanity-check that the sanitizer runtimes shipped with g++/clang++ work
+# end-to-end against the system loader: compile each example with both
+# compilers, run it, and confirm the expected diagnostic is emitted.
+
+set -eo pipefail
+
+cpp_files_dir="${1:?usage: $0 <cpp_files_dir>}"
+
+case "$(uname -m)" in
+    x86_64)  loader=/lib64/ld-linux-x86-64.so.2 ;;
+    aarch64) loader=/lib/ld-linux-aarch64.so.1 ;;
+    *) echo "Unsupported arch: $(uname -m)" >&2; exit 1 ;;
+esac
+
+declare -A sanitize=(
+    [asan]="-fsanitize=address"
+    [tsan]="-fsanitize=thread"
+    [ubsan]="-fsanitize=undefined"
+)
+declare -A expect=(
+    [asan]="heap-use-after-free"
+    [tsan]="data race"
+    [ubsan]="signed integer overflow"
+)
+
+for compiler in g++ clang++; do
+    for name in asan tsan ubsan; do
+        bin="/tmp/${name}-${compiler}"
+        echo "=== Build ${name} with ${compiler} ==="
+        "$compiler" -std=c++20 -O1 -g ${sanitize[$name]} \
+            -Wl,--dynamic-linker=$loader \
+            "${cpp_files_dir}/${name}.cpp" -o "$bin"
+        echo "=== Run ${name}-${compiler} ==="
+        output=$("$bin" 2>&1) || true
+        echo "$output"
+        echo "$output" | grep -q "${expect[$name]}" \
+            || { echo "expected '${expect[$name]}' from $bin"; exit 1; }
+        rm -f "$bin"
+    done
+done

docker/cpp_files/asan.cpp

@@ -0,0 +1,28 @@
+#include <atomic>
+#include <cstddef>
+#include <iostream>
+
+#if defined(__clang__) || defined(__GNUC__)
+__attribute__((noinline))
+#elif defined(_MSC_VER)
+__declspec(noinline)
+#endif
+int
+read_after_free(volatile int* array, std::size_t index)
+{
+    std::atomic_signal_fence(std::memory_order_seq_cst);
+    int value = array[index];
+    std::atomic_signal_fence(std::memory_order_seq_cst);
+    return value;
+}
+
+int
+main()
+{
+    int* array = new int[5]{10, 20, 30, 40, 50};
+    delete[] array;
+
+    std::cout << "Value at index 2: " << read_after_free(array, 2) << std::endl;
+
+    return 0;
+}

docker/cpp_files/tsan.cpp

@@ -0,0 +1,26 @@
+#include <iostream>
+#include <thread>
+
+static int kCounter = 0;
+
+void
+increment()
+{
+    for (int i = 0; i < 100'000; ++i)
+    {
+        ++kCounter;
+    }
+}
+
+int
+main()
+{
+    std::thread t1(increment);
+    std::thread t2(increment);
+
+    t1.join();
+    t2.join();
+
+    std::cout << "Final counter value: " << kCounter << std::endl;
+    return 0;
+}

docker/cpp_files/ubsan.cpp

@@ -0,0 +1,13 @@
+#include <iostream>
+#include <limits>
+
+int
+main()
+{
+    int maxInt = std::numeric_limits<int>::max();
+    int volatile one = 1;
+    std::cout << "Current max: " << maxInt << std::endl;
+    int overflowed = maxInt + one;
+    std::cout << "Overflowed result: " << overflowed << std::endl;
+    return 0;
+}

docker/nix.Dockerfile

@@ -45,8 +45,30 @@ COPY --from=builder /tmp/build/result /nix/ci-env
 
 ENV PATH="/nix/ci-env/bin:$PATH"
 
+# Externally-built dynamically-linked ELF binaries hard-code the loader path
+# (e.g. /lib64/ld-linux-x86-64.so.2) in their PT_INTERP header. Copy the
+# loader from the Nix store to that path when the base image doesn't already
+# provide one (i.e. on nixos/nix).
+RUN <<EOF
+case "$(uname -m)" in
+    x86_64)  target=/lib64/ld-linux-x86-64.so.2 ;;
+    aarch64) target=/lib/ld-linux-aarch64.so.1 ;;
+    *) echo "Unsupported arch: $(uname -m)" >&2; exit 1 ;;
+esac
+if [ ! -e "$target" ]; then
+    # Use the loader from the same glibc that gcc links libc against, so
+    # ld-linux and libc/libpthread share GLIBC_PRIVATE symbols at runtime.
+    src="$(dirname "$(gcc -print-file-name=libc.so.6)")/$(basename "$target")"
+    [ -e "$src" ] || { echo "ld-linux not found at $src" >&2; exit 1; }
+    mkdir -p "$(dirname "$target")"
+    cp "$src" "$target"
+fi
+EOF
+
 RUN <<EOF
 ccache --version
+clang --version
+clang++ --version
 clang-format --version
 cmake --version
 conan --version
@@ -64,3 +86,10 @@ python3 --version
 run-clang-tidy --help
 vim --version
 EOF
+
+# Sanity-check that the sanitizer runtimes shipped with g++/clang++ work
+# end-to-end against the system loader.
+COPY docker/cpp_files/ /tmp/cpp_files/
+COPY docker/check-sanitizers.sh /tmp/check-sanitizers.sh
+
+RUN grep -qi ubuntu /etc/os-release 2>/dev/null && /tmp/check-sanitizers.sh /tmp/cpp_files || true

flake.lock

@@ -15,7 +15,7 @@
         "type": "indirect"
       }
     },
-    "nixpkgs-glibc231": {
+    "nixpkgs-custom-glibc": {
       "flake": false,
       "locked": {
         "lastModified": 1593520194,
@@ -35,7 +35,7 @@
     "root": {
       "inputs": {
         "nixpkgs": "nixpkgs",
-        "nixpkgs-glibc231": "nixpkgs-glibc231"
+        "nixpkgs-custom-glibc": "nixpkgs-custom-glibc"
       }
     }
   },

flake.nix

@@ -6,16 +6,16 @@
     # version — matches the system libc on Ubuntu 20.04 LTS. Imported
     # manually (flake = false) because this revision predates nixpkgs'
     # own flake.nix.
-    nixpkgs-glibc231 = {
+    nixpkgs-custom-glibc = {
       url = "github:NixOS/nixpkgs/9cd98386a38891d1074fc18036b842dc4416f562";
       flake = false;
     };
   };
 
   outputs =
-    { nixpkgs, nixpkgs-glibc231, ... }:
+    { nixpkgs, nixpkgs-custom-glibc, ... }:
     let
-      forEachSystem = import ./nix/utils.nix { inherit nixpkgs nixpkgs-glibc231; };
+      forEachSystem = import ./nix/utils.nix { inherit nixpkgs nixpkgs-custom-glibc; };
     in
     {
       devShells = forEachSystem (import ./nix/devshell.nix);

nix/ci-env.nix

@@ -1,39 +1,102 @@
 {
   pkgs,
-  glibc231,
+  customGlibc,
   ...
 }:
 let
   inherit (import ./packages.nix { inherit pkgs; }) commonPackages;
+  inherit (pkgs) lib;
 
-  # binutils wrapped to emit binaries that reference glibc 2.31 (dynamic
-  # linker path, library search path, RPATH).
-  binutils231 = pkgs.wrapBintoolsWith {
+  # Underlying compiler toolchains to wrap. Bump these in one place to
+  # roll the whole environment forward.
+  customGccPackage = pkgs.gcc15;
+  customLlvmPackages = pkgs.llvmPackages_22;
+  customClangMajor = lib.versions.major (lib.getVersion customLlvmPackages.clang-unwrapped);
+
+  # binutils wrapped to emit binaries that reference the custom glibc
+  # (dynamic linker path, library search path, RPATH).
+  customBinutils = pkgs.wrapBintoolsWith {
     bintools = pkgs.binutils-unwrapped;
-    libc = glibc231;
+    libc = customGlibc;
   };
 
-  # Rebuild gcc 15 (specifically libstdc++ / libgcc_s) against glibc 2.31.
-  # The override swaps gcc15.cc's bootstrap stdenv for one that uses the
-  # existing gcc 15 binary but links against glibc 2.31, so the resulting
-  # compiler ships runtime libraries that only reference symbols available
-  # in glibc 2.31.
-  gcc15CcWithGlibc231 = pkgs.gcc15.cc.override {
+  # Rebuild gcc (specifically libstdc++ / libgcc_s) against the custom
+  # glibc. The override swaps gcc.cc's bootstrap stdenv for one that uses
+  # the existing gcc binary but links against the custom glibc, so the
+  # resulting compiler ships runtime libraries that only reference symbols
+  # available in that glibc.
+  customGccCc = customGccPackage.cc.override {
     stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv (
       pkgs.wrapCCWith {
-        cc = pkgs.gcc15.cc;
-        libc = glibc231;
-        bintools = binutils231;
+        cc = customGccPackage.cc;
+        libc = customGlibc;
+        bintools = customBinutils;
       }
     );
   };
 
-  # cc-wrapper around the rebuilt compiler, pointing at glibc 2.31 headers
-  # and libraries. This is what we actually expose to users.
-  gcc15WithGlibc231 = pkgs.wrapCCWith {
-    cc = gcc15CcWithGlibc231;
-    libc = glibc231;
-    bintools = binutils231;
+  # cc-wrapper around the rebuilt compiler, pointing at the custom glibc
+  # headers and libraries. This is what we actually expose to users.
+  customGcc = pkgs.wrapCCWith {
+    cc = customGccCc;
+    libc = customGlibc;
+    bintools = customBinutils;
+  };
+
+  # stdenv built around the rebuilt gcc / custom glibc. Used to rebuild
+  # compiler-rt below so its sanitizer runtimes see the custom glibc
+  # headers.
+  customStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customGcc;
+
+  # Rebuild compiler-rt against the custom glibc so the sanitizer runtimes
+  # don't use glibc symbols (or sysconf constants like _SC_SIGSTKSZ) that
+  # only exist in newer glibc versions. scudo is dropped because its CMake
+  # includes CheckAtomic with -nostdinc++ in CMAKE_REQUIRED_FLAGS, which
+  # makes std::atomic unfindable in our stdenv; we don't use scudo (only
+  # asan/ubsan/tsan etc.).
+  customCompilerRt =
+    (customLlvmPackages.compiler-rt.override {
+      stdenv = customStdenv;
+    }).overrideAttrs
+      (old: {
+        postPatch = (old.postPatch or "") + ''
+          substituteInPlace lib/CMakeLists.txt \
+            --replace-quiet 'add_subdirectory(scudo/standalone)' \
+                            '# scudo/standalone disabled in xrpld ci-env'
+        '';
+      });
+
+  # cc-wrapper around clang, pointing at the custom glibc headers and
+  # libraries. Reuses the rebuilt gcc for libstdc++ / libgcc_s so that
+  # C++ binaries produced by clang also only reference symbols available
+  # in the custom glibc. compiler-rt is wired into a resource-root so
+  # sanitizer runtimes (libclang_rt.*.a) are found at link time; this
+  # mirrors what nixpkgs does internally when building llvmPackages.clang.
+  customClang = pkgs.wrapCCWith {
+    cc = customLlvmPackages.clang-unwrapped;
+    libc = customGlibc;
+    bintools = customBinutils;
+    gccForLibs = customGccCc;
+    extraPackages = [ customCompilerRt ];
+    extraBuildCommands = ''
+      rsrc="$out/resource-root"
+      mkdir "$rsrc"
+      ln -s "${customLlvmPackages.clang-unwrapped.lib}/lib/clang/${customClangMajor}/include" "$rsrc/include"
+      ln -s "${customCompilerRt.out}/lib" "$rsrc/lib"
+      ln -s "${customCompilerRt.out}/share" "$rsrc/share" || true
+      echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags
+    '';
+  };
+
+  # Strip the generic cc/c++/cpp symlinks from the clang wrapper so it can
+  # coexist with the gcc wrapper in buildEnv. gcc remains the default
+  # compiler (cc/c++/cpp); clang is invoked explicitly as clang/clang++.
+  customClangForCiEnv = pkgs.symlinkJoin {
+    name = "clang-wrapper-custom-for-ci-env";
+    paths = [ customClang ];
+    postBuild = ''
+      rm -f $out/bin/cc $out/bin/c++ $out/bin/cpp
+    '';
   };
 
 in
@@ -41,8 +104,9 @@ in
   default = pkgs.buildEnv {
     name = "xrpld-ci-env";
     paths = commonPackages ++ [
-      gcc15WithGlibc231
-      binutils231
+      customGcc
+      customClangForCiEnv
+      customBinutils
     ];
     pathsToLink = [
       "/bin"

nix/packages.nix

@@ -17,6 +17,7 @@ in
     llvmPackages_22.clang-tools
     mold
     ninja
+    patchelf
     perl # needed for openssl
     pkg-config
     pre-commit

nix/utils.nix

@@ -1,4 +1,4 @@
-{ nixpkgs, nixpkgs-glibc231 }:
+{ nixpkgs, nixpkgs-custom-glibc }:
 function:
 nixpkgs.lib.genAttrs
   [
@@ -12,10 +12,10 @@ nixpkgs.lib.genAttrs
     function {
       pkgs = import nixpkgs { inherit system; };
       # glibc 2.31 — matches the system libc on Ubuntu 20.04 LTS. Sourced
-      # from the nixpkgs snapshot pinned via the `nixpkgs-glibc231` flake
-      # input, so the build uses the compiler from that snapshot
+      # from the nixpkgs snapshot pinned via the `nixpkgs-custom-glibc`
+      # flake input, so the build uses the compiler from that snapshot
       # (gcc 9.3.0) along with the matching patches, configure flags, and
       # hardening defaults.
-      glibc231 = (import nixpkgs-glibc231 { inherit system; }).glibc;
+      customGlibc = (import nixpkgs-custom-glibc { inherit system; }).glibc;
     }
   )