From 4113b105a57483573cb3df165f0ee5d5e0728458 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 18 Aug 2026 23:34:16 +0000 Subject: [PATCH] build: Run nix macos builds in CI; deny nix store references (#8023) --- .cspell.config.yaml | 7 ++ .github/actions/setup-nix-env/action.yml | 69 +++++++++++ .github/scripts/strategy-matrix/generate.py | 10 +- .github/scripts/strategy-matrix/macos.json | 13 ++ .github/workflows/on-pr.yml | 5 + .github/workflows/on-trigger.yml | 5 + .../workflows/reusable-build-test-config.yml | 31 +++++ .github/workflows/reusable-build-test.yml | 1 + .github/workflows/upload-conan-deps.yml | 11 ++ bin/check-nix-store-refs.sh | 111 ++++++++++++++++++ docs/build/nix.md | 78 +++++++++++- docs/build/nix_troubleshooting.md | 88 ++++++++++++++ nix/ci-env.nix | 74 ++++-------- nix/darwin.nix | 80 +++++++++++++ nix/devshell.nix | 29 +++-- nix/docker/Dockerfile | 2 +- nix/{compilers.nix => linux.nix} | 55 +++++++-- 17 files changed, 591 insertions(+), 78 deletions(-) create mode 100644 .github/actions/setup-nix-env/action.yml create mode 100755 bin/check-nix-store-refs.sh create mode 100644 nix/darwin.nix rename nix/{compilers.nix => linux.nix} (75%) diff --git a/.cspell.config.yaml b/.cspell.config.yaml index e194ee21f8..aa64a318fd 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -69,6 +69,7 @@ words: - Buildx - canonicality - canonicalised + - cctools - changespq - checkme - choco @@ -110,6 +111,7 @@ words: - disablerepo - distro - doxyfile + - dsymutil - dxrpl - elgamal - enabled @@ -168,6 +170,7 @@ words: - LOCALGOOD - logwstream - Lombrozo + - lresolv - lseq - lsmf - ltype @@ -221,6 +224,7 @@ words: - Nyffenegger - onlatest - ostr + - otool - oxalica - pargs - partitioner @@ -257,6 +261,8 @@ words: - rerandomized - rerandomizes - rerere + - retargeted + - retargets - retriable - RIPD - ripdtop @@ -367,6 +373,7 @@ words: - wthread - xbridge - xchain + - xcrun - ximinez - XMACRO - xored diff --git a/.github/actions/setup-nix-env/action.yml b/.github/actions/setup-nix-env/action.yml new file mode 100644 index 0000000000..a95053e536 --- /dev/null +++ b/.github/actions/setup-nix-env/action.yml @@ -0,0 +1,69 @@ +name: Setup Nix environment +description: "Build the flake's CI environment and put its tools on PATH." + +# The environment from nix/ci-env.nix, the same one the Linux CI images bake in +# (see nix/docker). Exported onto PATH rather than entered with `nix develop`: +# the composite actions below run plain `bash` and would escape a dev shell. + +runs: + using: composite + + steps: + - name: Build the CI environment + id: build + shell: bash + env: + # --out-link doubles as a GC root for the length of the job. + OUT_LINK: ${{ runner.temp }}/xrpld-ci-env + run: | + # --extra-experimental-features: flakes may not be on in the runner's nix.conf. + nix --extra-experimental-features "nix-command flakes" \ + build .#default --out-link "${OUT_LINK}" --print-build-logs + echo "path=$(readlink -f "${OUT_LINK}")" >>"${GITHUB_OUTPUT}" + + - name: Export the environment + shell: bash + env: + ENV_PATH: ${{ steps.build.outputs.path }} + run: | + echo "${ENV_PATH}/bin" >>"${GITHUB_PATH}" + + # Already KEY=VALUE per line. See `darwinEnv` in nix/ci-env.nix. + ENV_FILE="${ENV_PATH}/share/xrpld-ci-env/env" + if [ -f "${ENV_FILE}" ]; then + cat "${ENV_FILE}" >>"${GITHUB_ENV}" + fi + + # XrplSanity.cmake otherwise rejects a Nix compiler as one that leaked. + echo "XRPL_DEVSHELL=ci-env" >>"${GITHUB_ENV}" + + # Unlike the Linux nix images, macOS needs no SSL_CERT_FILE: it has its + # own trust store, and pinning would break TLS to hosts relying on it. + + # Workspace-local, so `cleanup-workspace` clears it, but not the + # `.conan2` prepare-runner hands the system toolchain: that Conan is a + # different version, and the two would migrate each other's cache. + echo "CONAN_HOME=${{ github.workspace }}/.conan2-nix" >>"${GITHUB_ENV}" + + # Config, profiles and remote, exactly as the dev shell sets them up on + # entry; the `setup-conan` action is skipped for this toolchain. + - name: Setup Conan + shell: bash + run: ./conan/init.sh + + # `Check tools` runs later but swallows failures; a bad export would just + # build with the system toolchain. + - name: Verify the toolchain resolves into the Nix store + shell: bash + run: | + for tool in clang clang++ cmake ninja conan; do + path="$(command -v "${tool}" || true)" + echo "${tool} -> ${path:-}" + case "${path}" in + /nix/store/*) ;; + *) + echo "::error::${tool} does not resolve into the Nix store" + exit 1 + ;; + esac + done diff --git a/.github/scripts/strategy-matrix/generate.py b/.github/scripts/strategy-matrix/generate.py index 47c7593892..83f3c67e7f 100755 --- a/.github/scripts/strategy-matrix/generate.py +++ b/.github/scripts/strategy-matrix/generate.py @@ -88,6 +88,9 @@ class PlatformConfig: build_only: bool = False # if true, skip tests (e.g. macos/Windows Debug) benchmark: bool = False # if true, smoke-run the benchmarks after testing extra_cmake_args: str = "" + # "" is the runner's system compiler, "nix" the flake's CI environment. + # macOS only: Linux always builds in a Nix image, Windows has no Nix. + toolchain: str = "" def __post_init__(self) -> None: if isinstance(self.build_type, str): @@ -137,6 +140,7 @@ class MatrixEntry: sanitizers: str image: str = "" # container image; empty for macOS/Windows (runs natively) compiler: str = "" # compiler name ("gcc" or "clang"); empty for macOS/Windows + toolchain: str = "" # "nix" for the flake's CI environment; see PlatformConfig @dataclasses.dataclass @@ -253,9 +257,12 @@ def expand_platform_matrix(pf: PlatformFile, minimal: bool) -> list[MatrixEntry] if minimal and not cfg.minimal: continue for build_type in cfg.build_type: + name = f"{platform_name}-{arch}-{build_type.lower()}" + if cfg.toolchain: + name += f"-{cfg.toolchain}" entries.append( MatrixEntry( - config_name=f"{platform_name}-{arch}-{build_type.lower()}", + config_name=name, cmake_args=get_cmake_args(build_type, cfg.extra_cmake_args), cmake_target="install" if is_windows else "all", build_only=cfg.build_only, @@ -263,6 +270,7 @@ def expand_platform_matrix(pf: PlatformFile, minimal: bool) -> list[MatrixEntry] build_type=build_type, architecture=Architecture(platform=pf.platform, runner=pf.runner), sanitizers="", + toolchain=cfg.toolchain, ) ) return entries diff --git a/.github/scripts/strategy-matrix/macos.json b/.github/scripts/strategy-matrix/macos.json index 98e0f13141..554031009c 100644 --- a/.github/scripts/strategy-matrix/macos.json +++ b/.github/scripts/strategy-matrix/macos.json @@ -12,6 +12,19 @@ "extra_cmake_args": "-DCMAKE_POLICY_VERSION_MINIMUM=3.5", "build_only": true, "minimal": false + }, + { + "build_type": "Release", + "extra_cmake_args": "-DCMAKE_POLICY_VERSION_MINIMUM=3.5", + "toolchain": "nix", + "minimal": false + }, + { + "build_type": "Debug", + "extra_cmake_args": "-DCMAKE_POLICY_VERSION_MINIMUM=3.5", + "toolchain": "nix", + "build_only": true, + "minimal": false } ] } diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index f14256b9e8..a8209ac16f 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -79,6 +79,7 @@ jobs: .github/actions/build-deps/** .github/actions/release-info/** .github/actions/setup-conan/** + .github/actions/setup-nix-env/** .github/scripts/strategy-matrix/** .github/workflows/reusable-build-test-config.yml .github/workflows/reusable-build-test.yml @@ -90,6 +91,7 @@ jobs: .github/workflows/reusable-upload-recipe.yml .clang-tidy .codecov.yml + bin/check-nix-store-refs.sh bin/check-tools.sh bin/default-loader-path.sh cfg/** @@ -102,6 +104,9 @@ jobs: CMakeLists.txt conanfile.py conan.lock + flake.lock + flake.nix + nix/** LICENSE.md package/** README.md diff --git a/.github/workflows/on-trigger.yml b/.github/workflows/on-trigger.yml index dcd14b7933..0d679318a9 100644 --- a/.github/workflows/on-trigger.yml +++ b/.github/workflows/on-trigger.yml @@ -17,6 +17,7 @@ on: - ".github/actions/build-deps/**" - ".github/actions/release-info/**" - ".github/actions/setup-conan/**" + - ".github/actions/setup-nix-env/**" - ".github/scripts/strategy-matrix/**" - ".github/workflows/reusable-build-test-config.yml" - ".github/workflows/reusable-build-test.yml" @@ -28,6 +29,7 @@ on: - ".github/workflows/reusable-upload-recipe.yml" - ".clang-tidy" - ".codecov.yml" + - "bin/check-nix-store-refs.sh" - "bin/check-tools.sh" - "bin/default-loader-path.sh" - "cfg/**" @@ -40,6 +42,9 @@ on: - "CMakeLists.txt" - "conanfile.py" - "conan.lock" + - "flake.lock" + - "flake.nix" + - "nix/**" - "LICENSE.md" - "package/**" - "README.md" diff --git a/.github/workflows/reusable-build-test-config.yml b/.github/workflows/reusable-build-test-config.yml index 7989d2c7f6..94d0706e70 100644 --- a/.github/workflows/reusable-build-test-config.yml +++ b/.github/workflows/reusable-build-test-config.yml @@ -69,6 +69,12 @@ on: type: string default: "" + toolchain: + description: 'Where the toolchain comes from ("nix" to build the flake CI environment on the runner, empty for the system one). macOS only: Linux always builds in a Nix image, and Nix has no Windows support.' + required: false + type: string + default: "" + secrets: CODECOV_TOKEN: description: "The Codecov token to use for uploading coverage reports." @@ -127,6 +133,11 @@ jobs: with: enable_ccache: ${{ inputs.ccache_enabled }} + # Before any step that uses a build tool, composite actions included. + - name: Setup Nix environment + if: ${{ inputs.toolchain == 'nix' }} + uses: ./.github/actions/setup-nix-env + - name: Set ccache log file if: ${{ inputs.ccache_enabled && runner.debug == '1' }} run: echo "CCACHE_LOGFILE=${{ runner.temp }}/ccache.log" >>"${GITHUB_ENV}" @@ -151,7 +162,9 @@ jobs: with: compiler: ${{ inputs.compiler }} + # `setup-nix-env` already did this for the Nix toolchain. - name: Setup Conan + if: ${{ inputs.toolchain != 'nix' }} env: SANITIZERS: ${{ inputs.sanitizers }} uses: ./.github/actions/setup-conan @@ -215,6 +228,24 @@ jobs: --target "${CMAKE_TARGET}" \ 2>&1 | tee "${GITHUB_WORKSPACE}/build.log" + # Nothing may reference the store, so whole trees are checked - the Conan + # cache included, since what it holds is what gets uploaded and reused. + - name: Check the build output for Nix store references (Nix toolchain) + if: ${{ inputs.toolchain == 'nix' }} + run: ./bin/check-nix-store-refs.sh "${BUILD_DIR}" + + - name: Check the Conan cache for Nix store references (Nix toolchain) + if: ${{ inputs.toolchain == 'nix' }} + run: ./bin/check-nix-store-refs.sh "${CONAN_HOME}" + + # Only what PatchNixBinary.cmake retargets: the toolchain in the Linux + # images always references the store. Same condition it uses. + - name: Check for Nix store references (Linux) + if: ${{ runner.os == 'Linux' && env.SANITIZERS_ENABLED == 'false' }} + run: | + ./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpld" + ./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpl_tests" + - name: Show ccache statistics if: ${{ inputs.ccache_enabled }} run: | diff --git a/.github/workflows/reusable-build-test.yml b/.github/workflows/reusable-build-test.yml index 5368274a16..7ea106f438 100644 --- a/.github/workflows/reusable-build-test.yml +++ b/.github/workflows/reusable-build-test.yml @@ -51,5 +51,6 @@ jobs: config_name: ${{ matrix.config_name }} sanitizers: ${{ matrix.sanitizers }} compiler: ${{ matrix.compiler || '' }} + toolchain: ${{ matrix.toolchain || '' }} secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/upload-conan-deps.yml b/.github/workflows/upload-conan-deps.yml index eb58650bdf..65a3f9c5b6 100644 --- a/.github/workflows/upload-conan-deps.yml +++ b/.github/workflows/upload-conan-deps.yml @@ -72,6 +72,11 @@ jobs: with: enable_ccache: false + # Before any step that uses a build tool, composite actions included. + - name: Setup Nix environment + if: ${{ matrix.toolchain == 'nix' }} + uses: ./.github/actions/setup-nix-env + - name: Print build environment uses: XRPLF/actions/print-build-env@59dec886e4afb05a1724443af08baccbc045b574 @@ -87,7 +92,9 @@ jobs: with: compiler: ${{ matrix.compiler }} + # `setup-nix-env` already did this for the Nix toolchain. - name: Setup Conan + if: ${{ matrix.toolchain != 'nix' }} env: SANITIZERS: ${{ matrix.sanitizers }} uses: ./.github/actions/setup-conan @@ -106,6 +113,10 @@ jobs: log_verbosity: ${{ runner.os == 'Windows' && 'quiet' || 'verbose' }} sanitizers: ${{ matrix.sanitizers }} + - name: Check the Conan cache for Nix store references (Nix toolchain) + if: ${{ matrix.toolchain == 'nix' }} + run: ./bin/check-nix-store-refs.sh "${CONAN_HOME}" + - name: Log into Conan remote if: ${{ github.repository == 'XRPLF/rippled' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }} run: conan remote login "${CONAN_REMOTE_NAME}" "${{ secrets.NEXUS_REMOTE_USERNAME }}" --password "${{ secrets.NEXUS_REMOTE_PASSWORD }}" diff --git a/bin/check-nix-store-refs.sh b/bin/check-nix-store-refs.sh new file mode 100755 index 0000000000..70413df75e --- /dev/null +++ b/bin/check-nix-store-refs.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Fail if a binary under records a /nix/store path it resolves at run +# time. See docs/build/nix.md#prebuilt-packages for why that matters. +# +# is a file or a directory. macOS: nothing may reference the store, so +# point it at whole trees. Linux: the toolchain always writes the store into +# PT_INTERP and RUNPATH, so only at what cmake/PatchNixBinary.cmake retargets. +# +# Only Mach-O / ELF is inspected. Static archives hold store paths in debug info +# alone; the scripts in a Conan cache are all git hook samples and autotools +# scratch, 36 false positives to 0 real. +# +# Usage: bin/check-nix-store-refs.sh + +set -euo pipefail + +if [ "$#" -ne 1 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +if [ ! -e "$1" ]; then + echo "$0: no such path: $1" >&2 + exit 2 +fi + +case "$(uname -s)" in + Darwin) + format=Mach-O + recorded_paths=macho_recorded_paths + tool=otool + ;; + Linux) + format=ELF + recorded_paths=elf_recorded_paths + tool=readelf + ;; + *) + echo "Unsupported OS - skipping the Nix store reference check." + exit 0 + ;; +esac + +# `pipefail` would catch this too, but only as a bare nonzero exit. +if ! command -v "${tool}" >/dev/null; then + echo "$0: ${tool} not found; cannot inspect binaries" >&2 + exit 2 +fi + +# Both list what the file records. `ldd` would answer what this machine resolves +# now, which is wrong both ways: store paths for a correctly patched binary, +# silence for a store RUNPATH that resolves nowhere. + +# `name` covers LC_ID_DYLIB and LC_LOAD*_DYLIB, `path` covers LC_RPATH. +macho_recorded_paths() { + otool -l "$1" | sed -nE 's#^ *(name|path) ([^ ]*).*#\2#p' +} + +# RPATH and RUNPATH are colon-separated. +elf_recorded_paths() { + readelf -ldW "$1" | + sed -nE \ + -e 's#.*program interpreter: ([^]]*)\].*#\1#p' \ + -e 's#.*\((RPATH|RUNPATH|NEEDED)\).*\[([^]]*)\].*#\2#p' | + tr ':' '\n' +} + +checked=0 +skipped=0 +leaked=0 + +while IFS= read -r file; do + case "$(file -b "${file}" 2>/dev/null)" in + *"${format}"*) ;; + *) + skipped=$((skipped + 1)) + continue + ;; + esac + checked=$((checked + 1)) + + # Filter after extracting, or a search path starting elsewhere ($ORIGIN) + # hides the rest. `sed` not `grep`: grep calls "no matches" a failure, and + # the `|| true` that would need masks a broken pipeline too. + refs="$("${recorded_paths}" "${file}" | sed -n '\#^/nix/store/#p' | sort -u)" + if [ -n "${refs}" ]; then + leaked=$((leaked + 1)) + echo "::error file=${file}::references the Nix store at run time" + echo "${file}" + echo "${refs}" | sed 's/^/ /' + fi +done < <(find "$1" -type f \( -perm -u+x -o -name '*.dylib' -o -name '*.so*' \)) + +echo "$1: checked ${checked}, skipped ${skipped}, ${leaked} with Nix store references." + +if [ "${leaked}" -ne 0 ]; then + cat >&2 <<'EOF' + +Fixes, in order of preference: + - A Conan package built before this check existed: drop it + (`conan remove '/*'`) and rebuild. + - A binary that should have been retargeted to the system loader: check that + cmake/PatchNixBinary.cmake ran for it. + - Link the macOS system library instead of the Nix one - see + libresolvSystemStub in nix/darwin.nix. + - No system library exists (libstdc++): link it statically. + - None of the above: pin the toolchain into the package ID, following + `user.package:libc_version` in conan/profiles/ci. +EOF + exit 1 +fi diff --git a/docs/build/nix.md b/docs/build/nix.md index d1e40fcc89..4c082afb28 100644 --- a/docs/build/nix.md +++ b/docs/build/nix.md @@ -7,7 +7,7 @@ This guide explains how to use Nix to set up a reproducible development environm ## Benefits of Using Nix - **Reproducible environment**: Everyone gets the same versions of tools and compilers -- **Matches CI**: The Linux CI runs in Docker images built from this exact Nix environment +- **Matches CI**: The Linux CI runs in Docker images built from this exact Nix environment, and CI builds some macOS configurations in it as well - **No system pollution**: Dependencies are isolated and don't affect your system packages - **Consistent compilers**: The GCC and Clang shells use the same versions as CI - **Quick setup**: Get started with a single command @@ -68,7 +68,7 @@ A compiler can be chosen by providing its name with the `.#` prefix, e.g. `nix d On Linux, `.#gcc` and `.#clang` provide the exact toolchain CI uses: the compiler (pinned in [`nix/packages.nix`](../../nix/packages.nix)) -rebuilt against the pinned custom glibc (see [`nix/compilers.nix`](../../nix/compilers.nix)). +rebuilt against the pinned custom glibc (see [`nix/linux.nix`](../../nix/linux.nix)). Building that toolchain the first time is slow unless it is fetched from a Nix binary cache. If you don't need the custom glibc, the Linux-only `.#gcc-plain` and `.#clang-plain` give you the stock nixpkgs compilers of the same versions. @@ -142,14 +142,80 @@ environment — CI runs in Docker images that bundle the dev shell's toolchain ( `-plain` shells do not match that toolchain's glibc, so binaries from the remote are not a reliable match there. -On **macOS**, CI builds with Apple Clang, so the remote holds nothing for the Nix -`clang` toolchain and dependencies are compiled locally. We do not publish -Nix-built macOS binaries because a Conan package ID records the compiler version -but not the nixpkgs revision. +On **macOS**, CI also builds in this Nix environment, in Debug and Release (the +`macos-arm64-*-nix` configurations — Debug because the profile defaults to it). +The Nix build resolves to `compiler=clang`, so it gets its own package IDs, +separate from the Apple Clang ones. The +[dependency upload](../../.github/workflows/upload-conan-deps.yml) publishes them +on pushes to `develop` and on manual runs — its nightly run rebuilds everything +from source but uploads nothing — so once a set has been published `nix develop` +can reuse it instead of compiling every dependency locally. These configurations +run outside the reduced pull-request matrix, so label a PR `Full CI build` when it +touches `flake.lock` or `nix/`. To compile everything from source, add `--build '*'` to the `conan install` command. +### Why the nixpkgs revision is not part of the package ID + +A Conan package ID records the compiler and its major version, but nothing about +the nixpkgs revision the toolchain came from — and `flake.lock` moves far more +often than the toolchain meaningfully changes, so folding it in would rebuild +every dependency on every bump for nothing. + +That is safe as long as no cached artifact resolves a `/nix/store` path at run +time, because store paths change on every update and the old ones disappear with +`nix-collect-garbage`. With the `clang` toolchain macOS CI and the dev shell use, +they do not: it links against `/usr/lib/libc++` and `/usr/lib/libSystem`, and +store paths reach the `.a` files only through debug info, which nothing resolves +at link or run time. + +> [!WARNING] +> This does not hold for `nix develop .#gcc` on macOS. There is no system +> libstdc++, so GCC links its own from the store and every binary keeps a +> `/nix/store` reference. That shell is fine for tooling, but it is not a build +> configuration CI covers, and no dependency binaries are published for it. + +This is checked rather than assumed. +[`bin/check-nix-store-refs.sh`](../../bin/check-nix-store-refs.sh) takes one file +or directory and fails if a binary under it resolves a store path at run time. +CI runs it over the build output and the Conan cache, and again in the upload job +before anything is published. You can run it yourself: + +```bash +bin/check-nix-store-refs.sh build +bin/check-nix-store-refs.sh ~/.conan2-nix +``` + +It works on Linux too, but asserts something narrower there: the toolchain always +writes the store into `PT_INTERP` and `RUNPATH`, and CI builds inside an image +whose store is fixed for its lifetime, so that is fine. Only the binaries +[`PatchNixBinary.cmake`](../../cmake/PatchNixBinary.cmake) retargets to the +system loader have to be clean, and those are what CI checks: + +```bash +bin/check-nix-store-refs.sh build/xrpld +``` + +### The libresolv stub + +This is not hypothetical: `xrpld` used to be caught by it. The c-ares package +tells the linker to pass `-lresolv`, and nixpkgs keeps `libresolv` out of the +macOS SDK and ships it as an ordinary store dylib — so every Nix-built `xrpld` +recorded a `/nix/store/…-libresolv-93/lib/libresolv.9.dylib` load command and +stopped running once that path was collected. Nothing in the link uses a single +symbol from it. + +Both environments now put a stub on the linker search path +(`libresolvSystemStub` in [`nix/darwin.nix`](../../nix/darwin.nix)): the +same library with its install name set to `/usr/lib/libresolv.9.dylib`, which is +exactly the load command the Apple Clang build records. + +Package IDs did not change, so Conan keeps serving anything built before the +stub landed. If a binary fails to start with `Library not loaded: /nix/store/…`, +see [that entry](./nix_troubleshooting.md#library-not-loaded-nixstore-from-a-binary-that-used-to-work) +in the troubleshooting guide. + ## Automatic Activation with direnv [direnv](https://direnv.net/) or [nix-direnv](https://github.com/nix-community/nix-direnv) can automatically activate the Nix development shell when you enter the repository directory. diff --git a/docs/build/nix_troubleshooting.md b/docs/build/nix_troubleshooting.md index fa766c0ee9..49088ab6b4 100644 --- a/docs/build/nix_troubleshooting.md +++ b/docs/build/nix_troubleshooting.md @@ -131,3 +131,91 @@ once it picks up that rebuild, then re-run the `grep libgit2` check above to confirm it reports `1.9.4` or newer. Until then, prefer the workarounds above. + +## `wint_t` / `uint32_t` errors from the Nix libc++ headers + +A build that mixes the Nix toolchain with the system SDK fails in libc++ itself, +with errors that look nothing like your code: + +``` +/nix/store/...-libcxx-.../include/c++/v1/cwchar:136:9: error: target of using declaration conflicts with declaration already in scope + 136 | using ::wint_t _LIBCPP_USING_IF_EXISTS; +/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/sys/_types/_wint_t.h:32:25: note: target of using declaration +... +error: use of undeclared identifier 'UINT32_C' +``` + +The give-away is the second path: Nix's libc++ headers are being combined with +the **Xcode Command Line Tools** SDK instead of the Nix one. + +### Why it happens + +`SDKROOT` and `DEVELOPER_DIR` are what point the toolchain at the Nix SDK, and +they are not baked into the compiler — a dev shell gets them from the +`apple-sdk` setup hook. CMake, finding neither, asks `xcrun`, which answers with +the system SDK. Nix's `libc++` and Apple's headers then declare the same types +twice. + +### Fix + +Run the build from inside the dev shell (`nix develop`), or from an environment +that exports both variables. To confirm which SDK a configured build is using: + +```bash +grep -o '\-isysroot [^ ]*' build/compile_commands.json | sort -u +``` + +It should print a `/nix/store/...-apple-sdk-*` path. If it prints +`/Library/Developer/CommandLineTools/...`, re-configure from within the shell — +CMake caches the sysroot, so an existing `build/` directory keeps the wrong one. + +## `Library not loaded: /nix/store/…` from a binary that used to work + +A binary stops starting after a `nix flake update`, or after +`nix-collect-garbage` removes the paths the previous toolchain used: + +``` +dyld[57271]: Library not loaded: /nix/store/…-libresolv-93/lib/libresolv.9.dylib +``` + +[`bin/check-nix-store-refs.sh`](../../bin/check-nix-store-refs.sh) finds the same +thing without having to run anything, and names the file: + +``` +$ bin/check-nix-store-refs.sh ~/.conan2-nix +::error file=/Users/you/.conan2-nix/p/b/c-area24ded30c388c/p/bin/adig::references the Nix store at run time +/Users/you/.conan2-nix/p/b/c-area24ded30c388c/p/bin/adig + /nix/store/p4lp3xq4imd1qzqh08x8vcq2zfhi7rca-libresolv-93/lib/libresolv.9.dylib +/Users/you/.conan2-nix: checked 135, skipped 2495, 1 with Nix store references. +``` + +Conan's cache folders are named after a truncated package name plus a hash, so +ask Conan which package the offending one belongs to — pass the folder holding +the hash, not the file itself: + +``` +$ conan cache ref ~/.conan2-nix/p/b/c-area24ded30c388c +c-ares/1.34.6#545240bb1c40e2cacd4362d6b8967650:dab5992496abe6d219defb7986ecbf367615a5e5#… +``` + +### Why it happens + +The binary records a store path that no longer exists. Nothing we build should: +see [Prebuilt packages](./nix.md#prebuilt-packages) for why, and +`libresolvSystemStub` in [`nix/darwin.nix`](../../nix/darwin.nix) for the one +dependency that needed help to comply. + +A Conan package ID does not encode the nixpkgs revision, so a package built +before that stub existed stays in your local cache and keeps being reused. The +dev shell is also what tends to produce one: it is a slightly _less_ isolated +build environment than CI's, because `mkShell` puts every tool's headers and +libraries on the compiler's search path — which is how c-ares found the Nix +`libresolv` in the first place. + +### Fix + +Drop that package and let Conan refetch or rebuild it: + +```bash +conan remove 'c-ares/*' +``` diff --git a/nix/ci-env.nix b/nix/ci-env.nix index 787b94406e..779b5b7230 100644 --- a/nix/ci-env.nix +++ b/nix/ci-env.nix @@ -1,67 +1,39 @@ +# The environment CI builds in: every tool on PATH, no Nix stdenv setup hooks. +# Baked into the `nix-*` Docker images on Linux (see nix/docker), built on the +# runner on macOS (see .github/actions/setup-nix-env). { pkgs, customGlibc, ... }: let - inherit (import ./packages.nix { inherit pkgs; }) - commonPackages - gccVersion - llvmVersion - mkVersionedToolLinks - ; + inherit (import ./packages.nix { inherit pkgs; }) commonPackages; - # Custom-glibc toolchain, shared with the Linux dev shell (see compilers.nix). - inherit (import ./compilers.nix { inherit pkgs customGlibc; }) - customGcc - customClang - customBinutils - customGcov - ; + # Each forces something absent on the other platform, so both stay lazy. + linux = import ./linux.nix { inherit pkgs customGlibc; }; + darwin = import ./darwin.nix { inherit pkgs; }; - # Strip the generic cc/c++/cpp symlinks from the clang wrapper so it can - # coexist with the gcc wrapper in buildEnv. gcc remains the default - # compiler (cc/c++/cpp); clang is invoked explicitly as clang/clang++. - customClangForCiEnv = pkgs.symlinkJoin { - name = "clang-wrapper-custom-for-ci-env"; - paths = [ customClang ]; - postBuild = '' - rm -f $out/bin/cc $out/bin/c++ $out/bin/cpp - ''; - }; + # What a buildEnv cannot express: environment variables. $GITHUB_ENV format; + # `set -a; . env; set +a` loads it in a shell. + darwinEnv = pkgs.writeTextDir "share/xrpld-ci-env/env" ( + pkgs.lib.concatStrings ( + pkgs.lib.mapAttrsToList (name: value: "${name}=${value}\n") (darwin.sdkEnv // darwin.libresolvEnv) + ) + ); + toolchain = if pkgs.stdenv.isLinux then linux.toolchain else (darwin.toolchain ++ [ darwinEnv ]); in { default = pkgs.buildEnv { name = "xrpld-ci-env"; - paths = commonPackages ++ [ - customGcc - customGcov - customClangForCiEnv - customBinutils - (mkVersionedToolLinks { - name = "gcc"; - package = customGcc; - version = gccVersion; - tools = [ - "gcc" - "g++" - "cpp" - ]; - }) - (mkVersionedToolLinks { - name = "clang"; - package = customClang; - version = llvmVersion; - tools = [ - "clang" - "clang++" - ]; - }) - # CA certificate bundle so HTTPS clients (git, curl, conan) can verify - # TLS connections without ca-certificates being installed in the system. - pkgs.cacert - ]; + paths = + commonPackages + ++ toolchain + ++ [ + # CA certificate bundle so HTTPS clients (git, curl, conan) can verify + # TLS connections without ca-certificates being installed in the system. + pkgs.cacert + ]; pathsToLink = [ "/bin" "/etc/ssl/certs" diff --git a/nix/darwin.nix b/nix/darwin.nix new file mode 100644 index 0000000000..837752fc6a --- /dev/null +++ b/nix/darwin.nix @@ -0,0 +1,80 @@ +# The darwin toolchain, counterpart to linux.nix. Split by consumer: a dev +# shell's stdenv provides the SDK variables, nothing provides libresolv. +# +# darwin only - `libresolv` does not exist on Linux. +{ pkgs }: +let + inherit (import ./packages.nix { inherit pkgs; }) + llvmVersion + llvmPackages + mkVersionedToolLinks + ; + + # nixpkgs keeps libresolv out of the macOS SDK, so neither c-ares' `-lresolv` + # nor grpc's resolves. Headers can come from nixpkgs; the + # library cannot, or its store path lands in xrpld - hence this copy. + libresolvSystemStub = + pkgs.runCommand "libresolv-system-stub" + { + nativeBuildInputs = [ llvmPackages.bintools ]; + } + '' + mkdir -p "$out/lib" + cp ${pkgs.darwin.libresolv}/lib/libresolv.9.dylib "$out/lib/" + chmod +w "$out/lib/libresolv.9.dylib" + llvm-install-name-tool -id /usr/lib/libresolv.9.dylib "$out/lib/libresolv.9.dylib" + ln -s libresolv.9.dylib "$out/lib/libresolv.dylib" + ''; +in +{ + # For an environment that only puts binaries on PATH. + toolchain = [ + llvmPackages.clang + # The wrappers re-export only part of cctools; a bare env has no stdenv to + # supply the rest, and without `dsymutil` even `clang -g` cannot link. One + # by one, because buildEnv rejects any name a wrapper owns (notably `ld`). + (pkgs.linkFarm "cctools-extra" ( + map + (tool: { + name = "bin/${tool}"; + path = "${llvmPackages.clang.bintools.bintools}/bin/${tool}"; + }) + [ + "codesign_allocate" + "dsymutil" + "dwarfdump" + "install_name_tool" + "lipo" + "otool" + ] + )) + (mkVersionedToolLinks { + name = "clang"; + package = llvmPackages.clang; + version = llvmVersion; + tools = [ + "clang" + "clang++" + ]; + }) + ]; + + # Without these CMake asks `xcrun` and gets the Command Line Tools SDK, whose + # headers clash with the Nix libc++ ones. + sdkEnv = { + DEVELOPER_DIR = "${pkgs.apple-sdk}"; + SDKROOT = "${pkgs.apple-sdk}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"; + }; + + # Salted names: the wrappers only read plain NIX_CFLAGS_COMPILE / NIX_LDFLAGS + # through role variables a Nix stdenv would set. The salt is the target + # platform, so this fits the gcc wrapper too. + # + # No space after -isystem: these are written one per line as KEY=VALUE, and a + # shell sourcing that reads the space as the end of the assignment. + libresolvEnv = { + "NIX_CFLAGS_COMPILE_${llvmPackages.clang.suffixSalt}" = + "-isystem${pkgs.darwin.libresolv.dev}/include"; + "NIX_LDFLAGS_${llvmPackages.clang.bintools.suffixSalt}" = "-L${libresolvSystemStub}/lib"; + }; +} diff --git a/nix/devshell.nix b/nix/devshell.nix index ac0b84e169..07f7143c5b 100644 --- a/nix/devshell.nix +++ b/nix/devshell.nix @@ -14,21 +14,21 @@ let plainGccStdenv = pkgs."gcc${toString gccVersion}Stdenv"; plainClangStdenv = llvmPackages.stdenv; - # Custom-glibc stdenvs, matching the CI environment (see compilers.nix). The - # pinned glibc snapshot only builds on Linux, so on darwin these fall back to - # the plain stdenvs; the `if isLinux` guard keeps `customGlibc` from being - # forced (and erroring) on macOS. - customCompilers = import ./compilers.nix { inherit pkgs customGlibc; }; - customGccStdenv = if pkgs.stdenv.isLinux then customCompilers.customStdenv else plainGccStdenv; - customClangStdenv = - if pkgs.stdenv.isLinux then customCompilers.customClangStdenv else plainClangStdenv; + # Each forces something absent on the other platform, so both stay lazy. + linux = import ./linux.nix { inherit pkgs customGlibc; }; + darwin = import ./darwin.nix { inherit pkgs; }; + + # Custom-glibc stdenvs, matching the CI environment. darwin has no custom + # glibc, so there they fall back to the plain nixpkgs stdenvs. + customGccStdenv = if pkgs.stdenv.isLinux then linux.gccStdenv else plainGccStdenv; + customClangStdenv = if pkgs.stdenv.isLinux then linux.clangStdenv else plainClangStdenv; # gcov matching each gcc shell, so `-Dcoverage=ON` builds work in the shell. plainGcov = mkGcov { name = "plain"; cc = gccPackage.cc; }; - customGccGcov = if pkgs.stdenv.isLinux then customCompilers.customGcov else plainGcov; + customGccGcov = if pkgs.stdenv.isLinux then linux.gcov else plainGcov; # Whole directory: init.sh locates the profiles relative to itself. conanDir = ../conan; @@ -49,6 +49,16 @@ let unset _xrpl_conan_stamp ''; + # Not sdkEnv: a shell's stdenv already sets that up. Prepended so the stub + # beats the nixpkgs libresolv this shell's tooling drags in. + darwinLibresolvHook = pkgs.lib.optionalString pkgs.stdenv.isDarwin ( + pkgs.lib.concatLines ( + pkgs.lib.mapAttrsToList ( + name: value: ''export ${name}="${value} ''${${name}:-}"'' + ) darwin.libresolvEnv + ) + ); + # Shown when entering a *-plain shell. These exist only on Linux (see below), # where the stock toolchain diverges from CI. plainWarningHook = '' @@ -106,6 +116,7 @@ let shellHook = '' echo "Welcome to xrpld development shell"; ${compilerVersionHook} + ${darwinLibresolvHook} ${conanHook} ${warningHook} ''; diff --git a/nix/docker/Dockerfile b/nix/docker/Dockerfile index 74c630cb61..5506bc3c77 100644 --- a/nix/docker/Dockerfile +++ b/nix/docker/Dockerfile @@ -8,7 +8,7 @@ RUN mkdir -p ~/.config/nix && \ # Copy our source and setup our working dir. COPY nix/ci-env.nix /tmp/build/nix/ci-env.nix -COPY nix/compilers.nix /tmp/build/nix/compilers.nix +COPY nix/linux.nix /tmp/build/nix/linux.nix COPY nix/packages.nix /tmp/build/nix/packages.nix COPY nix/utils.nix /tmp/build/nix/utils.nix COPY flake.nix /tmp/build/ diff --git a/nix/compilers.nix b/nix/linux.nix similarity index 75% rename from nix/compilers.nix rename to nix/linux.nix index 90856afacc..ea808fbf50 100644 --- a/nix/compilers.nix +++ b/nix/linux.nix @@ -1,7 +1,9 @@ -# Custom-glibc compiler toolchain shared by the CI environment (ci-env.nix) and -# the Linux dev shell (devshell.nix): gcc / clang / binutils rebuilt to target -# the pinned custom glibc. Linux only — the pinned glibc snapshot does not build -# on darwin, so callers must not evaluate this on macOS. +# The Linux toolchain: gcc / clang / binutils rebuilt to target the pinned +# custom glibc, shared by the CI environment (ci-env.nix) and the dev shell +# (devshell.nix). The counterpart to darwin.nix. +# +# Linux only — the pinned glibc snapshot does not build on darwin, so callers +# must not evaluate this on macOS. { pkgs, customGlibc, @@ -9,9 +11,11 @@ let inherit (import ./packages.nix { inherit pkgs; }) gccPackage + gccVersion llvmPackages llvmVersion mkGcov + mkVersionedToolLinks ; # binutils wrapped to emit binaries that reference the custom glibc @@ -103,15 +107,46 @@ let echo "-isystem ${customCompilerRt.dev}/include" >> $out/nix-support/cc-cflags ''; }; + # Strip the generic cc/c++/cpp symlinks from the clang wrapper so it can + # coexist with the gcc wrapper in buildEnv. gcc remains the default + # compiler (cc/c++/cpp); clang is invoked explicitly as clang/clang++. + customClangForCiEnv = pkgs.symlinkJoin { + name = "clang-wrapper-custom-for-ci-env"; + paths = [ customClang ]; + postBuild = '' + rm -f $out/bin/cc $out/bin/c++ $out/bin/cpp + ''; + }; in { - inherit + # For an environment that only puts binaries on PATH. + toolchain = [ customGcc - customClang - customBinutils - customStdenv customGcov - ; + customClangForCiEnv + customBinutils + (mkVersionedToolLinks { + name = "gcc"; + package = customGcc; + version = gccVersion; + tools = [ + "gcc" + "g++" + "cpp" + ]; + }) + (mkVersionedToolLinks { + name = "clang"; + package = customClang; + version = llvmVersion; + tools = [ + "clang" + "clang++" + ]; + }) + ]; - customClangStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customClang; + gccStdenv = customStdenv; + clangStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customClang; + gcov = customGcov; }