build: Run nix macos builds in CI; deny nix store references (#8023)

.cspell.config.yaml

@@ -69,6 +69,7 @@ words:
   - Buildx
   - canonicality
   - canonicalised
+  - cctools
   - changespq
   - checkme
   - choco
@@ -110,6 +111,7 @@ words:
   - disablerepo
   - distro
   - doxyfile
+  - dsymutil
   - dxrpl
   - elgamal
   - enabled
@@ -168,6 +170,7 @@ words:
   - LOCALGOOD
   - logwstream
   - Lombrozo
+  - lresolv
   - lseq
   - lsmf
   - ltype
@@ -221,6 +224,7 @@ words:
   - Nyffenegger
   - onlatest
   - ostr
+  - otool
   - oxalica
   - pargs
   - partitioner
@@ -257,6 +261,8 @@ words:
   - rerandomized
   - rerandomizes
   - rerere
+  - retargeted
+  - retargets
   - retriable
   - RIPD
   - ripdtop
@@ -367,6 +373,7 @@ words:
   - wthread
   - xbridge
   - xchain
+  - xcrun
   - ximinez
   - XMACRO
   - xored

.github/actions/setup-nix-env/action.yml

@@ -0,0 +1,69 @@
+name: Setup Nix environment
+description: "Build the flake's CI environment and put its tools on PATH."
+
+# The environment from nix/ci-env.nix, the same one the Linux CI images bake in
+# (see nix/docker). Exported onto PATH rather than entered with `nix develop`:
+# the composite actions below run plain `bash` and would escape a dev shell.
+
+runs:
+  using: composite
+
+  steps:
+    - name: Build the CI environment
+      id: build
+      shell: bash
+      env:
+        # --out-link doubles as a GC root for the length of the job.
+        OUT_LINK: ${{ runner.temp }}/xrpld-ci-env
+      run: |
+        # --extra-experimental-features: flakes may not be on in the runner's nix.conf.
+        nix --extra-experimental-features "nix-command flakes" \
+            build .#default --out-link "${OUT_LINK}" --print-build-logs
+        echo "path=$(readlink -f "${OUT_LINK}")" >>"${GITHUB_OUTPUT}"
+
+    - name: Export the environment
+      shell: bash
+      env:
+        ENV_PATH: ${{ steps.build.outputs.path }}
+      run: |
+        echo "${ENV_PATH}/bin" >>"${GITHUB_PATH}"
+
+        # Already KEY=VALUE per line. See `darwinEnv` in nix/ci-env.nix.
+        ENV_FILE="${ENV_PATH}/share/xrpld-ci-env/env"
+        if [ -f "${ENV_FILE}" ]; then
+            cat "${ENV_FILE}" >>"${GITHUB_ENV}"
+        fi
+
+        # XrplSanity.cmake otherwise rejects a Nix compiler as one that leaked.
+        echo "XRPL_DEVSHELL=ci-env" >>"${GITHUB_ENV}"
+
+        # Unlike the Linux nix images, macOS needs no SSL_CERT_FILE: it has its
+        # own trust store, and pinning would break TLS to hosts relying on it.
+
+        # Workspace-local, so `cleanup-workspace` clears it, but not the
+        # `.conan2` prepare-runner hands the system toolchain: that Conan is a
+        # different version, and the two would migrate each other's cache.
+        echo "CONAN_HOME=${{ github.workspace }}/.conan2-nix" >>"${GITHUB_ENV}"
+
+    # Config, profiles and remote, exactly as the dev shell sets them up on
+    # entry; the `setup-conan` action is skipped for this toolchain.
+    - name: Setup Conan
+      shell: bash
+      run: ./conan/init.sh
+
+    # `Check tools` runs later but swallows failures; a bad export would just
+    # build with the system toolchain.
+    - name: Verify the toolchain resolves into the Nix store
+      shell: bash
+      run: |
+        for tool in clang clang++ cmake ninja conan; do
+            path="$(command -v "${tool}" || true)"
+            echo "${tool} -> ${path:-<not found>}"
+            case "${path}" in
+                /nix/store/*) ;;
+                *)
+                    echo "::error::${tool} does not resolve into the Nix store"
+                    exit 1
+                    ;;
+            esac
+        done

.github/scripts/strategy-matrix/generate.py

@@ -88,6 +88,9 @@ class PlatformConfig:
     build_only: bool = False  # if true, skip tests (e.g. macos/Windows Debug)
     benchmark: bool = False  # if true, smoke-run the benchmarks after testing
     extra_cmake_args: str = ""
+    # "" is the runner's system compiler, "nix" the flake's CI environment.
+    # macOS only: Linux always builds in a Nix image, Windows has no Nix.
+    toolchain: str = ""
 
     def __post_init__(self) -> None:
         if isinstance(self.build_type, str):
@@ -137,6 +140,7 @@ class MatrixEntry:
     sanitizers: str
     image: str = ""  # container image; empty for macOS/Windows (runs natively)
     compiler: str = ""  # compiler name ("gcc" or "clang"); empty for macOS/Windows
+    toolchain: str = ""  # "nix" for the flake's CI environment; see PlatformConfig
 
 
 @dataclasses.dataclass
@@ -253,9 +257,12 @@ def expand_platform_matrix(pf: PlatformFile, minimal: bool) -> list[MatrixEntry]
         if minimal and not cfg.minimal:
             continue
         for build_type in cfg.build_type:
+            name = f"{platform_name}-{arch}-{build_type.lower()}"
+            if cfg.toolchain:
+                name += f"-{cfg.toolchain}"
             entries.append(
                 MatrixEntry(
-                    config_name=f"{platform_name}-{arch}-{build_type.lower()}",
+                    config_name=name,
                     cmake_args=get_cmake_args(build_type, cfg.extra_cmake_args),
                     cmake_target="install" if is_windows else "all",
                     build_only=cfg.build_only,
@@ -263,6 +270,7 @@ def expand_platform_matrix(pf: PlatformFile, minimal: bool) -> list[MatrixEntry]
                     build_type=build_type,
                     architecture=Architecture(platform=pf.platform, runner=pf.runner),
                     sanitizers="",
+                    toolchain=cfg.toolchain,
                 )
             )
     return entries

.github/scripts/strategy-matrix/macos.json

@@ -12,6 +12,19 @@
       "extra_cmake_args": "-DCMAKE_POLICY_VERSION_MINIMUM=3.5",
       "build_only": true,
       "minimal": false
+    },
+    {
+      "build_type": "Release",
+      "extra_cmake_args": "-DCMAKE_POLICY_VERSION_MINIMUM=3.5",
+      "toolchain": "nix",
+      "minimal": false
+    },
+    {
+      "build_type": "Debug",
+      "extra_cmake_args": "-DCMAKE_POLICY_VERSION_MINIMUM=3.5",
+      "toolchain": "nix",
+      "build_only": true,
+      "minimal": false
     }
   ]
 }

.github/workflows/on-pr.yml

@@ -79,6 +79,7 @@ jobs:
             .github/actions/build-deps/**
             .github/actions/release-info/**
             .github/actions/setup-conan/**
+            .github/actions/setup-nix-env/**
             .github/scripts/strategy-matrix/**
             .github/workflows/reusable-build-test-config.yml
             .github/workflows/reusable-build-test.yml
@@ -90,6 +91,7 @@ jobs:
             .github/workflows/reusable-upload-recipe.yml
             .clang-tidy
             .codecov.yml
+            bin/check-nix-store-refs.sh
             bin/check-tools.sh
             bin/default-loader-path.sh
             cfg/**
@@ -102,6 +104,9 @@ jobs:
             CMakeLists.txt
             conanfile.py
             conan.lock
+            flake.lock
+            flake.nix
+            nix/**
             LICENSE.md
             package/**
             README.md

.github/workflows/on-trigger.yml

@@ -17,6 +17,7 @@ on:
       - ".github/actions/build-deps/**"
       - ".github/actions/release-info/**"
       - ".github/actions/setup-conan/**"
+      - ".github/actions/setup-nix-env/**"
       - ".github/scripts/strategy-matrix/**"
       - ".github/workflows/reusable-build-test-config.yml"
       - ".github/workflows/reusable-build-test.yml"
@@ -28,6 +29,7 @@ on:
       - ".github/workflows/reusable-upload-recipe.yml"
       - ".clang-tidy"
       - ".codecov.yml"
+      - "bin/check-nix-store-refs.sh"
       - "bin/check-tools.sh"
       - "bin/default-loader-path.sh"
       - "cfg/**"
@@ -40,6 +42,9 @@ on:
       - "CMakeLists.txt"
       - "conanfile.py"
       - "conan.lock"
+      - "flake.lock"
+      - "flake.nix"
+      - "nix/**"
       - "LICENSE.md"
       - "package/**"
       - "README.md"

.github/workflows/reusable-build-test-config.yml

@@ -69,6 +69,12 @@ on:
         type: string
         default: ""
 
+      toolchain:
+        description: 'Where the toolchain comes from ("nix" to build the flake CI environment on the runner, empty for the system one). macOS only: Linux always builds in a Nix image, and Nix has no Windows support.'
+        required: false
+        type: string
+        default: ""
+
     secrets:
       CODECOV_TOKEN:
         description: "The Codecov token to use for uploading coverage reports."
@@ -127,6 +133,11 @@ jobs:
         with:
           enable_ccache: ${{ inputs.ccache_enabled }}
 
+      # Before any step that uses a build tool, composite actions included.
+      - name: Setup Nix environment
+        if: ${{ inputs.toolchain == 'nix' }}
+        uses: ./.github/actions/setup-nix-env
+
       - name: Set ccache log file
         if: ${{ inputs.ccache_enabled && runner.debug == '1' }}
         run: echo "CCACHE_LOGFILE=${{ runner.temp }}/ccache.log" >>"${GITHUB_ENV}"
@@ -151,7 +162,9 @@ jobs:
         with:
           compiler: ${{ inputs.compiler }}
 
+      # `setup-nix-env` already did this for the Nix toolchain.
       - name: Setup Conan
+        if: ${{ inputs.toolchain != 'nix' }}
         env:
           SANITIZERS: ${{ inputs.sanitizers }}
         uses: ./.github/actions/setup-conan
@@ -215,6 +228,24 @@ jobs:
               --target "${CMAKE_TARGET}" \
               2>&1 | tee "${GITHUB_WORKSPACE}/build.log"
 
+      # Nothing may reference the store, so whole trees are checked - the Conan
+      # cache included, since what it holds is what gets uploaded and reused.
+      - name: Check the build output for Nix store references (Nix toolchain)
+        if: ${{ inputs.toolchain == 'nix' }}
+        run: ./bin/check-nix-store-refs.sh "${BUILD_DIR}"
+
+      - name: Check the Conan cache for Nix store references (Nix toolchain)
+        if: ${{ inputs.toolchain == 'nix' }}
+        run: ./bin/check-nix-store-refs.sh "${CONAN_HOME}"
+
+      # Only what PatchNixBinary.cmake retargets: the toolchain in the Linux
+      # images always references the store. Same condition it uses.
+      - name: Check for Nix store references (Linux)
+        if: ${{ runner.os == 'Linux' && env.SANITIZERS_ENABLED == 'false' }}
+        run: |
+          ./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpld"
+          ./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpl_tests"
+
       - name: Show ccache statistics
         if: ${{ inputs.ccache_enabled }}
         run: |

.github/workflows/reusable-build-test.yml

@@ -51,5 +51,6 @@ jobs:
       config_name: ${{ matrix.config_name }}
       sanitizers: ${{ matrix.sanitizers }}
       compiler: ${{ matrix.compiler || '' }}
+      toolchain: ${{ matrix.toolchain || '' }}
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/upload-conan-deps.yml

@@ -72,6 +72,11 @@ jobs:
         with:
           enable_ccache: false
 
+      # Before any step that uses a build tool, composite actions included.
+      - name: Setup Nix environment
+        if: ${{ matrix.toolchain == 'nix' }}
+        uses: ./.github/actions/setup-nix-env
+
       - name: Print build environment
         uses: XRPLF/actions/print-build-env@59dec886e4afb05a1724443af08baccbc045b574
 
@@ -87,7 +92,9 @@ jobs:
         with:
           compiler: ${{ matrix.compiler }}
 
+      # `setup-nix-env` already did this for the Nix toolchain.
       - name: Setup Conan
+        if: ${{ matrix.toolchain != 'nix' }}
         env:
           SANITIZERS: ${{ matrix.sanitizers }}
         uses: ./.github/actions/setup-conan
@@ -106,6 +113,10 @@ jobs:
           log_verbosity: ${{ runner.os == 'Windows' && 'quiet' || 'verbose' }}
           sanitizers: ${{ matrix.sanitizers }}
 
+      - name: Check the Conan cache for Nix store references (Nix toolchain)
+        if: ${{ matrix.toolchain == 'nix' }}
+        run: ./bin/check-nix-store-refs.sh "${CONAN_HOME}"
+
       - name: Log into Conan remote
         if: ${{ github.repository == 'XRPLF/rippled' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }}
         run: conan remote login "${CONAN_REMOTE_NAME}" "${{ secrets.NEXUS_REMOTE_USERNAME }}" --password "${{ secrets.NEXUS_REMOTE_PASSWORD }}"

bin/check-nix-store-refs.sh

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+# Fail if a binary under <path> records a /nix/store path it resolves at run
+# time. See docs/build/nix.md#prebuilt-packages for why that matters.
+#
+# <path> is a file or a directory. macOS: nothing may reference the store, so
+# point it at whole trees. Linux: the toolchain always writes the store into
+# PT_INTERP and RUNPATH, so only at what cmake/PatchNixBinary.cmake retargets.
+#
+# Only Mach-O / ELF is inspected. Static archives hold store paths in debug info
+# alone; the scripts in a Conan cache are all git hook samples and autotools
+# scratch, 36 false positives to 0 real.
+#
+# Usage: bin/check-nix-store-refs.sh <path>
+
+set -euo pipefail
+
+if [ "$#" -ne 1 ]; then
+    echo "usage: $0 <path>" >&2
+    exit 2
+fi
+
+if [ ! -e "$1" ]; then
+    echo "$0: no such path: $1" >&2
+    exit 2
+fi
+
+case "$(uname -s)" in
+    Darwin)
+        format=Mach-O
+        recorded_paths=macho_recorded_paths
+        tool=otool
+        ;;
+    Linux)
+        format=ELF
+        recorded_paths=elf_recorded_paths
+        tool=readelf
+        ;;
+    *)
+        echo "Unsupported OS - skipping the Nix store reference check."
+        exit 0
+        ;;
+esac
+
+# `pipefail` would catch this too, but only as a bare nonzero exit.
+if ! command -v "${tool}" >/dev/null; then
+    echo "$0: ${tool} not found; cannot inspect binaries" >&2
+    exit 2
+fi
+
+# Both list what the file records. `ldd` would answer what this machine resolves
+# now, which is wrong both ways: store paths for a correctly patched binary,
+# silence for a store RUNPATH that resolves nowhere.
+
+# `name` covers LC_ID_DYLIB and LC_LOAD*_DYLIB, `path` covers LC_RPATH.
+macho_recorded_paths() {
+    otool -l "$1" | sed -nE 's#^ *(name|path) ([^ ]*).*#\2#p'
+}
+
+# RPATH and RUNPATH are colon-separated.
+elf_recorded_paths() {
+    readelf -ldW "$1" |
+        sed -nE \
+            -e 's#.*program interpreter: ([^]]*)\].*#\1#p' \
+            -e 's#.*\((RPATH|RUNPATH|NEEDED)\).*\[([^]]*)\].*#\2#p' |
+        tr ':' '\n'
+}
+
+checked=0
+skipped=0
+leaked=0
+
+while IFS= read -r file; do
+    case "$(file -b "${file}" 2>/dev/null)" in
+        *"${format}"*) ;;
+        *)
+            skipped=$((skipped + 1))
+            continue
+            ;;
+    esac
+    checked=$((checked + 1))
+
+    # Filter after extracting, or a search path starting elsewhere ($ORIGIN)
+    # hides the rest. `sed` not `grep`: grep calls "no matches" a failure, and
+    # the `|| true` that would need masks a broken pipeline too.
+    refs="$("${recorded_paths}" "${file}" | sed -n '\#^/nix/store/#p' | sort -u)"
+    if [ -n "${refs}" ]; then
+        leaked=$((leaked + 1))
+        echo "::error file=${file}::references the Nix store at run time"
+        echo "${file}"
+        echo "${refs}" | sed 's/^/    /'
+    fi
+done < <(find "$1" -type f \( -perm -u+x -o -name '*.dylib' -o -name '*.so*' \))
+
+echo "$1: checked ${checked}, skipped ${skipped}, ${leaked} with Nix store references."
+
+if [ "${leaked}" -ne 0 ]; then
+    cat >&2 <<'EOF'
+
+Fixes, in order of preference:
+  - A Conan package built before this check existed: drop it
+    (`conan remove '<name>/*'`) and rebuild.
+  - A binary that should have been retargeted to the system loader: check that
+    cmake/PatchNixBinary.cmake ran for it.
+  - Link the macOS system library instead of the Nix one - see
+    libresolvSystemStub in nix/darwin.nix.
+  - No system library exists (libstdc++): link it statically.
+  - None of the above: pin the toolchain into the package ID, following
+    `user.package:libc_version` in conan/profiles/ci.
+EOF
+    exit 1
+fi

docs/build/nix.md

@@ -7,7 +7,7 @@ This guide explains how to use Nix to set up a reproducible development environm
 ## Benefits of Using Nix
 
 - **Reproducible environment**: Everyone gets the same versions of tools and compilers
-- **Matches CI**: The Linux CI runs in Docker images built from this exact Nix environment
+- **Matches CI**: The Linux CI runs in Docker images built from this exact Nix environment, and CI builds some macOS configurations in it as well
 - **No system pollution**: Dependencies are isolated and don't affect your system packages
 - **Consistent compilers**: The GCC and Clang shells use the same versions as CI
 - **Quick setup**: Get started with a single command
@@ -68,7 +68,7 @@ A compiler can be chosen by providing its name with the `.#` prefix, e.g. `nix d
 
 On Linux, `.#gcc` and `.#clang` provide the exact toolchain CI uses:
 the compiler (pinned in [`nix/packages.nix`](../../nix/packages.nix))
-rebuilt against the pinned custom glibc (see [`nix/compilers.nix`](../../nix/compilers.nix)).
+rebuilt against the pinned custom glibc (see [`nix/linux.nix`](../../nix/linux.nix)).
 Building that toolchain the first time is slow unless it is fetched from a Nix binary cache.
 If you don't need the custom glibc, the Linux-only `.#gcc-plain` and `.#clang-plain`
 give you the stock nixpkgs compilers of the same versions.
@@ -142,14 +142,80 @@ environment — CI runs in Docker images that bundle the dev shell's toolchain (
 `-plain` shells do not match that toolchain's glibc, so binaries from the remote
 are not a reliable match there.
 
-On **macOS**, CI builds with Apple Clang, so the remote holds nothing for the Nix
-`clang` toolchain and dependencies are compiled locally. We do not publish
-Nix-built macOS binaries because a Conan package ID records the compiler version
-but not the nixpkgs revision.
+On **macOS**, CI also builds in this Nix environment, in Debug and Release (the
+`macos-arm64-*-nix` configurations — Debug because the profile defaults to it).
+The Nix build resolves to `compiler=clang`, so it gets its own package IDs,
+separate from the Apple Clang ones. The
+[dependency upload](../../.github/workflows/upload-conan-deps.yml) publishes them
+on pushes to `develop` and on manual runs — its nightly run rebuilds everything
+from source but uploads nothing — so once a set has been published `nix develop`
+can reuse it instead of compiling every dependency locally. These configurations
+run outside the reduced pull-request matrix, so label a PR `Full CI build` when it
+touches `flake.lock` or `nix/`.
 
 To compile everything from source, add `--build '*'` to the `conan install`
 command.
 
+### Why the nixpkgs revision is not part of the package ID
+
+A Conan package ID records the compiler and its major version, but nothing about
+the nixpkgs revision the toolchain came from — and `flake.lock` moves far more
+often than the toolchain meaningfully changes, so folding it in would rebuild
+every dependency on every bump for nothing.
+
+That is safe as long as no cached artifact resolves a `/nix/store` path at run
+time, because store paths change on every update and the old ones disappear with
+`nix-collect-garbage`. With the `clang` toolchain macOS CI and the dev shell use,
+they do not: it links against `/usr/lib/libc++` and `/usr/lib/libSystem`, and
+store paths reach the `.a` files only through debug info, which nothing resolves
+at link or run time.
+
+> [!WARNING]
+> This does not hold for `nix develop .#gcc` on macOS. There is no system
+> libstdc++, so GCC links its own from the store and every binary keeps a
+> `/nix/store` reference. That shell is fine for tooling, but it is not a build
+> configuration CI covers, and no dependency binaries are published for it.
+
+This is checked rather than assumed.
+[`bin/check-nix-store-refs.sh`](../../bin/check-nix-store-refs.sh) takes one file
+or directory and fails if a binary under it resolves a store path at run time.
+CI runs it over the build output and the Conan cache, and again in the upload job
+before anything is published. You can run it yourself:
+
+```bash
+bin/check-nix-store-refs.sh build
+bin/check-nix-store-refs.sh ~/.conan2-nix
+```
+
+It works on Linux too, but asserts something narrower there: the toolchain always
+writes the store into `PT_INTERP` and `RUNPATH`, and CI builds inside an image
+whose store is fixed for its lifetime, so that is fine. Only the binaries
+[`PatchNixBinary.cmake`](../../cmake/PatchNixBinary.cmake) retargets to the
+system loader have to be clean, and those are what CI checks:
+
+```bash
+bin/check-nix-store-refs.sh build/xrpld
+```
+
+### The libresolv stub
+
+This is not hypothetical: `xrpld` used to be caught by it. The c-ares package
+tells the linker to pass `-lresolv`, and nixpkgs keeps `libresolv` out of the
+macOS SDK and ships it as an ordinary store dylib — so every Nix-built `xrpld`
+recorded a `/nix/store/…-libresolv-93/lib/libresolv.9.dylib` load command and
+stopped running once that path was collected. Nothing in the link uses a single
+symbol from it.
+
+Both environments now put a stub on the linker search path
+(`libresolvSystemStub` in [`nix/darwin.nix`](../../nix/darwin.nix)): the
+same library with its install name set to `/usr/lib/libresolv.9.dylib`, which is
+exactly the load command the Apple Clang build records.
+
+Package IDs did not change, so Conan keeps serving anything built before the
+stub landed. If a binary fails to start with `Library not loaded: /nix/store/…`,
+see [that entry](./nix_troubleshooting.md#library-not-loaded-nixstore-from-a-binary-that-used-to-work)
+in the troubleshooting guide.
+
 ## Automatic Activation with direnv
 
 [direnv](https://direnv.net/) or [nix-direnv](https://github.com/nix-community/nix-direnv) can automatically activate the Nix development shell when you enter the repository directory.

docs/build/nix_troubleshooting.md

@@ -131,3 +131,91 @@ once it picks up that rebuild, then re-run the `grep libgit2` check above to
 confirm it reports `1.9.4` or newer.
 
 Until then, prefer the workarounds above.
+
+## `wint_t` / `uint32_t` errors from the Nix libc++ headers
+
+A build that mixes the Nix toolchain with the system SDK fails in libc++ itself,
+with errors that look nothing like your code:
+
+```
+/nix/store/...-libcxx-.../include/c++/v1/cwchar:136:9: error: target of using declaration conflicts with declaration already in scope
+  136 | using ::wint_t _LIBCPP_USING_IF_EXISTS;
+/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/sys/_types/_wint_t.h:32:25: note: target of using declaration
+...
+error: use of undeclared identifier 'UINT32_C'
+```
+
+The give-away is the second path: Nix's libc++ headers are being combined with
+the **Xcode Command Line Tools** SDK instead of the Nix one.
+
+### Why it happens
+
+`SDKROOT` and `DEVELOPER_DIR` are what point the toolchain at the Nix SDK, and
+they are not baked into the compiler — a dev shell gets them from the
+`apple-sdk` setup hook. CMake, finding neither, asks `xcrun`, which answers with
+the system SDK. Nix's `libc++` and Apple's headers then declare the same types
+twice.
+
+### Fix
+
+Run the build from inside the dev shell (`nix develop`), or from an environment
+that exports both variables. To confirm which SDK a configured build is using:
+
+```bash
+grep -o '\-isysroot [^ ]*' build/compile_commands.json | sort -u
+```
+
+It should print a `/nix/store/...-apple-sdk-*` path. If it prints
+`/Library/Developer/CommandLineTools/...`, re-configure from within the shell —
+CMake caches the sysroot, so an existing `build/` directory keeps the wrong one.
+
+## `Library not loaded: /nix/store/…` from a binary that used to work
+
+A binary stops starting after a `nix flake update`, or after
+`nix-collect-garbage` removes the paths the previous toolchain used:
+
+```
+dyld[57271]: Library not loaded: /nix/store/…-libresolv-93/lib/libresolv.9.dylib
+```
+
+[`bin/check-nix-store-refs.sh`](../../bin/check-nix-store-refs.sh) finds the same
+thing without having to run anything, and names the file:
+
+```
+$ bin/check-nix-store-refs.sh ~/.conan2-nix
+::error file=/Users/you/.conan2-nix/p/b/c-area24ded30c388c/p/bin/adig::references the Nix store at run time
+/Users/you/.conan2-nix/p/b/c-area24ded30c388c/p/bin/adig
+    /nix/store/p4lp3xq4imd1qzqh08x8vcq2zfhi7rca-libresolv-93/lib/libresolv.9.dylib
+/Users/you/.conan2-nix: checked 135, skipped 2495, 1 with Nix store references.
+```
+
+Conan's cache folders are named after a truncated package name plus a hash, so
+ask Conan which package the offending one belongs to — pass the folder holding
+the hash, not the file itself:
+
+```
+$ conan cache ref ~/.conan2-nix/p/b/c-area24ded30c388c
+c-ares/1.34.6#545240bb1c40e2cacd4362d6b8967650:dab5992496abe6d219defb7986ecbf367615a5e5#…
+```
+
+### Why it happens
+
+The binary records a store path that no longer exists. Nothing we build should:
+see [Prebuilt packages](./nix.md#prebuilt-packages) for why, and
+`libresolvSystemStub` in [`nix/darwin.nix`](../../nix/darwin.nix) for the one
+dependency that needed help to comply.
+
+A Conan package ID does not encode the nixpkgs revision, so a package built
+before that stub existed stays in your local cache and keeps being reused. The
+dev shell is also what tends to produce one: it is a slightly _less_ isolated
+build environment than CI's, because `mkShell` puts every tool's headers and
+libraries on the compiler's search path — which is how c-ares found the Nix
+`libresolv` in the first place.
+
+### Fix
+
+Drop that package and let Conan refetch or rebuild it:
+
+```bash
+conan remove 'c-ares/*'
+```

nix/ci-env.nix

@@ -1,67 +1,39 @@
+# The environment CI builds in: every tool on PATH, no Nix stdenv setup hooks.
+# Baked into the `nix-*` Docker images on Linux (see nix/docker), built on the
+# runner on macOS (see .github/actions/setup-nix-env).
 {
   pkgs,
   customGlibc,
   ...
 }:
 let
-  inherit (import ./packages.nix { inherit pkgs; })
-    commonPackages
-    gccVersion
-    llvmVersion
-    mkVersionedToolLinks
-    ;
+  inherit (import ./packages.nix { inherit pkgs; }) commonPackages;
 
-  # Custom-glibc toolchain, shared with the Linux dev shell (see compilers.nix).
-  inherit (import ./compilers.nix { inherit pkgs customGlibc; })
-    customGcc
-    customClang
-    customBinutils
-    customGcov
-    ;
+  # Each forces something absent on the other platform, so both stay lazy.
+  linux = import ./linux.nix { inherit pkgs customGlibc; };
+  darwin = import ./darwin.nix { inherit pkgs; };
 
-  # Strip the generic cc/c++/cpp symlinks from the clang wrapper so it can
-  # coexist with the gcc wrapper in buildEnv. gcc remains the default
-  # compiler (cc/c++/cpp); clang is invoked explicitly as clang/clang++.
-  customClangForCiEnv = pkgs.symlinkJoin {
-    name = "clang-wrapper-custom-for-ci-env";
-    paths = [ customClang ];
-    postBuild = ''
-      rm -f $out/bin/cc $out/bin/c++ $out/bin/cpp
-    '';
-  };
+  # What a buildEnv cannot express: environment variables. $GITHUB_ENV format;
+  # `set -a; . env; set +a` loads it in a shell.
+  darwinEnv = pkgs.writeTextDir "share/xrpld-ci-env/env" (
+    pkgs.lib.concatStrings (
+      pkgs.lib.mapAttrsToList (name: value: "${name}=${value}\n") (darwin.sdkEnv // darwin.libresolvEnv)
+    )
+  );
 
+  toolchain = if pkgs.stdenv.isLinux then linux.toolchain else (darwin.toolchain ++ [ darwinEnv ]);
 in
 {
   default = pkgs.buildEnv {
     name = "xrpld-ci-env";
-    paths = commonPackages ++ [
-      customGcc
-      customGcov
-      customClangForCiEnv
-      customBinutils
-      (mkVersionedToolLinks {
-        name = "gcc";
-        package = customGcc;
-        version = gccVersion;
-        tools = [
-          "gcc"
-          "g++"
-          "cpp"
-        ];
-      })
-      (mkVersionedToolLinks {
-        name = "clang";
-        package = customClang;
-        version = llvmVersion;
-        tools = [
-          "clang"
-          "clang++"
-        ];
-      })
-      # CA certificate bundle so HTTPS clients (git, curl, conan) can verify
-      # TLS connections without ca-certificates being installed in the system.
-      pkgs.cacert
-    ];
+    paths =
+      commonPackages
+      ++ toolchain
+      ++ [
+        # CA certificate bundle so HTTPS clients (git, curl, conan) can verify
+        # TLS connections without ca-certificates being installed in the system.
+        pkgs.cacert
+      ];
     pathsToLink = [
       "/bin"
       "/etc/ssl/certs"

nix/darwin.nix

@@ -0,0 +1,80 @@
+# The darwin toolchain, counterpart to linux.nix. Split by consumer: a dev
+# shell's stdenv provides the SDK variables, nothing provides libresolv.
+#
+# darwin only - `libresolv` does not exist on Linux.
+{ pkgs }:
+let
+  inherit (import ./packages.nix { inherit pkgs; })
+    llvmVersion
+    llvmPackages
+    mkVersionedToolLinks
+    ;
+
+  # nixpkgs keeps libresolv out of the macOS SDK, so neither c-ares' `-lresolv`
+  # nor grpc's <arpa/nameser.h> resolves. Headers can come from nixpkgs; the
+  # library cannot, or its store path lands in xrpld - hence this copy.
+  libresolvSystemStub =
+    pkgs.runCommand "libresolv-system-stub"
+      {
+        nativeBuildInputs = [ llvmPackages.bintools ];
+      }
+      ''
+        mkdir -p "$out/lib"
+        cp ${pkgs.darwin.libresolv}/lib/libresolv.9.dylib "$out/lib/"
+        chmod +w "$out/lib/libresolv.9.dylib"
+        llvm-install-name-tool -id /usr/lib/libresolv.9.dylib "$out/lib/libresolv.9.dylib"
+        ln -s libresolv.9.dylib "$out/lib/libresolv.dylib"
+      '';
+in
+{
+  # For an environment that only puts binaries on PATH.
+  toolchain = [
+    llvmPackages.clang
+    # The wrappers re-export only part of cctools; a bare env has no stdenv to
+    # supply the rest, and without `dsymutil` even `clang -g` cannot link. One
+    # by one, because buildEnv rejects any name a wrapper owns (notably `ld`).
+    (pkgs.linkFarm "cctools-extra" (
+      map
+        (tool: {
+          name = "bin/${tool}";
+          path = "${llvmPackages.clang.bintools.bintools}/bin/${tool}";
+        })
+        [
+          "codesign_allocate"
+          "dsymutil"
+          "dwarfdump"
+          "install_name_tool"
+          "lipo"
+          "otool"
+        ]
+    ))
+    (mkVersionedToolLinks {
+      name = "clang";
+      package = llvmPackages.clang;
+      version = llvmVersion;
+      tools = [
+        "clang"
+        "clang++"
+      ];
+    })
+  ];
+
+  # Without these CMake asks `xcrun` and gets the Command Line Tools SDK, whose
+  # headers clash with the Nix libc++ ones.
+  sdkEnv = {
+    DEVELOPER_DIR = "${pkgs.apple-sdk}";
+    SDKROOT = "${pkgs.apple-sdk}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk";
+  };
+
+  # Salted names: the wrappers only read plain NIX_CFLAGS_COMPILE / NIX_LDFLAGS
+  # through role variables a Nix stdenv would set. The salt is the target
+  # platform, so this fits the gcc wrapper too.
+  #
+  # No space after -isystem: these are written one per line as KEY=VALUE, and a
+  # shell sourcing that reads the space as the end of the assignment.
+  libresolvEnv = {
+    "NIX_CFLAGS_COMPILE_${llvmPackages.clang.suffixSalt}" =
+      "-isystem${pkgs.darwin.libresolv.dev}/include";
+    "NIX_LDFLAGS_${llvmPackages.clang.bintools.suffixSalt}" = "-L${libresolvSystemStub}/lib";
+  };
+}

nix/devshell.nix

@@ -14,21 +14,21 @@ let
   plainGccStdenv = pkgs."gcc${toString gccVersion}Stdenv";
   plainClangStdenv = llvmPackages.stdenv;
 
-  # Custom-glibc stdenvs, matching the CI environment (see compilers.nix). The
-  # pinned glibc snapshot only builds on Linux, so on darwin these fall back to
-  # the plain stdenvs; the `if isLinux` guard keeps `customGlibc` from being
-  # forced (and erroring) on macOS.
-  customCompilers = import ./compilers.nix { inherit pkgs customGlibc; };
-  customGccStdenv = if pkgs.stdenv.isLinux then customCompilers.customStdenv else plainGccStdenv;
-  customClangStdenv =
-    if pkgs.stdenv.isLinux then customCompilers.customClangStdenv else plainClangStdenv;
+  # Each forces something absent on the other platform, so both stay lazy.
+  linux = import ./linux.nix { inherit pkgs customGlibc; };
+  darwin = import ./darwin.nix { inherit pkgs; };
+
+  # Custom-glibc stdenvs, matching the CI environment. darwin has no custom
+  # glibc, so there they fall back to the plain nixpkgs stdenvs.
+  customGccStdenv = if pkgs.stdenv.isLinux then linux.gccStdenv else plainGccStdenv;
+  customClangStdenv = if pkgs.stdenv.isLinux then linux.clangStdenv else plainClangStdenv;
 
   # gcov matching each gcc shell, so `-Dcoverage=ON` builds work in the shell.
   plainGcov = mkGcov {
     name = "plain";
     cc = gccPackage.cc;
   };
-  customGccGcov = if pkgs.stdenv.isLinux then customCompilers.customGcov else plainGcov;
+  customGccGcov = if pkgs.stdenv.isLinux then linux.gcov else plainGcov;
 
   # Whole directory: init.sh locates the profiles relative to itself.
   conanDir = ../conan;
@@ -49,6 +49,16 @@ let
     unset _xrpl_conan_stamp
   '';
 
+  # Not sdkEnv: a shell's stdenv already sets that up. Prepended so the stub
+  # beats the nixpkgs libresolv this shell's tooling drags in.
+  darwinLibresolvHook = pkgs.lib.optionalString pkgs.stdenv.isDarwin (
+    pkgs.lib.concatLines (
+      pkgs.lib.mapAttrsToList (
+        name: value: ''export ${name}="${value} ''${${name}:-}"''
+      ) darwin.libresolvEnv
+    )
+  );
+
   # Shown when entering a *-plain shell. These exist only on Linux (see below),
   # where the stock toolchain diverges from CI.
   plainWarningHook = ''
@@ -106,6 +116,7 @@ let
         shellHook = ''
           echo "Welcome to xrpld development shell";
           ${compilerVersionHook}
+          ${darwinLibresolvHook}
           ${conanHook}
           ${warningHook}
         '';

nix/docker/Dockerfile

@@ -8,7 +8,7 @@ RUN mkdir -p ~/.config/nix && \
 
 # Copy our source and setup our working dir.
 COPY nix/ci-env.nix /tmp/build/nix/ci-env.nix
-COPY nix/compilers.nix /tmp/build/nix/compilers.nix
+COPY nix/linux.nix /tmp/build/nix/linux.nix
 COPY nix/packages.nix /tmp/build/nix/packages.nix
 COPY nix/utils.nix /tmp/build/nix/utils.nix
 COPY flake.nix /tmp/build/

nix/linux.nix

@@ -1,7 +1,9 @@
-# Custom-glibc compiler toolchain shared by the CI environment (ci-env.nix) and
-# the Linux dev shell (devshell.nix): gcc / clang / binutils rebuilt to target
-# the pinned custom glibc. Linux only — the pinned glibc snapshot does not build
-# on darwin, so callers must not evaluate this on macOS.
+# The Linux toolchain: gcc / clang / binutils rebuilt to target the pinned
+# custom glibc, shared by the CI environment (ci-env.nix) and the dev shell
+# (devshell.nix). The counterpart to darwin.nix.
+#
+# Linux only — the pinned glibc snapshot does not build on darwin, so callers
+# must not evaluate this on macOS.
 {
   pkgs,
   customGlibc,
@@ -9,9 +11,11 @@
 let
   inherit (import ./packages.nix { inherit pkgs; })
     gccPackage
+    gccVersion
     llvmPackages
     llvmVersion
     mkGcov
+    mkVersionedToolLinks
     ;
 
   # binutils wrapped to emit binaries that reference the custom glibc
@@ -103,15 +107,46 @@ let
       echo "-isystem ${customCompilerRt.dev}/include" >> $out/nix-support/cc-cflags
     '';
   };
+  # Strip the generic cc/c++/cpp symlinks from the clang wrapper so it can
+  # coexist with the gcc wrapper in buildEnv. gcc remains the default
+  # compiler (cc/c++/cpp); clang is invoked explicitly as clang/clang++.
+  customClangForCiEnv = pkgs.symlinkJoin {
+    name = "clang-wrapper-custom-for-ci-env";
+    paths = [ customClang ];
+    postBuild = ''
+      rm -f $out/bin/cc $out/bin/c++ $out/bin/cpp
+    '';
+  };
 in
 {
-  inherit
+  # For an environment that only puts binaries on PATH.
+  toolchain = [
     customGcc
-    customClang
-    customBinutils
-    customStdenv
     customGcov
-    ;
+    customClangForCiEnv
+    customBinutils
+    (mkVersionedToolLinks {
+      name = "gcc";
+      package = customGcc;
+      version = gccVersion;
+      tools = [
+        "gcc"
+        "g++"
+        "cpp"
+      ];
+    })
+    (mkVersionedToolLinks {
+      name = "clang";
+      package = customClang;
+      version = llvmVersion;
+      tools = [
+        "clang"
+        "clang++"
+      ];
+    })
+  ];
 
-  customClangStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customClang;
+  gccStdenv = customStdenv;
+  clangStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customClang;
+  gcov = customGcov;
 }