From 40cdf49d155322595764ff9b7edc33f1f553ab83 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Thu, 23 Jul 2026 20:05:24 +0100 Subject: [PATCH] build: Use custom libc in a devshell by default (#7852) --- docs/build/nix.md | 30 ++++++++--- nix/ci-env.nix | 104 +++---------------------------------- nix/compilers.nix | 117 ++++++++++++++++++++++++++++++++++++++++++ nix/devshell.nix | 91 +++++++++++++++++++++++++------- nix/docker/Dockerfile | 1 + nix/packages.nix | 12 +++++ 6 files changed, 234 insertions(+), 121 deletions(-) create mode 100644 nix/compilers.nix diff --git a/docs/build/nix.md b/docs/build/nix.md index d6e53a254a..b95b82fc42 100644 --- a/docs/build/nix.md +++ b/docs/build/nix.md @@ -38,8 +38,10 @@ The first time you run this command, it will take a few minutes to download and ### Platform notes -- **Linux**: `nix develop` gives you a shell with all the tooling necessary to - develop xrpld and with GCC 15.2 (also provided by Nix). There are no caveats. +- **Linux**: `nix develop` gives you a shell with all the tooling necessary to develop xrpld + and with the same GCC/glibc toolchain that Nix builds for CI. + See [Choosing a different compiler](#choosing-a-different-compiler) + for the custom-vs-plain toolchain trade-off. - **macOS**: `nix develop` gives you a full environment too, with Clang (and every other tool, including Conan) provided by Nix. To use your system-wide Apple Clang instead, enter `nix develop .#apple-clang`. Conan has no binary in @@ -63,8 +65,16 @@ The first time you run this command, it will take a few minutes to download and ### Choosing a different compiler A compiler can be chosen by providing its name with the `.#` prefix, e.g. `nix develop .#clang`. -The `.#gcc` and `.#clang` shells provide the same GCC and Clang versions used in CI -(pinned in [`nix/packages.nix`](../../nix/packages.nix)). + +On Linux, `.#gcc` and `.#clang` provide the exact toolchain CI uses: +the compiler (pinned in [`nix/packages.nix`](../../nix/packages.nix)) +rebuilt against the pinned custom glibc (see [`nix/compilers.nix`](../../nix/compilers.nix)). +Building that toolchain the first time is slow unless it is fetched from a Nix binary cache. +If you don't need the custom glibc, the Linux-only `.#gcc-plain` and `.#clang-plain` +give you the stock nixpkgs compilers of the same versions. +On macOS there is no custom glibc, so `.#gcc` and `.#clang` are already the plain nixpkgs toolchain, +and the `-plain` variants do not exist. + Use `nix flake show` to see all the available development shells. Use `nix develop .#no-compiler` to use the compiler from your system. @@ -72,14 +82,18 @@ Use `nix develop .#no-compiler` to use the compiler from your system. ### Example Usage ```bash -# Use GCC (same version as CI) +# Use GCC — same toolchain as CI (custom glibc on Linux) nix develop .#gcc -# Use Clang (same version as CI) +# Use Clang — same toolchain as CI (custom glibc on Linux) nix develop .#clang # Use default for your platform nix develop + +# Stock nixpkgs GCC/Clang, Linux only — skips the custom-glibc build, but does not match CI +nix develop .#gcc-plain +nix develop .#clang-plain ``` ### Using a different shell @@ -110,6 +124,10 @@ nix develop -c "$SHELL" Once inside the Nix development shell, follow the standard [build instructions](../../BUILD.md#steps). The Nix shell provides all necessary tools (CMake, Ninja, Conan, etc.). +Coverage builds (`-Dcoverage=ON`) work in the `gcc` shell (and `gcc-plain` on Linux): +each ships a `gcov` matching its compiler, since Nix's cc-wrapper does not expose one. +The `clang` shells do not include `llvm-cov`, so use a `gcc` shell for coverage. + ## Automatic Activation with direnv [direnv](https://direnv.net/) or [nix-direnv](https://github.com/nix-community/nix-direnv) can automatically activate the Nix development shell when you enter the repository directory. diff --git a/nix/ci-env.nix b/nix/ci-env.nix index 63bddb46d8..787b94406e 100644 --- a/nix/ci-env.nix +++ b/nix/ci-env.nix @@ -6,108 +6,18 @@ let inherit (import ./packages.nix { inherit pkgs; }) commonPackages - gccPackage gccVersion - llvmPackages llvmVersion mkVersionedToolLinks ; - # Underlying compiler toolchains to wrap (versions pinned in packages.nix). - customGccPackage = gccPackage; - customLlvmPackages = llvmPackages; - - # binutils wrapped to emit binaries that reference the custom glibc - # (dynamic linker path, library search path, RPATH). - customBinutils = pkgs.wrapBintoolsWith { - bintools = pkgs.binutils-unwrapped; - libc = customGlibc; - }; - - # Rebuild gcc (specifically libstdc++ / libgcc_s) against the custom - # glibc. The override swaps gcc.cc's bootstrap stdenv for one that uses - # the existing gcc binary but links against the custom glibc, so the - # resulting compiler ships runtime libraries that only reference symbols - # available in that glibc. - customGccCc = customGccPackage.cc.override { - stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv ( - pkgs.wrapCCWith { - cc = customGccPackage.cc; - libc = customGlibc; - bintools = customBinutils; - } - ); - }; - - # cc-wrapper around the rebuilt compiler, pointing at the custom glibc - # headers and libraries. This is what we actually expose to users. - customGcc = pkgs.wrapCCWith { - cc = customGccCc; - libc = customGlibc; - bintools = customBinutils; - }; - - # gcov ships in gcc's `cc` output, but the cc-wrapper doesn't expose it. - # Surface the gcov from our rebuilt gcc (linked against the custom glibc, so - # it runs under the loader installed in the image) and matching the exact - # compiler version, so gcovr can produce coverage reports in the CI env. - customGcov = pkgs.runCommand "gcov-custom-for-ci-env" { } '' - mkdir -p "$out/bin" - ln -s "${customGccCc}/bin/gcov" "$out/bin/gcov" - ''; - - # stdenv built around the rebuilt gcc / custom glibc. Used to rebuild - # compiler-rt below so its sanitizer runtimes see the custom glibc - # headers. - customStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customGcc; - - # Rebuild compiler-rt against the custom glibc so the sanitizer runtimes - # don't use glibc symbols (or sysconf constants like _SC_SIGSTKSZ) that - # only exist in newer glibc versions. scudo is dropped because its CMake - # includes CheckAtomic with -nostdinc++ in CMAKE_REQUIRED_FLAGS, which - # makes std::atomic unfindable in our stdenv; we don't use scudo (only - # asan/ubsan/tsan etc.). - customCompilerRt = - (customLlvmPackages.compiler-rt.override { - stdenv = customStdenv; - }).overrideAttrs - (old: { - postPatch = (old.postPatch or "") + '' - substituteInPlace lib/CMakeLists.txt \ - --replace-quiet 'add_subdirectory(scudo/standalone)' \ - '# scudo/standalone disabled in xrpld ci-env' - ''; - }); - - # cc-wrapper around clang, pointing at the custom glibc headers and - # libraries. Reuses the rebuilt gcc for libstdc++ / libgcc_s so that - # C++ binaries produced by clang also only reference symbols available - # in the custom glibc. compiler-rt is wired into a resource-root so - # sanitizer runtimes (libclang_rt.*.a) are found at link time; this - # mirrors what nixpkgs does internally when building llvmPackages.clang. - customClang = pkgs.wrapCCWith { - cc = customLlvmPackages.clang-unwrapped; - libc = customGlibc; - bintools = customBinutils; - gccForLibs = customGccCc; - extraPackages = [ customCompilerRt ]; - extraBuildCommands = '' - rsrc="$out/resource-root" - mkdir "$rsrc" - ln -s "${customLlvmPackages.clang-unwrapped.lib}/lib/clang/${toString llvmVersion}/include" "$rsrc/include" - ln -s "${customCompilerRt.out}/lib" "$rsrc/lib" - ln -s "${customCompilerRt.out}/share" "$rsrc/share" || true - echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags - # compiler-rt ships the sanitizer/profile/xray interface headers (e.g. - # ) in its `dev` output. In a normal Nix - # build these reach the include path because compiler-rt is propagated - # via depsTargetTargetPropagated and stdenv's setup hooks add its - # dev/include. The CI image runs clang outside a Nix stdenv (binaries - # on PATH, no setup hooks), so that never happens; add the headers - # explicitly. gcc ships its own copy, which is why this is clang-only. - echo "-isystem ${customCompilerRt.dev}/include" >> $out/nix-support/cc-cflags - ''; - }; + # Custom-glibc toolchain, shared with the Linux dev shell (see compilers.nix). + inherit (import ./compilers.nix { inherit pkgs customGlibc; }) + customGcc + customClang + customBinutils + customGcov + ; # Strip the generic cc/c++/cpp symlinks from the clang wrapper so it can # coexist with the gcc wrapper in buildEnv. gcc remains the default diff --git a/nix/compilers.nix b/nix/compilers.nix new file mode 100644 index 0000000000..90856afacc --- /dev/null +++ b/nix/compilers.nix @@ -0,0 +1,117 @@ +# Custom-glibc compiler toolchain shared by the CI environment (ci-env.nix) and +# the Linux dev shell (devshell.nix): gcc / clang / binutils rebuilt to target +# the pinned custom glibc. Linux only — the pinned glibc snapshot does not build +# on darwin, so callers must not evaluate this on macOS. +{ + pkgs, + customGlibc, +}: +let + inherit (import ./packages.nix { inherit pkgs; }) + gccPackage + llvmPackages + llvmVersion + mkGcov + ; + + # binutils wrapped to emit binaries that reference the custom glibc + # (dynamic linker path, library search path, RPATH). + customBinutils = pkgs.wrapBintoolsWith { + bintools = pkgs.binutils-unwrapped; + libc = customGlibc; + }; + + # Rebuild gcc (specifically libstdc++ / libgcc_s) against the custom + # glibc. The override swaps gcc.cc's bootstrap stdenv for one that uses + # the existing gcc binary but links against the custom glibc, so the + # resulting compiler ships runtime libraries that only reference symbols + # available in that glibc. + customGccCc = gccPackage.cc.override { + stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv ( + pkgs.wrapCCWith { + cc = gccPackage.cc; + libc = customGlibc; + bintools = customBinutils; + } + ); + }; + + # cc-wrapper around the rebuilt compiler, pointing at the custom glibc + # headers and libraries. This is what we actually expose to users. + customGcc = pkgs.wrapCCWith { + cc = customGccCc; + libc = customGlibc; + bintools = customBinutils; + }; + + # gcov matching the rebuilt gcc (linked against the custom glibc), so gcovr + # can produce coverage reports both in CI and in the dev shell. + customGcov = mkGcov { + name = "custom"; + cc = customGccCc; + }; + + # stdenv built around the rebuilt gcc / custom glibc. Exported as the dev + # shell's gcc stdenv, and used below to rebuild compiler-rt so its sanitizer + # runtimes see the custom glibc headers. + customStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customGcc; + + # Rebuild compiler-rt against the custom glibc so the sanitizer runtimes + # don't use glibc symbols (or sysconf constants like _SC_SIGSTKSZ) that + # only exist in newer glibc versions. scudo is dropped because its CMake + # includes CheckAtomic with -nostdinc++ in CMAKE_REQUIRED_FLAGS, which + # makes std::atomic unfindable in our stdenv; we don't use scudo (only + # asan/ubsan/tsan etc.). + customCompilerRt = + (llvmPackages.compiler-rt.override { + stdenv = customStdenv; + }).overrideAttrs + (old: { + postPatch = (old.postPatch or "") + '' + substituteInPlace lib/CMakeLists.txt \ + --replace-quiet 'add_subdirectory(scudo/standalone)' \ + '# scudo/standalone disabled in xrpld ci-env' + ''; + }); + + # cc-wrapper around clang, pointing at the custom glibc headers and + # libraries. Reuses the rebuilt gcc for libstdc++ / libgcc_s so that + # C++ binaries produced by clang also only reference symbols available + # in the custom glibc. compiler-rt is wired into a resource-root so + # sanitizer runtimes (libclang_rt.*.a) are found at link time; this + # mirrors what nixpkgs does internally when building llvmPackages.clang. + customClang = pkgs.wrapCCWith { + cc = llvmPackages.clang-unwrapped; + libc = customGlibc; + bintools = customBinutils; + gccForLibs = customGccCc; + extraPackages = [ customCompilerRt ]; + extraBuildCommands = '' + rsrc="$out/resource-root" + mkdir "$rsrc" + ln -s "${llvmPackages.clang-unwrapped.lib}/lib/clang/${toString llvmVersion}/include" "$rsrc/include" + ln -s "${customCompilerRt.out}/lib" "$rsrc/lib" + ln -s "${customCompilerRt.out}/share" "$rsrc/share" || true + echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags + # compiler-rt ships the sanitizer/profile/xray interface headers (e.g. + # ) in its `dev` output. In a normal Nix + # build these reach the include path because compiler-rt is propagated + # via depsTargetTargetPropagated and stdenv's setup hooks add its + # dev/include. The CI image runs clang outside a Nix stdenv (binaries + # on PATH, no setup hooks), so that never happens; add the headers + # explicitly. gcc ships its own copy, which is why this is clang-only. + echo "-isystem ${customCompilerRt.dev}/include" >> $out/nix-support/cc-cflags + ''; + }; +in +{ + inherit + customGcc + customClang + customBinutils + customStdenv + customGcov + ; + + customClangStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customClang; +} diff --git a/nix/devshell.nix b/nix/devshell.nix index 1316fe4234..9b453ddef8 100644 --- a/nix/devshell.nix +++ b/nix/devshell.nix @@ -1,16 +1,51 @@ -{ pkgs, ... }: +{ pkgs, customGlibc, ... }: let inherit (import ./packages.nix { inherit pkgs; }) commonPackages + gccPackage gccVersion llvmVersion llvmPackages mkVersionedToolLinks + mkGcov ; - # Plain nixpkgs stdenvs — no custom glibc, unlike ci-env.nix. - gccStdenv = pkgs."gcc${toString gccVersion}Stdenv"; - clangStdenv = llvmPackages.stdenv; + # Plain nixpkgs stdenvs — no custom glibc. + plainGccStdenv = pkgs."gcc${toString gccVersion}Stdenv"; + plainClangStdenv = llvmPackages.stdenv; + + # Custom-glibc stdenvs, matching the CI environment (see compilers.nix). The + # pinned glibc snapshot only builds on Linux, so on darwin these fall back to + # the plain stdenvs; the `if isLinux` guard keeps `customGlibc` from being + # forced (and erroring) on macOS. + customCompilers = import ./compilers.nix { inherit pkgs customGlibc; }; + customGccStdenv = if pkgs.stdenv.isLinux then customCompilers.customStdenv else plainGccStdenv; + customClangStdenv = + if pkgs.stdenv.isLinux then customCompilers.customClangStdenv else plainClangStdenv; + + # gcov matching each gcc shell, so `-Dcoverage=ON` builds work in the shell. + plainGcov = mkGcov { + name = "plain"; + cc = gccPackage.cc; + }; + customGccGcov = if pkgs.stdenv.isLinux then customCompilers.customGcov else plainGcov; + + # Shown when entering a *-plain shell. These exist only on Linux (see below), + # where the stock toolchain diverges from CI. + plainWarningHook = '' + echo "⚠️ WARNING: this is the stock nixpkgs toolchain and does not match CI's glibc. Prefer 'nix develop .#gcc' / '.#clang' unless you need to skip the custom-glibc build." + ''; + + # Tools to expose under version-suffixed names (see mkVersionedToolLinks). + gccVersionedTools = [ + "gcc" + "g++" + "cpp" + ]; + clangVersionedTools = [ + "clang" + "clang++" + ]; # compilerName is the command used to print the version, or null for none. makeShell = @@ -19,9 +54,11 @@ let compilerName, version ? null, versionedTools ? [ ], + extraPackages ? [ ], + warningHook ? "", }: let - compilerVersion = + compilerVersionHook = if compilerName == null then ''echo "No compiler specified - using system compiler"'' else @@ -37,10 +74,11 @@ let }); in (pkgs.mkShell.override { inherit stdenv; }) { - packages = commonPackages ++ versionedLinks; + packages = commonPackages ++ versionedLinks ++ extraPackages; shellHook = '' echo "Welcome to xrpld development shell"; - ${compilerVersion} + ${compilerVersionHook} + ${warningHook} ''; }; in @@ -48,25 +86,21 @@ rec { # macOS: Nix Clang. Linux: Nix GCC. default = if pkgs.stdenv.isDarwin then clang else gcc; + # gcc/clang use the custom-glibc toolchain, matching CI. On darwin there is no + # custom glibc, so they fall back to the plain nixpkgs toolchain. gcc = makeShell { - stdenv = gccStdenv; + stdenv = customGccStdenv; compilerName = "gcc"; version = gccVersion; - versionedTools = [ - "gcc" - "g++" - "cpp" - ]; + versionedTools = gccVersionedTools; + extraPackages = [ customGccGcov ]; }; clang = makeShell { - stdenv = clangStdenv; + stdenv = customClangStdenv; compilerName = "clang"; version = llvmVersion; - versionedTools = [ - "clang" - "clang++" - ]; + versionedTools = clangVersionedTools; }; # Nix provides no compiler; use the one from your system (e.g. Apple Clang). @@ -76,3 +110,24 @@ rec { }; apple-clang = no-compiler; } +# The *-plain shells (stock nixpkgs toolchain) exist only on Linux: on darwin +# gcc/clang are already plain, so these would be redundant and are omitted, which +# makes `nix develop .#gcc-plain` fail there rather than silently aliasing gcc. +// pkgs.lib.optionalAttrs pkgs.stdenv.isLinux { + gcc-plain = makeShell { + stdenv = plainGccStdenv; + compilerName = "gcc"; + version = gccVersion; + versionedTools = gccVersionedTools; + extraPackages = [ plainGcov ]; + warningHook = plainWarningHook; + }; + + clang-plain = makeShell { + stdenv = plainClangStdenv; + compilerName = "clang"; + version = llvmVersion; + versionedTools = clangVersionedTools; + warningHook = plainWarningHook; + }; +} diff --git a/nix/docker/Dockerfile b/nix/docker/Dockerfile index 8b851fd9e0..53b646ac7c 100644 --- a/nix/docker/Dockerfile +++ b/nix/docker/Dockerfile @@ -8,6 +8,7 @@ RUN mkdir -p ~/.config/nix && \ # Copy our source and setup our working dir. COPY nix/ci-env.nix /tmp/build/nix/ci-env.nix +COPY nix/compilers.nix /tmp/build/nix/compilers.nix COPY nix/packages.nix /tmp/build/nix/packages.nix COPY nix/utils.nix /tmp/build/nix/utils.nix COPY flake.nix /tmp/build/ diff --git a/nix/packages.nix b/nix/packages.nix index 1d9b6cd2f8..3cf0f57c3e 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -48,6 +48,17 @@ let }) tools ); + # The cc-wrapper doesn't re-export gcov, but coverage tooling (gcovr) needs a + # gcov that exactly matches the compiler. Surface it from a gcc `cc` output. + mkGcov = + { name, cc }: + pkgs.linkFarm "gcov-${name}" [ + { + name = "bin/gcov"; + path = "${cc}/bin/gcov"; + } + ]; + clangToolLinks = mkVersionedToolLinks { name = "clang-tools"; package = clangTools; @@ -72,6 +83,7 @@ in gccPackage llvmPackages mkVersionedToolLinks + mkGcov ; commonPackages = with pkgs; [