build: Use custom libc in a devshell by default (#7852)

docs/build/nix.md

@@ -38,8 +38,10 @@ The first time you run this command, it will take a few minutes to download and
 
 ### Platform notes
 
-- **Linux**: `nix develop` gives you a shell with all the tooling necessary to
-  develop xrpld and with GCC 15.2 (also provided by Nix). There are no caveats.
+- **Linux**: `nix develop` gives you a shell with all the tooling necessary to develop xrpld
+  and with the same GCC/glibc toolchain that Nix builds for CI.
+  See [Choosing a different compiler](#choosing-a-different-compiler)
+  for the custom-vs-plain toolchain trade-off.
 - **macOS**: `nix develop` gives you a full environment too, with Clang (and
   every other tool, including Conan) provided by Nix. To use your system-wide
   Apple Clang instead, enter `nix develop .#apple-clang`. Conan has no binary in
@@ -63,8 +65,16 @@ The first time you run this command, it will take a few minutes to download and
 ### Choosing a different compiler
 
 A compiler can be chosen by providing its name with the `.#` prefix, e.g. `nix develop .#clang`.
-The `.#gcc` and `.#clang` shells provide the same GCC and Clang versions used in CI
-(pinned in [`nix/packages.nix`](../../nix/packages.nix)).
+
+On Linux, `.#gcc` and `.#clang` provide the exact toolchain CI uses:
+the compiler (pinned in [`nix/packages.nix`](../../nix/packages.nix))
+rebuilt against the pinned custom glibc (see [`nix/compilers.nix`](../../nix/compilers.nix)).
+Building that toolchain the first time is slow unless it is fetched from a Nix binary cache.
+If you don't need the custom glibc, the Linux-only `.#gcc-plain` and `.#clang-plain`
+give you the stock nixpkgs compilers of the same versions.
+On macOS there is no custom glibc, so `.#gcc` and `.#clang` are already the plain nixpkgs toolchain,
+and the `-plain` variants do not exist.
+
 Use `nix flake show` to see all the available development shells.
 
 Use `nix develop .#no-compiler` to use the compiler from your system.
@@ -72,14 +82,18 @@ Use `nix develop .#no-compiler` to use the compiler from your system.
 ### Example Usage
 
 ```bash
-# Use GCC (same version as CI)
+# Use GCC — same toolchain as CI (custom glibc on Linux)
 nix develop .#gcc
 
-# Use Clang (same version as CI)
+# Use Clang — same toolchain as CI (custom glibc on Linux)
 nix develop .#clang
 
 # Use default for your platform
 nix develop
+
+# Stock nixpkgs GCC/Clang, Linux only — skips the custom-glibc build, but does not match CI
+nix develop .#gcc-plain
+nix develop .#clang-plain
 ```
 
 ### Using a different shell
@@ -110,6 +124,10 @@ nix develop -c "$SHELL"
 
 Once inside the Nix development shell, follow the standard [build instructions](../../BUILD.md#steps). The Nix shell provides all necessary tools (CMake, Ninja, Conan, etc.).
 
+Coverage builds (`-Dcoverage=ON`) work in the `gcc` shell (and `gcc-plain` on Linux):
+each ships a `gcov` matching its compiler, since Nix's cc-wrapper does not expose one.
+The `clang` shells do not include `llvm-cov`, so use a `gcc` shell for coverage.
+
 ## Automatic Activation with direnv
 
 [direnv](https://direnv.net/) or [nix-direnv](https://github.com/nix-community/nix-direnv) can automatically activate the Nix development shell when you enter the repository directory.

nix/ci-env.nix

@@ -6,108 +6,18 @@
 let
   inherit (import ./packages.nix { inherit pkgs; })
     commonPackages
-    gccPackage
     gccVersion
-    llvmPackages
     llvmVersion
     mkVersionedToolLinks
     ;
 
-  # Underlying compiler toolchains to wrap (versions pinned in packages.nix).
-  customGccPackage = gccPackage;
-  customLlvmPackages = llvmPackages;
-
-  # binutils wrapped to emit binaries that reference the custom glibc
-  # (dynamic linker path, library search path, RPATH).
-  customBinutils = pkgs.wrapBintoolsWith {
-    bintools = pkgs.binutils-unwrapped;
-    libc = customGlibc;
-  };
-
-  # Rebuild gcc (specifically libstdc++ / libgcc_s) against the custom
-  # glibc. The override swaps gcc.cc's bootstrap stdenv for one that uses
-  # the existing gcc binary but links against the custom glibc, so the
-  # resulting compiler ships runtime libraries that only reference symbols
-  # available in that glibc.
-  customGccCc = customGccPackage.cc.override {
-    stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv (
-      pkgs.wrapCCWith {
-        cc = customGccPackage.cc;
-        libc = customGlibc;
-        bintools = customBinutils;
-      }
-    );
-  };
-
-  # cc-wrapper around the rebuilt compiler, pointing at the custom glibc
-  # headers and libraries. This is what we actually expose to users.
-  customGcc = pkgs.wrapCCWith {
-    cc = customGccCc;
-    libc = customGlibc;
-    bintools = customBinutils;
-  };
-
-  # gcov ships in gcc's `cc` output, but the cc-wrapper doesn't expose it.
-  # Surface the gcov from our rebuilt gcc (linked against the custom glibc, so
-  # it runs under the loader installed in the image) and matching the exact
-  # compiler version, so gcovr can produce coverage reports in the CI env.
-  customGcov = pkgs.runCommand "gcov-custom-for-ci-env" { } ''
-    mkdir -p "$out/bin"
-    ln -s "${customGccCc}/bin/gcov" "$out/bin/gcov"
-  '';
-
-  # stdenv built around the rebuilt gcc / custom glibc. Used to rebuild
-  # compiler-rt below so its sanitizer runtimes see the custom glibc
-  # headers.
-  customStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customGcc;
-
-  # Rebuild compiler-rt against the custom glibc so the sanitizer runtimes
-  # don't use glibc symbols (or sysconf constants like _SC_SIGSTKSZ) that
-  # only exist in newer glibc versions. scudo is dropped because its CMake
-  # includes CheckAtomic with -nostdinc++ in CMAKE_REQUIRED_FLAGS, which
-  # makes std::atomic unfindable in our stdenv; we don't use scudo (only
-  # asan/ubsan/tsan etc.).
-  customCompilerRt =
-    (customLlvmPackages.compiler-rt.override {
-      stdenv = customStdenv;
-    }).overrideAttrs
-      (old: {
-        postPatch = (old.postPatch or "") + ''
-          substituteInPlace lib/CMakeLists.txt \
-            --replace-quiet 'add_subdirectory(scudo/standalone)' \
-                            '# scudo/standalone disabled in xrpld ci-env'
-        '';
-      });
-
-  # cc-wrapper around clang, pointing at the custom glibc headers and
-  # libraries. Reuses the rebuilt gcc for libstdc++ / libgcc_s so that
-  # C++ binaries produced by clang also only reference symbols available
-  # in the custom glibc. compiler-rt is wired into a resource-root so
-  # sanitizer runtimes (libclang_rt.*.a) are found at link time; this
-  # mirrors what nixpkgs does internally when building llvmPackages.clang.
-  customClang = pkgs.wrapCCWith {
-    cc = customLlvmPackages.clang-unwrapped;
-    libc = customGlibc;
-    bintools = customBinutils;
-    gccForLibs = customGccCc;
-    extraPackages = [ customCompilerRt ];
-    extraBuildCommands = ''
-      rsrc="$out/resource-root"
-      mkdir "$rsrc"
-      ln -s "${customLlvmPackages.clang-unwrapped.lib}/lib/clang/${toString llvmVersion}/include" "$rsrc/include"
-      ln -s "${customCompilerRt.out}/lib" "$rsrc/lib"
-      ln -s "${customCompilerRt.out}/share" "$rsrc/share" || true
-      echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags
-      # compiler-rt ships the sanitizer/profile/xray interface headers (e.g.
-      # <sanitizer/lsan_interface.h>) in its `dev` output. In a normal Nix
-      # build these reach the include path because compiler-rt is propagated
-      # via depsTargetTargetPropagated and stdenv's setup hooks add its
-      # dev/include. The CI image runs clang outside a Nix stdenv (binaries
-      # on PATH, no setup hooks), so that never happens; add the headers
-      # explicitly. gcc ships its own copy, which is why this is clang-only.
-      echo "-isystem ${customCompilerRt.dev}/include" >> $out/nix-support/cc-cflags
-    '';
-  };
+  # Custom-glibc toolchain, shared with the Linux dev shell (see compilers.nix).
+  inherit (import ./compilers.nix { inherit pkgs customGlibc; })
+    customGcc
+    customClang
+    customBinutils
+    customGcov
+    ;
 
   # Strip the generic cc/c++/cpp symlinks from the clang wrapper so it can
   # coexist with the gcc wrapper in buildEnv. gcc remains the default

nix/compilers.nix

@@ -0,0 +1,117 @@
+# Custom-glibc compiler toolchain shared by the CI environment (ci-env.nix) and
+# the Linux dev shell (devshell.nix): gcc / clang / binutils rebuilt to target
+# the pinned custom glibc. Linux only — the pinned glibc snapshot does not build
+# on darwin, so callers must not evaluate this on macOS.
+{
+  pkgs,
+  customGlibc,
+}:
+let
+  inherit (import ./packages.nix { inherit pkgs; })
+    gccPackage
+    llvmPackages
+    llvmVersion
+    mkGcov
+    ;
+
+  # binutils wrapped to emit binaries that reference the custom glibc
+  # (dynamic linker path, library search path, RPATH).
+  customBinutils = pkgs.wrapBintoolsWith {
+    bintools = pkgs.binutils-unwrapped;
+    libc = customGlibc;
+  };
+
+  # Rebuild gcc (specifically libstdc++ / libgcc_s) against the custom
+  # glibc. The override swaps gcc.cc's bootstrap stdenv for one that uses
+  # the existing gcc binary but links against the custom glibc, so the
+  # resulting compiler ships runtime libraries that only reference symbols
+  # available in that glibc.
+  customGccCc = gccPackage.cc.override {
+    stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv (
+      pkgs.wrapCCWith {
+        cc = gccPackage.cc;
+        libc = customGlibc;
+        bintools = customBinutils;
+      }
+    );
+  };
+
+  # cc-wrapper around the rebuilt compiler, pointing at the custom glibc
+  # headers and libraries. This is what we actually expose to users.
+  customGcc = pkgs.wrapCCWith {
+    cc = customGccCc;
+    libc = customGlibc;
+    bintools = customBinutils;
+  };
+
+  # gcov matching the rebuilt gcc (linked against the custom glibc), so gcovr
+  # can produce coverage reports both in CI and in the dev shell.
+  customGcov = mkGcov {
+    name = "custom";
+    cc = customGccCc;
+  };
+
+  # stdenv built around the rebuilt gcc / custom glibc. Exported as the dev
+  # shell's gcc stdenv, and used below to rebuild compiler-rt so its sanitizer
+  # runtimes see the custom glibc headers.
+  customStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customGcc;
+
+  # Rebuild compiler-rt against the custom glibc so the sanitizer runtimes
+  # don't use glibc symbols (or sysconf constants like _SC_SIGSTKSZ) that
+  # only exist in newer glibc versions. scudo is dropped because its CMake
+  # includes CheckAtomic with -nostdinc++ in CMAKE_REQUIRED_FLAGS, which
+  # makes std::atomic unfindable in our stdenv; we don't use scudo (only
+  # asan/ubsan/tsan etc.).
+  customCompilerRt =
+    (llvmPackages.compiler-rt.override {
+      stdenv = customStdenv;
+    }).overrideAttrs
+      (old: {
+        postPatch = (old.postPatch or "") + ''
+          substituteInPlace lib/CMakeLists.txt \
+            --replace-quiet 'add_subdirectory(scudo/standalone)' \
+                            '# scudo/standalone disabled in xrpld ci-env'
+        '';
+      });
+
+  # cc-wrapper around clang, pointing at the custom glibc headers and
+  # libraries. Reuses the rebuilt gcc for libstdc++ / libgcc_s so that
+  # C++ binaries produced by clang also only reference symbols available
+  # in the custom glibc. compiler-rt is wired into a resource-root so
+  # sanitizer runtimes (libclang_rt.*.a) are found at link time; this
+  # mirrors what nixpkgs does internally when building llvmPackages.clang.
+  customClang = pkgs.wrapCCWith {
+    cc = llvmPackages.clang-unwrapped;
+    libc = customGlibc;
+    bintools = customBinutils;
+    gccForLibs = customGccCc;
+    extraPackages = [ customCompilerRt ];
+    extraBuildCommands = ''
+      rsrc="$out/resource-root"
+      mkdir "$rsrc"
+      ln -s "${llvmPackages.clang-unwrapped.lib}/lib/clang/${toString llvmVersion}/include" "$rsrc/include"
+      ln -s "${customCompilerRt.out}/lib" "$rsrc/lib"
+      ln -s "${customCompilerRt.out}/share" "$rsrc/share" || true
+      echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags
+      # compiler-rt ships the sanitizer/profile/xray interface headers (e.g.
+      # <sanitizer/lsan_interface.h>) in its `dev` output. In a normal Nix
+      # build these reach the include path because compiler-rt is propagated
+      # via depsTargetTargetPropagated and stdenv's setup hooks add its
+      # dev/include. The CI image runs clang outside a Nix stdenv (binaries
+      # on PATH, no setup hooks), so that never happens; add the headers
+      # explicitly. gcc ships its own copy, which is why this is clang-only.
+      echo "-isystem ${customCompilerRt.dev}/include" >> $out/nix-support/cc-cflags
+    '';
+  };
+in
+{
+  inherit
+    customGcc
+    customClang
+    customBinutils
+    customStdenv
+    customGcov
+    ;
+
+  customClangStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customClang;
+}

nix/devshell.nix

@@ -1,16 +1,51 @@
-{ pkgs, ... }:
+{ pkgs, customGlibc, ... }:
 let
   inherit (import ./packages.nix { inherit pkgs; })
     commonPackages
+    gccPackage
     gccVersion
     llvmVersion
     llvmPackages
     mkVersionedToolLinks
+    mkGcov
     ;
 
-  # Plain nixpkgs stdenvs — no custom glibc, unlike ci-env.nix.
-  gccStdenv = pkgs."gcc${toString gccVersion}Stdenv";
-  clangStdenv = llvmPackages.stdenv;
+  # Plain nixpkgs stdenvs — no custom glibc.
+  plainGccStdenv = pkgs."gcc${toString gccVersion}Stdenv";
+  plainClangStdenv = llvmPackages.stdenv;
+
+  # Custom-glibc stdenvs, matching the CI environment (see compilers.nix). The
+  # pinned glibc snapshot only builds on Linux, so on darwin these fall back to
+  # the plain stdenvs; the `if isLinux` guard keeps `customGlibc` from being
+  # forced (and erroring) on macOS.
+  customCompilers = import ./compilers.nix { inherit pkgs customGlibc; };
+  customGccStdenv = if pkgs.stdenv.isLinux then customCompilers.customStdenv else plainGccStdenv;
+  customClangStdenv =
+    if pkgs.stdenv.isLinux then customCompilers.customClangStdenv else plainClangStdenv;
+
+  # gcov matching each gcc shell, so `-Dcoverage=ON` builds work in the shell.
+  plainGcov = mkGcov {
+    name = "plain";
+    cc = gccPackage.cc;
+  };
+  customGccGcov = if pkgs.stdenv.isLinux then customCompilers.customGcov else plainGcov;
+
+  # Shown when entering a *-plain shell. These exist only on Linux (see below),
+  # where the stock toolchain diverges from CI.
+  plainWarningHook = ''
+    echo "⚠️  WARNING: this is the stock nixpkgs toolchain and does not match CI's glibc. Prefer 'nix develop .#gcc' / '.#clang' unless you need to skip the custom-glibc build."
+  '';
+
+  # Tools to expose under version-suffixed names (see mkVersionedToolLinks).
+  gccVersionedTools = [
+    "gcc"
+    "g++"
+    "cpp"
+  ];
+  clangVersionedTools = [
+    "clang"
+    "clang++"
+  ];
 
   # compilerName is the command used to print the version, or null for none.
   makeShell =
@@ -19,9 +54,11 @@ let
       compilerName,
       version ? null,
       versionedTools ? [ ],
+      extraPackages ? [ ],
+      warningHook ? "",
     }:
     let
-      compilerVersion =
+      compilerVersionHook =
         if compilerName == null then
           ''echo "No compiler specified - using system compiler"''
         else
@@ -37,10 +74,11 @@ let
       });
     in
     (pkgs.mkShell.override { inherit stdenv; }) {
-      packages = commonPackages ++ versionedLinks;
+      packages = commonPackages ++ versionedLinks ++ extraPackages;
       shellHook = ''
         echo "Welcome to xrpld development shell";
-        ${compilerVersion}
+        ${compilerVersionHook}
+        ${warningHook}
       '';
     };
 in
@@ -48,25 +86,21 @@ rec {
   # macOS: Nix Clang. Linux: Nix GCC.
   default = if pkgs.stdenv.isDarwin then clang else gcc;
 
+  # gcc/clang use the custom-glibc toolchain, matching CI. On darwin there is no
+  # custom glibc, so they fall back to the plain nixpkgs toolchain.
   gcc = makeShell {
-    stdenv = gccStdenv;
+    stdenv = customGccStdenv;
     compilerName = "gcc";
     version = gccVersion;
-    versionedTools = [
-      "gcc"
-      "g++"
-      "cpp"
-    ];
+    versionedTools = gccVersionedTools;
+    extraPackages = [ customGccGcov ];
   };
 
   clang = makeShell {
-    stdenv = clangStdenv;
+    stdenv = customClangStdenv;
     compilerName = "clang";
     version = llvmVersion;
-    versionedTools = [
-      "clang"
-      "clang++"
-    ];
+    versionedTools = clangVersionedTools;
   };
 
   # Nix provides no compiler; use the one from your system (e.g. Apple Clang).
@@ -76,3 +110,24 @@ rec {
   };
   apple-clang = no-compiler;
 }
+# The *-plain shells (stock nixpkgs toolchain) exist only on Linux: on darwin
+# gcc/clang are already plain, so these would be redundant and are omitted, which
+# makes `nix develop .#gcc-plain` fail there rather than silently aliasing gcc.
+// pkgs.lib.optionalAttrs pkgs.stdenv.isLinux {
+  gcc-plain = makeShell {
+    stdenv = plainGccStdenv;
+    compilerName = "gcc";
+    version = gccVersion;
+    versionedTools = gccVersionedTools;
+    extraPackages = [ plainGcov ];
+    warningHook = plainWarningHook;
+  };
+
+  clang-plain = makeShell {
+    stdenv = plainClangStdenv;
+    compilerName = "clang";
+    version = llvmVersion;
+    versionedTools = clangVersionedTools;
+    warningHook = plainWarningHook;
+  };
+}

nix/docker/Dockerfile

@@ -8,6 +8,7 @@ RUN mkdir -p ~/.config/nix && \
 
 # Copy our source and setup our working dir.
 COPY nix/ci-env.nix /tmp/build/nix/ci-env.nix
+COPY nix/compilers.nix /tmp/build/nix/compilers.nix
 COPY nix/packages.nix /tmp/build/nix/packages.nix
 COPY nix/utils.nix /tmp/build/nix/utils.nix
 COPY flake.nix /tmp/build/

nix/packages.nix

@@ -48,6 +48,17 @@ let
       }) tools
     );
 
+  # The cc-wrapper doesn't re-export gcov, but coverage tooling (gcovr) needs a
+  # gcov that exactly matches the compiler. Surface it from a gcc `cc` output.
+  mkGcov =
+    { name, cc }:
+    pkgs.linkFarm "gcov-${name}" [
+      {
+        name = "bin/gcov";
+        path = "${cc}/bin/gcov";
+      }
+    ];
+
   clangToolLinks = mkVersionedToolLinks {
     name = "clang-tools";
     package = clangTools;
@@ -72,6 +83,7 @@ in
     gccPackage
     llvmPackages
     mkVersionedToolLinks
+    mkGcov
     ;
 
   commonPackages = with pkgs; [