diff --git a/.cspell.config.yaml b/.cspell.config.yaml index e220cd0249..c862379f08 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -216,6 +216,7 @@ words: - Nyffenegger - onlatest - ostr + - oxalica - pargs - partitioner - paychan @@ -340,6 +341,7 @@ words: - unsquelch - unsquelched - unsquelching + - unsuffixed - unvalidated - unveto - unvetoed diff --git a/.github/workflows/build-nix-images.yml b/.github/workflows/build-nix-images.yml index da28b8db49..fd480cff67 100644 --- a/.github/workflows/build-nix-images.yml +++ b/.github/workflows/build-nix-images.yml @@ -8,6 +8,7 @@ on: - ".github/workflows/build-nix-images.yml" - "flake.nix" - "flake.lock" + - "rust-toolchain.toml" - "nix/**" - "!nix/docker/README.md" - "!nix/devshell.nix" @@ -18,6 +19,7 @@ on: - ".github/workflows/build-nix-images.yml" - "flake.nix" - "flake.lock" + - "rust-toolchain.toml" - "nix/**" - "!nix/docker/README.md" - "!nix/devshell.nix" diff --git a/.gitignore b/.gitignore index 6bd34ece04..13b59a7e2c 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,9 @@ DerivedData # Python __pycache__ +# Rust build artifacts. +target/ + # Direnv's directory /.direnv diff --git a/flake.lock b/flake.lock index 80243ccf15..cd9289c998 100644 --- a/flake.lock +++ b/flake.lock @@ -36,7 +36,28 @@ "root": { "inputs": { "nixpkgs": "nixpkgs", - "nixpkgs-custom-glibc": "nixpkgs-custom-glibc" + "nixpkgs-custom-glibc": "nixpkgs-custom-glibc", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1784611586, + "narHash": "sha256-OfqgY+0hp/zseZB7uyH0U8kIDPS4scZZCyAurEplvG0=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "14f58845249f3552a89b07772626b8d3c632fa86", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index c52f4d050e..ee2fd13efc 100644 --- a/flake.nix +++ b/flake.nix @@ -10,12 +10,25 @@ url = "github:NixOS/nixpkgs/9cd98386a38891d1074fc18036b842dc4416f562"; flake = false; }; + # Pinned Rust toolchains, delivered from the Nix store. Lets the Nix CI + # image and dev shell honour the single `rust-toolchain.toml` pin (shared + # with the rustup-based non-Nix runners) while staying hermetic — the + # toolchain lands in the image's Nix closure and is locked by flake.lock. + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = - { nixpkgs, nixpkgs-custom-glibc, ... }: + { + nixpkgs, + nixpkgs-custom-glibc, + rust-overlay, + ... + }: let - forEachSystem = import ./nix/utils.nix { inherit nixpkgs nixpkgs-custom-glibc; }; + forEachSystem = import ./nix/utils.nix { inherit nixpkgs nixpkgs-custom-glibc rust-overlay; }; in { devShells = forEachSystem (import ./nix/devshell.nix); diff --git a/nix/ci-env.nix b/nix/ci-env.nix index 9b754af97d..63bddb46d8 100644 --- a/nix/ci-env.nix +++ b/nix/ci-env.nix @@ -7,8 +7,10 @@ let inherit (import ./packages.nix { inherit pkgs; }) commonPackages gccPackage + gccVersion llvmPackages llvmVersion + mkVersionedToolLinks ; # Underlying compiler toolchains to wrap (versions pinned in packages.nix). @@ -127,6 +129,25 @@ in customGcov customClangForCiEnv customBinutils + (mkVersionedToolLinks { + name = "gcc"; + package = customGcc; + version = gccVersion; + tools = [ + "gcc" + "g++" + "cpp" + ]; + }) + (mkVersionedToolLinks { + name = "clang"; + package = customClang; + version = llvmVersion; + tools = [ + "clang" + "clang++" + ]; + }) # CA certificate bundle so HTTPS clients (git, curl, conan) can verify # TLS connections without ca-certificates being installed in the system. pkgs.cacert diff --git a/nix/devshell.nix b/nix/devshell.nix index 34f173ef08..1316fe4234 100644 --- a/nix/devshell.nix +++ b/nix/devshell.nix @@ -3,7 +3,9 @@ let inherit (import ./packages.nix { inherit pkgs; }) commonPackages gccVersion + llvmVersion llvmPackages + mkVersionedToolLinks ; # Plain nixpkgs stdenvs — no custom glibc, unlike ci-env.nix. @@ -15,6 +17,8 @@ let { stdenv, compilerName, + version ? null, + versionedTools ? [ ], }: let compilerVersion = @@ -25,9 +29,15 @@ let echo "Compiler: " ${compilerName} --version ''; + versionedLinks = pkgs.lib.optional (version != null) (mkVersionedToolLinks { + name = compilerName; + package = stdenv.cc; + inherit version; + tools = versionedTools; + }); in (pkgs.mkShell.override { inherit stdenv; }) { - packages = commonPackages; + packages = commonPackages ++ versionedLinks; shellHook = '' echo "Welcome to xrpld development shell"; ${compilerVersion} @@ -41,11 +51,22 @@ rec { gcc = makeShell { stdenv = gccStdenv; compilerName = "gcc"; + version = gccVersion; + versionedTools = [ + "gcc" + "g++" + "cpp" + ]; }; clang = makeShell { stdenv = clangStdenv; compilerName = "clang"; + version = llvmVersion; + versionedTools = [ + "clang" + "clang++" + ]; }; # Nix provides no compiler; use the one from your system (e.g. Apple Clang). diff --git a/nix/docker/Dockerfile b/nix/docker/Dockerfile index 7222cc8fa8..8b851fd9e0 100644 --- a/nix/docker/Dockerfile +++ b/nix/docker/Dockerfile @@ -12,6 +12,7 @@ COPY nix/packages.nix /tmp/build/nix/packages.nix COPY nix/utils.nix /tmp/build/nix/utils.nix COPY flake.nix /tmp/build/ COPY flake.lock /tmp/build/ +COPY rust-toolchain.toml /tmp/build/ WORKDIR /tmp/build FROM builder-source AS builder diff --git a/nix/docker/README.md b/nix/docker/README.md index 23ad05049e..7b4d4b9387 100644 --- a/nix/docker/README.md +++ b/nix/docker/README.md @@ -47,7 +47,9 @@ work without `ca-certificates` being installed in the base image. [`test_files/cpp/sources/`](./test_files/cpp/sources) with both `g++` and `clang++`, and sanitizers, and - compiles the Rust test programs in - [`test_files/rust/sources/`](./test_files/rust/sources) with `rustc`. + [`test_files/rust/sources/`](./test_files/rust/sources) with `rustc`, and + builds the [`test_files/rust/proc_macro/`](./test_files/rust/proc_macro) + workspace with `cargo` to exercise proc-macro dylib loading. 3. **`tester`** — Start again from a clean `BASE_IMAGE` (no Nix toolchain), install only the sanitizer runtime libraries ([`install-sanitizer-libs.sh`](./install-sanitizer-libs.sh)), and run the @@ -76,20 +78,23 @@ toolchain being present at runtime. Two pieces make that work: [`loader-path.sh`](./loader-path.sh) reports the expected loader path for the current architecture, so we can patch the binaries to use the correct loader. -The build then verifies all of this end to end: the C++ test programs in -`test_files/cpp/sources/` (a regular binary plus ASan/TSan/UBSan variants) and -the Rust test programs in `test_files/rust/sources/` (a hello binary plus panic -and overflow-check variants) are compiled in `final`, their `PT_INTERP` is -patched to the target loader, and they are run in the clean `tester` stage to -confirm each emits the expected diagnostic on a stock base image. +The build then verifies all of this end to end, and the C++ and Rust programs +go through the same pipeline: each is compiled in `final`, has its `PT_INTERP` +patched to the target loader, and is then run in the clean `tester` stage to +confirm it emits the expected diagnostic on a stock base image. The C++ programs +are in `test_files/cpp/sources/` (a regular binary plus ASan/TSan/UBSan +variants); the Rust programs are in `test_files/rust/sources/` (a hello binary +plus panic and overflow-check variants), plus the `test_files/rust/proc_macro/` +workspace — a crate whose compilation additionally loads a proc-macro dylib, and +whose resulting binary is patched and run like the others. ## Files -| File | Purpose | -| ----------------------------------------------------------------------- | ----------------------------------------------------------------------------- | -| [`./Dockerfile`](./Dockerfile) | Multi-stage build described above. | -| [`./loader-path.sh`](./loader-path.sh) | Print the dynamic-linker (`PT_INTERP`) path for the current architecture. | -| [`./test_files/cpp/`](./test_files/cpp) | C++ sanitizer smoke test: sources + compile/run scripts. | -| [`./test_files/rust/`](./test_files/rust) | Rust rustc smoke test: sources + compile/run scripts. | -| [`/bin/check-tools.sh`](../../bin/check-tools.sh) | Verify every expected tools are present and runnable. | -| [`/bin/install-sanitizer-libs.sh`](../../bin/install-sanitizer-libs.sh) | Install `libasan`/`libtsan`/`libubsan` runtimes on the supported base images. | +| File | Purpose | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------ | +| [`./Dockerfile`](./Dockerfile) | Multi-stage build described above. | +| [`./loader-path.sh`](./loader-path.sh) | Print the dynamic-linker (`PT_INTERP`) path for the current architecture. | +| [`./test_files/cpp/`](./test_files/cpp) | C++ sanitizer smoke test: sources + compile/run scripts. | +| [`./test_files/rust/`](./test_files/rust) | Rust smoke test: rustc sources + a cargo proc-macro workspace + compile/run scripts. | +| [`/bin/check-tools.sh`](../../bin/check-tools.sh) | Verify every expected tools are present and runnable. | +| [`/bin/install-sanitizer-libs.sh`](../../bin/install-sanitizer-libs.sh) | Install `libasan`/`libtsan`/`libubsan` runtimes on the supported base images. | diff --git a/nix/docker/test_files/rust/compile-sources.sh b/nix/docker/test_files/rust/compile-sources.sh index 5e5ecc0dcb..106855a3ab 100755 --- a/nix/docker/test_files/rust/compile-sources.sh +++ b/nix/docker/test_files/rust/compile-sources.sh @@ -40,6 +40,29 @@ compile hello compile panic compile overflow "-C overflow-checks=on" +function compile_proc_macro() { + local proj="${src_dir}/../proc_macro" + + echo "=== Building proc-macro workspace (cargo) ===" + cargo build --manifest-path "${proj}/Cargo.toml" --offline + + local built="${proj}/target/debug/test_macro" + if [ ! -f "${built}" ]; then + echo "ERROR: built test_macro binary not found at ${built}" >&2 + exit 1 + fi + + local binary="${dst_dir}/proc_macro" + cp "${built}" "${binary}" + + echo "=== Patching ${binary} to use ${loader} as PT_INTERP ===" + patchelf --set-interpreter "${loader}" --remove-rpath "${binary}" + + rm -rf "${proj}/target" +} + +compile_proc_macro + echo "=== All binaries compiled ===" ls -la "${dst_dir}" diff --git a/nix/docker/test_files/rust/proc_macro/Cargo.lock b/nix/docker/test_files/rust/proc_macro/Cargo.lock new file mode 100644 index 0000000000..acab3fa0b9 --- /dev/null +++ b/nix/docker/test_files/rust/proc_macro/Cargo.lock @@ -0,0 +1,14 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "echo_macro" +version = "0.0.0" + +[[package]] +name = "test_macro" +version = "0.0.0" +dependencies = [ + "echo_macro", +] diff --git a/nix/docker/test_files/rust/proc_macro/Cargo.toml b/nix/docker/test_files/rust/proc_macro/Cargo.toml new file mode 100644 index 0000000000..d54955de3d --- /dev/null +++ b/nix/docker/test_files/rust/proc_macro/Cargo.toml @@ -0,0 +1,3 @@ +[workspace] +resolver = "2" +members = ["echo_macro", "test_macro"] diff --git a/nix/docker/test_files/rust/proc_macro/echo_macro/Cargo.toml b/nix/docker/test_files/rust/proc_macro/echo_macro/Cargo.toml new file mode 100644 index 0000000000..b0f92ce75a --- /dev/null +++ b/nix/docker/test_files/rust/proc_macro/echo_macro/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "echo_macro" +version = "0.0.0" +edition = "2024" +publish = false + +[lib] +proc-macro = true diff --git a/nix/docker/test_files/rust/proc_macro/echo_macro/src/lib.rs b/nix/docker/test_files/rust/proc_macro/echo_macro/src/lib.rs new file mode 100644 index 0000000000..e4b90d58fe --- /dev/null +++ b/nix/docker/test_files/rust/proc_macro/echo_macro/src/lib.rs @@ -0,0 +1,6 @@ +use proc_macro::TokenStream; + +#[proc_macro] +pub fn define_echo(item: TokenStream) -> TokenStream { + format!("fn echo() -> u32 {{ {item} }}").parse().unwrap() +} diff --git a/nix/docker/test_files/rust/proc_macro/test_macro/Cargo.toml b/nix/docker/test_files/rust/proc_macro/test_macro/Cargo.toml new file mode 100644 index 0000000000..25b6ba8e45 --- /dev/null +++ b/nix/docker/test_files/rust/proc_macro/test_macro/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "test_macro" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +echo_macro = { path = "../echo_macro" } diff --git a/nix/docker/test_files/rust/proc_macro/test_macro/src/main.rs b/nix/docker/test_files/rust/proc_macro/test_macro/src/main.rs new file mode 100644 index 0000000000..77718b9d75 --- /dev/null +++ b/nix/docker/test_files/rust/proc_macro/test_macro/src/main.rs @@ -0,0 +1,9 @@ +use echo_macro::define_echo; + +define_echo!(42); + +fn main() { + let a = echo(); + println!("proc-macro answer = {a}"); + assert_eq!(a, 42, "proc-macro expansion produced the wrong value"); +} diff --git a/nix/docker/test_files/rust/run-binaries.sh b/nix/docker/test_files/rust/run-binaries.sh index b627c12609..4cafee00ec 100755 --- a/nix/docker/test_files/rust/run-binaries.sh +++ b/nix/docker/test_files/rust/run-binaries.sh @@ -1,7 +1,7 @@ #!/bin/bash # Run pre-compiled Rust binaries and confirm each emits its expected diagnostic. # Binaries must already exist in as for name in -# {hello,panic,overflow}. +# {hello,panic,overflow,proc_macro}. set -eo pipefail @@ -54,12 +54,13 @@ declare -A expect=( [hello]="Hello from main thread" [panic]="explicit panic from test" [overflow]="attempt to add with overflow" + [proc_macro]="proc-macro answer = 42" ) -for name in hello panic overflow; do +for name in hello panic overflow proc_macro; do binary="${bins_dir}/${name}" - if [ "${name}" = "hello" ]; then + if [ "${name}" = "hello" ] || [ "${name}" = "proc_macro" ]; then expected_rc=0 else expected_rc=nonzero diff --git a/nix/packages.nix b/nix/packages.nix index 41d7e97328..1d9b6cd2f8 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -15,6 +15,55 @@ let runClangTidy = pkgs.writeShellScriptBin "run-clang-tidy" '' exec ${pkgs.python3}/bin/python3 ${llvmPackages.clang-unwrapped}/bin/run-clang-tidy "$@" ''; + + rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ../rust-toolchain.toml; + + # Nix wraps its toolchain so that binaries are exposed only under unsuffixed + # names (gcc, g++, clang-tidy, ...). Several tools probe for a + # version-suffixed name first and fall back to a system binary on the PATH + # when Nix doesn't provide it: + # - Conan's Boost recipe looks up `g++-` before plain `g++`. + # - bin/pre-commit/clang_tidy_check.py looks up `run-clang-tidy-` and + # `clang-apply-replacements-` before the unsuffixed names. + # On a host that also has the matching system binary (e.g. Ubuntu's + # `/usr/bin/g++-15` or `clang-tidy-22`) the probe escapes Nix and mixes a + # system tool into the Nix environment. Generate version-suffixed symlinks + # next to a package's tools so those probes resolve to the Nix ones. + # + # Compiler links must point at whichever compiler is active in a given + # environment (the plain stdenv compiler in the dev shell, the custom-glibc + # wrappers in ci-env.nix), so those callers pass their own `package`; the + # clang tooling is environment-independent and is linked in commonPackages. + mkVersionedToolLinks = + { + name, + package, + version, + tools, + }: + pkgs.linkFarm "${name}-${toString version}-versioned-links" ( + map (tool: { + name = "bin/${tool}-${toString version}"; + path = "${package}/bin/${tool}"; + }) tools + ); + + clangToolLinks = mkVersionedToolLinks { + name = "clang-tools"; + package = clangTools; + version = llvmVersion; + tools = [ + "clang-tidy" + "clang-apply-replacements" + "clang-format" + ]; + }; + runClangTidyLink = mkVersionedToolLinks { + name = "run-clang-tidy"; + package = runClangTidy; + version = llvmVersion; + tools = [ "run-clang-tidy" ]; + }; in { inherit @@ -22,9 +71,12 @@ in llvmVersion gccPackage llvmPackages + mkVersionedToolLinks ; commonPackages = with pkgs; [ + clangToolLinks + runClangTidyLink ccache clangbuildanalyzer clangTools @@ -63,14 +115,10 @@ in vim zip # Rust packages - cargo cargo-audit cargo-llvm-cov cargo-nextest - clippy corrosion - rust-analyzer - rustc - rustfmt + rustToolchain ]; } diff --git a/nix/utils.nix b/nix/utils.nix index d83e612c16..0b70183ef3 100644 --- a/nix/utils.nix +++ b/nix/utils.nix @@ -1,4 +1,8 @@ -{ nixpkgs, nixpkgs-custom-glibc }: +{ + nixpkgs, + nixpkgs-custom-glibc, + rust-overlay, +}: function: nixpkgs.lib.genAttrs [ @@ -10,7 +14,12 @@ nixpkgs.lib.genAttrs ( system: function { - pkgs = import nixpkgs { inherit system; }; + # rust-overlay adds `pkgs.rust-bin`, from which we build the pinned Rust + # toolchain (see packages.nix). Consumed by both the CI image and dev shell. + pkgs = import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + }; # glibc 2.31 — matches the system libc on Ubuntu 20.04 LTS. Sourced # from the nixpkgs snapshot pinned via the `nixpkgs-custom-glibc` # flake input, so the build uses the compiler from that snapshot diff --git a/rust-toolchain.toml b/rust-toolchain.toml index dbc9e74c5d..3206cf59c0 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,8 +1,4 @@ -# Rust toolchain pin for rustup-based CI runners and local development. -# rustup reads this file and installs the pinned toolchain (see the -# prepare-runner action in XRPLF/actions, which runs `rustup toolchain install`). -# NOTE: the Nix CI image and development shell ignore this file; its rustc comes from flake.lock. [toolchain] channel = "1.95" -components = ["rustfmt", "clippy"] +components = ["rustfmt", "clippy", "rust-analyzer"] profile = "minimal"