Merge branch 'develop' into tapanito/lending-cash-basis

.cspell.config.yaml

@@ -216,6 +216,7 @@ words:
   - Nyffenegger
   - onlatest
   - ostr
+  - oxalica
   - pargs
   - partitioner
   - paychan
@@ -340,6 +341,7 @@ words:
   - unsquelch
   - unsquelched
   - unsquelching
+  - unsuffixed
   - unvalidated
   - unveto
   - unvetoed

.github/pull_request_template.md

@@ -1,10 +1,10 @@
 <!--
-This PR template helps you to write a good pull request description.
+This PR template helps you write a good pull request description.
 Please feel free to include additional useful information even beyond what is requested below.
 
 If your branch is on a personal fork and has a name that allows it to
 run CI build/test jobs (e.g. "ci/foo"), remember to rename it BEFORE
-opening the PR.  This avoids unnecessary redundant test runs. Renaming
+opening the PR. This avoids redundant test runs. Renaming
 the branch after opening the PR will close the PR.
 https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-branches-in-your-repository/renaming-a-branch
 -->
@@ -15,7 +15,7 @@ https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-
 Please include a summary of the changes.
 This may be a direct input to the release notes.
 If too broad, please consider splitting into multiple PRs.
-If a relevant task or issue, please link it here.
+If there is a relevant task or issue, please link it here.
 -->
 
 ### Context of Change
@@ -65,5 +65,5 @@ This section may not be needed if your change includes thoroughly commented unit
 
 <!--
 ## Future Tasks
-For future tasks related to PR.
+For future tasks related to this PR.
 -->

.github/workflows/build-nix-images.yml

@@ -8,6 +8,7 @@ on:
       - ".github/workflows/build-nix-images.yml"
       - "flake.nix"
       - "flake.lock"
+      - "rust-toolchain.toml"
       - "nix/**"
       - "!nix/docker/README.md"
       - "!nix/devshell.nix"
@@ -18,6 +19,7 @@ on:
       - ".github/workflows/build-nix-images.yml"
       - "flake.nix"
       - "flake.lock"
+      - "rust-toolchain.toml"
       - "nix/**"
       - "!nix/docker/README.md"
       - "!nix/devshell.nix"

.gitignore

@@ -81,6 +81,9 @@ DerivedData
 # Python
 __pycache__
 
+# Rust build artifacts.
+target/
+
 # Direnv's directory
 /.direnv
 

flake.lock

@@ -36,7 +36,28 @@
     "root": {
       "inputs": {
         "nixpkgs": "nixpkgs",
-        "nixpkgs-custom-glibc": "nixpkgs-custom-glibc"
+        "nixpkgs-custom-glibc": "nixpkgs-custom-glibc",
+        "rust-overlay": "rust-overlay"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1784611586,
+        "narHash": "sha256-OfqgY+0hp/zseZB7uyH0U8kIDPS4scZZCyAurEplvG0=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "14f58845249f3552a89b07772626b8d3c632fa86",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
       }
     }
   },

flake.nix

@@ -10,12 +10,25 @@
       url = "github:NixOS/nixpkgs/9cd98386a38891d1074fc18036b842dc4416f562";
       flake = false;
     };
+    # Pinned Rust toolchains, delivered from the Nix store. Lets the Nix CI
+    # image and dev shell honour the single `rust-toolchain.toml` pin (shared
+    # with the rustup-based non-Nix runners) while staying hermetic — the
+    # toolchain lands in the image's Nix closure and is locked by flake.lock.
+    rust-overlay = {
+      url = "github:oxalica/rust-overlay";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs =
-    { nixpkgs, nixpkgs-custom-glibc, ... }:
+    {
+      nixpkgs,
+      nixpkgs-custom-glibc,
+      rust-overlay,
+      ...
+    }:
     let
-      forEachSystem = import ./nix/utils.nix { inherit nixpkgs nixpkgs-custom-glibc; };
+      forEachSystem = import ./nix/utils.nix { inherit nixpkgs nixpkgs-custom-glibc rust-overlay; };
     in
     {
       devShells = forEachSystem (import ./nix/devshell.nix);

nix/ci-env.nix

@@ -7,8 +7,10 @@ let
   inherit (import ./packages.nix { inherit pkgs; })
     commonPackages
     gccPackage
+    gccVersion
     llvmPackages
     llvmVersion
+    mkVersionedToolLinks
     ;
 
   # Underlying compiler toolchains to wrap (versions pinned in packages.nix).
@@ -127,6 +129,25 @@ in
       customGcov
       customClangForCiEnv
       customBinutils
+      (mkVersionedToolLinks {
+        name = "gcc";
+        package = customGcc;
+        version = gccVersion;
+        tools = [
+          "gcc"
+          "g++"
+          "cpp"
+        ];
+      })
+      (mkVersionedToolLinks {
+        name = "clang";
+        package = customClang;
+        version = llvmVersion;
+        tools = [
+          "clang"
+          "clang++"
+        ];
+      })
       # CA certificate bundle so HTTPS clients (git, curl, conan) can verify
       # TLS connections without ca-certificates being installed in the system.
       pkgs.cacert

nix/devshell.nix

@@ -3,7 +3,9 @@ let
   inherit (import ./packages.nix { inherit pkgs; })
     commonPackages
     gccVersion
+    llvmVersion
     llvmPackages
+    mkVersionedToolLinks
     ;
 
   # Plain nixpkgs stdenvs — no custom glibc, unlike ci-env.nix.
@@ -15,6 +17,8 @@ let
     {
       stdenv,
       compilerName,
+      version ? null,
+      versionedTools ? [ ],
     }:
     let
       compilerVersion =
@@ -25,9 +29,15 @@ let
             echo "Compiler: "
             ${compilerName} --version
           '';
+      versionedLinks = pkgs.lib.optional (version != null) (mkVersionedToolLinks {
+        name = compilerName;
+        package = stdenv.cc;
+        inherit version;
+        tools = versionedTools;
+      });
     in
     (pkgs.mkShell.override { inherit stdenv; }) {
-      packages = commonPackages;
+      packages = commonPackages ++ versionedLinks;
       shellHook = ''
         echo "Welcome to xrpld development shell";
         ${compilerVersion}
@@ -41,11 +51,22 @@ rec {
   gcc = makeShell {
     stdenv = gccStdenv;
     compilerName = "gcc";
+    version = gccVersion;
+    versionedTools = [
+      "gcc"
+      "g++"
+      "cpp"
+    ];
   };
 
   clang = makeShell {
     stdenv = clangStdenv;
     compilerName = "clang";
+    version = llvmVersion;
+    versionedTools = [
+      "clang"
+      "clang++"
+    ];
   };
 
   # Nix provides no compiler; use the one from your system (e.g. Apple Clang).

nix/docker/Dockerfile

@@ -12,6 +12,7 @@ COPY nix/packages.nix /tmp/build/nix/packages.nix
 COPY nix/utils.nix /tmp/build/nix/utils.nix
 COPY flake.nix /tmp/build/
 COPY flake.lock /tmp/build/
+COPY rust-toolchain.toml /tmp/build/
 WORKDIR /tmp/build
 
 FROM builder-source AS builder

nix/docker/README.md

@@ -47,7 +47,9 @@ work without `ca-certificates` being installed in the base image.
      [`test_files/cpp/sources/`](./test_files/cpp/sources) with both `g++` and
      `clang++`, and sanitizers, and
    - compiles the Rust test programs in
-     [`test_files/rust/sources/`](./test_files/rust/sources) with `rustc`.
+     [`test_files/rust/sources/`](./test_files/rust/sources) with `rustc`, and
+     builds the [`test_files/rust/proc_macro/`](./test_files/rust/proc_macro)
+     workspace with `cargo` to exercise proc-macro dylib loading.
 3. **`tester`** — Start again from a clean `BASE_IMAGE` (no Nix toolchain),
    install only the sanitizer runtime libraries
    ([`install-sanitizer-libs.sh`](./install-sanitizer-libs.sh)), and run the
@@ -76,20 +78,23 @@ toolchain being present at runtime. Two pieces make that work:
   [`loader-path.sh`](./loader-path.sh) reports the expected loader path for the
   current architecture, so we can patch the binaries to use the correct loader.
 
-The build then verifies all of this end to end: the C++ test programs in
-`test_files/cpp/sources/` (a regular binary plus ASan/TSan/UBSan variants) and
-the Rust test programs in `test_files/rust/sources/` (a hello binary plus panic
-and overflow-check variants) are compiled in `final`, their `PT_INTERP` is
-patched to the target loader, and they are run in the clean `tester` stage to
-confirm each emits the expected diagnostic on a stock base image.
+The build then verifies all of this end to end, and the C++ and Rust programs
+go through the same pipeline: each is compiled in `final`, has its `PT_INTERP`
+patched to the target loader, and is then run in the clean `tester` stage to
+confirm it emits the expected diagnostic on a stock base image. The C++ programs
+are in `test_files/cpp/sources/` (a regular binary plus ASan/TSan/UBSan
+variants); the Rust programs are in `test_files/rust/sources/` (a hello binary
+plus panic and overflow-check variants), plus the `test_files/rust/proc_macro/`
+workspace — a crate whose compilation additionally loads a proc-macro dylib, and
+whose resulting binary is patched and run like the others.
 
 ## Files
 
-| File                                                                    | Purpose                                                                       |
-| ----------------------------------------------------------------------- | ----------------------------------------------------------------------------- |
-| [`./Dockerfile`](./Dockerfile)                                          | Multi-stage build described above.                                            |
-| [`./loader-path.sh`](./loader-path.sh)                                  | Print the dynamic-linker (`PT_INTERP`) path for the current architecture.     |
-| [`./test_files/cpp/`](./test_files/cpp)                                 | C++ sanitizer smoke test: sources + compile/run scripts.                      |
-| [`./test_files/rust/`](./test_files/rust)                               | Rust rustc smoke test: sources + compile/run scripts.                         |
-| [`/bin/check-tools.sh`](../../bin/check-tools.sh)                       | Verify every expected tools are present and runnable.                         |
-| [`/bin/install-sanitizer-libs.sh`](../../bin/install-sanitizer-libs.sh) | Install `libasan`/`libtsan`/`libubsan` runtimes on the supported base images. |
+| File                                                                    | Purpose                                                                              |
+| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
+| [`./Dockerfile`](./Dockerfile)                                          | Multi-stage build described above.                                                   |
+| [`./loader-path.sh`](./loader-path.sh)                                  | Print the dynamic-linker (`PT_INTERP`) path for the current architecture.            |
+| [`./test_files/cpp/`](./test_files/cpp)                                 | C++ sanitizer smoke test: sources + compile/run scripts.                             |
+| [`./test_files/rust/`](./test_files/rust)                               | Rust smoke test: rustc sources + a cargo proc-macro workspace + compile/run scripts. |
+| [`/bin/check-tools.sh`](../../bin/check-tools.sh)                       | Verify every expected tools are present and runnable.                                |
+| [`/bin/install-sanitizer-libs.sh`](../../bin/install-sanitizer-libs.sh) | Install `libasan`/`libtsan`/`libubsan` runtimes on the supported base images.        |

nix/docker/test_files/rust/compile-sources.sh

@@ -40,6 +40,29 @@ compile hello
 compile panic
 compile overflow "-C overflow-checks=on"
 
+function compile_proc_macro() {
+    local proj="${src_dir}/../proc_macro"
+
+    echo "=== Building proc-macro workspace (cargo) ==="
+    cargo build --manifest-path "${proj}/Cargo.toml" --offline
+
+    local built="${proj}/target/debug/test_macro"
+    if [ ! -f "${built}" ]; then
+        echo "ERROR: built test_macro binary not found at ${built}" >&2
+        exit 1
+    fi
+
+    local binary="${dst_dir}/proc_macro"
+    cp "${built}" "${binary}"
+
+    echo "=== Patching ${binary} to use ${loader} as PT_INTERP ==="
+    patchelf --set-interpreter "${loader}" --remove-rpath "${binary}"
+
+    rm -rf "${proj}/target"
+}
+
+compile_proc_macro
+
 echo "=== All binaries compiled ==="
 
 ls -la "${dst_dir}"

nix/docker/test_files/rust/proc_macro/Cargo.lock

@@ -0,0 +1,14 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "echo_macro"
+version = "0.0.0"
+
+[[package]]
+name = "test_macro"
+version = "0.0.0"
+dependencies = [
+ "echo_macro",
+]

nix/docker/test_files/rust/proc_macro/Cargo.toml

@@ -0,0 +1,3 @@
+[workspace]
+resolver = "2"
+members = ["echo_macro", "test_macro"]

nix/docker/test_files/rust/proc_macro/echo_macro/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "echo_macro"
+version = "0.0.0"
+edition = "2024"
+publish = false
+
+[lib]
+proc-macro = true

nix/docker/test_files/rust/proc_macro/echo_macro/src/lib.rs

@@ -0,0 +1,6 @@
+use proc_macro::TokenStream;
+
+#[proc_macro]
+pub fn define_echo(item: TokenStream) -> TokenStream {
+    format!("fn echo() -> u32 {{ {item} }}").parse().unwrap()
+}

nix/docker/test_files/rust/proc_macro/test_macro/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "test_macro"
+version = "0.0.0"
+edition = "2024"
+publish = false
+
+[dependencies]
+echo_macro = { path = "../echo_macro" }

nix/docker/test_files/rust/proc_macro/test_macro/src/main.rs

@@ -0,0 +1,9 @@
+use echo_macro::define_echo;
+
+define_echo!(42);
+
+fn main() {
+    let a = echo();
+    println!("proc-macro answer = {a}");
+    assert_eq!(a, 42, "proc-macro expansion produced the wrong value");
+}

nix/docker/test_files/rust/run-binaries.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Run pre-compiled Rust binaries and confirm each emits its expected diagnostic.
 # Binaries must already exist in <bins_dir> as <name> for name in
-# {hello,panic,overflow}.
+# {hello,panic,overflow,proc_macro}.
 
 set -eo pipefail
 
@@ -54,12 +54,13 @@ declare -A expect=(
     [hello]="Hello from main thread"
     [panic]="explicit panic from test"
     [overflow]="attempt to add with overflow"
+    [proc_macro]="proc-macro answer = 42"
 )
 
-for name in hello panic overflow; do
+for name in hello panic overflow proc_macro; do
     binary="${bins_dir}/${name}"
 
-    if [ "${name}" = "hello" ]; then
+    if [ "${name}" = "hello" ] || [ "${name}" = "proc_macro" ]; then
         expected_rc=0
     else
         expected_rc=nonzero

nix/packages.nix

@@ -15,6 +15,55 @@ let
   runClangTidy = pkgs.writeShellScriptBin "run-clang-tidy" ''
     exec ${pkgs.python3}/bin/python3 ${llvmPackages.clang-unwrapped}/bin/run-clang-tidy "$@"
   '';
+
+  rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ../rust-toolchain.toml;
+
+  # Nix wraps its toolchain so that binaries are exposed only under unsuffixed
+  # names (gcc, g++, clang-tidy, ...). Several tools probe for a
+  # version-suffixed name first and fall back to a system binary on the PATH
+  # when Nix doesn't provide it:
+  #   - Conan's Boost recipe looks up `g++-<major>` before plain `g++`.
+  #   - bin/pre-commit/clang_tidy_check.py looks up `run-clang-tidy-<v>` and
+  #     `clang-apply-replacements-<v>` before the unsuffixed names.
+  # On a host that also has the matching system binary (e.g. Ubuntu's
+  # `/usr/bin/g++-15` or `clang-tidy-22`) the probe escapes Nix and mixes a
+  # system tool into the Nix environment. Generate version-suffixed symlinks
+  # next to a package's tools so those probes resolve to the Nix ones.
+  #
+  # Compiler links must point at whichever compiler is active in a given
+  # environment (the plain stdenv compiler in the dev shell, the custom-glibc
+  # wrappers in ci-env.nix), so those callers pass their own `package`; the
+  # clang tooling is environment-independent and is linked in commonPackages.
+  mkVersionedToolLinks =
+    {
+      name,
+      package,
+      version,
+      tools,
+    }:
+    pkgs.linkFarm "${name}-${toString version}-versioned-links" (
+      map (tool: {
+        name = "bin/${tool}-${toString version}";
+        path = "${package}/bin/${tool}";
+      }) tools
+    );
+
+  clangToolLinks = mkVersionedToolLinks {
+    name = "clang-tools";
+    package = clangTools;
+    version = llvmVersion;
+    tools = [
+      "clang-tidy"
+      "clang-apply-replacements"
+      "clang-format"
+    ];
+  };
+  runClangTidyLink = mkVersionedToolLinks {
+    name = "run-clang-tidy";
+    package = runClangTidy;
+    version = llvmVersion;
+    tools = [ "run-clang-tidy" ];
+  };
 in
 {
   inherit
@@ -22,9 +71,12 @@ in
     llvmVersion
     gccPackage
     llvmPackages
+    mkVersionedToolLinks
     ;
 
   commonPackages = with pkgs; [
+    clangToolLinks
+    runClangTidyLink
     ccache
     clangbuildanalyzer
     clangTools
@@ -63,14 +115,10 @@ in
     vim
     zip
     # Rust packages
-    cargo
     cargo-audit
     cargo-llvm-cov
     cargo-nextest
-    clippy
     corrosion
-    rust-analyzer
-    rustc
-    rustfmt
+    rustToolchain
   ];
 }

nix/utils.nix

@@ -1,4 +1,8 @@
-{ nixpkgs, nixpkgs-custom-glibc }:
+{
+  nixpkgs,
+  nixpkgs-custom-glibc,
+  rust-overlay,
+}:
 function:
 nixpkgs.lib.genAttrs
   [
@@ -10,7 +14,12 @@ nixpkgs.lib.genAttrs
   (
     system:
     function {
-      pkgs = import nixpkgs { inherit system; };
+      # rust-overlay adds `pkgs.rust-bin`, from which we build the pinned Rust
+      # toolchain (see packages.nix). Consumed by both the CI image and dev shell.
+      pkgs = import nixpkgs {
+        inherit system;
+        overlays = [ (import rust-overlay) ];
+      };
       # glibc 2.31 — matches the system libc on Ubuntu 20.04 LTS. Sourced
       # from the nixpkgs snapshot pinned via the `nixpkgs-custom-glibc`
       # flake input, so the build uses the compiler from that snapshot

rust-toolchain.toml

@@ -1,8 +1,4 @@
-# Rust toolchain pin for rustup-based CI runners and local development.
-# rustup reads this file and installs the pinned toolchain (see the
-# prepare-runner action in XRPLF/actions, which runs `rustup toolchain install`).
-# NOTE: the Nix CI image and development shell ignore this file; its rustc comes from flake.lock.
 [toolchain]
 channel = "1.95"
-components = ["rustfmt", "clippy"]
+components = ["rustfmt", "clippy", "rust-analyzer"]
 profile = "minimal"