refactor: Group binaries in subdirectories (#8535)

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Ayaz Salikhov
2026-10-08 12:34:06 +00:00
committed by GitHub
parent 3c24b605a1
commit ede8af8191
22 changed files with 50 additions and 51 deletions

View File

@@ -52,4 +52,4 @@ runs:
- name: Check build-context packages for Nix store dependencies (Linux)
if: ${{ runner.os == 'Linux' }}
shell: bash
run: ./bin/check-build-context-runtime.sh "${RUNNER_TEMP}/conan-graph.json"
run: ./bin/nix/check-build-context-runtime.sh "${RUNNER_TEMP}/conan-graph.json"

View File

@@ -12,7 +12,7 @@ on:
- "!nix/docker/README.md"
- "!nix/devshell.nix"
- "!nix/check-tools/**"
- "bin/default-loader-path.sh"
- "bin/nix/default-loader-path.sh"
pull_request:
paths:
- ".github/workflows/build-nix-images.yml"
@@ -24,8 +24,8 @@ on:
- "!nix/devshell.nix"
- "!nix/check-tools/**"
- "bin/check-tools.sh"
- "bin/default-loader-path.sh"
- "bin/install-sanitizer-libs.sh"
- "bin/nix/default-loader-path.sh"
- "bin/install/sanitizer-libs.sh"
workflow_dispatch:
concurrency:

View File

@@ -5,12 +5,12 @@ on:
branches:
- develop
paths:
- "bin/install-packaging-tools.sh"
- "bin/install/packaging-tools.sh"
- "package/images/packaging/**"
pull_request:
paths:
- ".github/workflows/build-packaging-images.yml"
- "bin/install-packaging-tools.sh"
- "bin/install/packaging-tools.sh"
- "package/images/packaging/**"
workflow_dispatch:

View File

@@ -93,9 +93,8 @@ jobs:
.github/workflows/reusable-upload-recipe.yml
.clang-tidy
.codecov.yml
bin/check-nix-store-refs.sh
bin/check-tools.sh
bin/default-loader-path.sh
bin/nix/**
cfg/**
cmake/**
conan/**

View File

@@ -31,9 +31,8 @@ on:
- ".github/workflows/reusable-upload-recipe.yml"
- ".clang-tidy"
- ".codecov.yml"
- "bin/check-nix-store-refs.sh"
- "bin/check-tools.sh"
- "bin/default-loader-path.sh"
- "bin/nix/**"
- "cfg/**"
- "cmake/**"
- "conan/**"

View File

@@ -257,20 +257,20 @@ jobs:
# cache included, since what it holds is what gets uploaded and reused.
- name: Check the build output for Nix store references (Nix toolchain)
if: ${{ inputs.toolchain == 'nix' }}
run: ./bin/check-nix-store-refs.sh "${BUILD_DIR}"
run: ./bin/nix/check-nix-store-refs.sh "${BUILD_DIR}"
- name: Check the Conan cache for Nix store references (Nix toolchain)
if: ${{ inputs.toolchain == 'nix' }}
run: ./bin/check-nix-store-refs.sh "${CONAN_HOME}"
run: ./bin/nix/check-nix-store-refs.sh "${CONAN_HOME}"
# Only what PatchNixBinary.cmake retargets: the toolchain in the Linux
# images always references the store. Same condition it uses.
- name: Check for Nix store references (Linux)
if: ${{ runner.os == 'Linux' && env.SANITIZERS_ENABLED == 'false' }}
run: |
./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpld"
./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpl_tests"
./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpld_tests"
./bin/nix/check-nix-store-refs.sh "${BUILD_DIR}/xrpld"
./bin/nix/check-nix-store-refs.sh "${BUILD_DIR}/xrpl_tests"
./bin/nix/check-nix-store-refs.sh "${BUILD_DIR}/xrpld_tests"
- name: Show ccache statistics
if: ${{ inputs.ccache_enabled }}

View File

@@ -115,7 +115,7 @@ jobs:
- name: Check the Conan cache for Nix store references (Nix toolchain)
if: ${{ matrix.toolchain == 'nix' }}
run: ./bin/check-nix-store-refs.sh "${CONAN_HOME}"
run: ./bin/nix/check-nix-store-refs.sh "${CONAN_HOME}"
- name: Log into Conan remote
if: ${{ github.repository == 'XRPLF/rippled' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }}

View File

@@ -11,7 +11,7 @@
# update moves the GCC runtime to a new store path, so a cached binary has to
# get by with the pinned glibc alone. See docs/build/nix.md.
#
# Usage: bin/check-build-context-runtime.sh <graph.json>
# Usage: bin/nix/check-build-context-runtime.sh <graph.json>
# <graph.json> is the output of `conan install --format=json`.
set -euo pipefail

View File

@@ -10,7 +10,7 @@
# alone; the scripts in a Conan cache are all git hook samples and autotools
# scratch, 36 false positives to 0 real.
#
# Usage: bin/check-nix-store-refs.sh <path>
# Usage: bin/nix/check-nix-store-refs.sh <path>
set -euo pipefail

View File

@@ -17,7 +17,7 @@
(runtime libraries resolved through the rpath) are skipped too.
Everywhere else `patch_nix_binary` is a no-op.
The default loader is resolved by bin/default-loader-path.sh.
The default loader is resolved by bin/nix/default-loader-path.sh.
#]===================================================================]
include_guard(GLOBAL)
@@ -25,7 +25,7 @@ include_guard(GLOBAL)
include(CompilationEnv)
# Resolves the system default ELF loader path for the current architecture.
set(_loader_path_script "${CMAKE_SOURCE_DIR}/bin/default-loader-path.sh")
set(_loader_path_script "${CMAKE_SOURCE_DIR}/bin/nix/default-loader-path.sh")
if(
is_linux

View File

@@ -154,7 +154,7 @@ class Xrpl(ConanFile):
self.requires("xxhash/0.8.3", transitive_headers=True)
exports_sources = (
"bin/default-loader-path.sh",
"bin/nix/default-loader-path.sh",
"CMakeLists.txt",
"cfg/*",
"cmake/*",

12
docs/build/nix.md vendored
View File

@@ -183,14 +183,14 @@ at link or run time.
> configuration CI covers, and no dependency binaries are published for it.
This is checked rather than assumed.
[`bin/check-nix-store-refs.sh`](../../bin/check-nix-store-refs.sh) takes one file
or directory and fails if a binary under it resolves a store path at run time.
[`bin/nix/check-nix-store-refs.sh`](../../bin/nix/check-nix-store-refs.sh) takes one
file or directory and fails if a binary under it resolves a store path at run time.
CI runs it over the build output and the Conan cache, and again in the upload job
before anything is published. You can run it yourself:
```bash
bin/check-nix-store-refs.sh build
bin/check-nix-store-refs.sh ~/.conan2-nix
bin/nix/check-nix-store-refs.sh build
bin/nix/check-nix-store-refs.sh ~/.conan2-nix
```
It works on Linux too, but asserts something narrower there: the toolchain always
@@ -204,7 +204,7 @@ instrument them. CI checks that they load nothing from the store but glibc, from
the graph `conan install --format=json` writes:
```bash
bin/check-build-context-runtime.sh graph.json
bin/nix/check-build-context-runtime.sh graph.json
```
Only the binaries [`PatchNixBinary.cmake`](../../cmake/PatchNixBinary.cmake)
@@ -212,7 +212,7 @@ retargets to the system loader have to be fully clean, and those are what CI
checks:
```bash
bin/check-nix-store-refs.sh build/xrpld
bin/nix/check-nix-store-refs.sh build/xrpld
```
### The libresolv stub

View File

@@ -178,11 +178,11 @@ A binary stops starting after a `nix flake update`, or after
dyld[57271]: Library not loaded: /nix/store/…-libresolv-93/lib/libresolv.9.dylib
```
[`bin/check-nix-store-refs.sh`](../../bin/check-nix-store-refs.sh) finds the same
thing without having to run anything, and names the file:
[`bin/nix/check-nix-store-refs.sh`](../../bin/nix/check-nix-store-refs.sh) finds the
same thing without having to run anything, and names the file:
```
$ bin/check-nix-store-refs.sh ~/.conan2-nix
$ bin/nix/check-nix-store-refs.sh ~/.conan2-nix
::error file=/Users/you/.conan2-nix/p/b/c-area24ded30c388c/p/bin/adig::references the Nix store at run time
/Users/you/.conan2-nix/p/b/c-area24ded30c388c/p/bin/adig
/nix/store/p4lp3xq4imd1qzqh08x8vcq2zfhi7rca-libresolv-93/lib/libresolv.9.dylib

View File

@@ -60,10 +60,10 @@ ENV GIT_SSL_CAINFO="/nix/ci-env/etc/ssl/certs/ca-bundle.crt"
# Externally-built dynamically-linked ELF binaries hard-code the loader path
# (e.g. /lib64/ld-linux-x86-64.so.2) in their PT_INTERP header. Install it
# from the Nix store when the base image doesn't already provide one.
COPY bin/default-loader-path.sh /tmp/loader-path.sh
COPY bin/nix/default-loader-path.sh /usr/local/bin/default-loader-path.sh
RUN <<EOF
target="$(/tmp/loader-path.sh)"
target="$(/usr/local/bin/default-loader-path.sh)"
if [ ! -e "${target}" ]; then
# Use the loader from the same glibc that gcc links libc against, so
@@ -101,7 +101,7 @@ RUN if echo "${BASE_IMAGE}" | grep -qiE 'nixos'; then \
SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]
# Sanity-check that the built binaries run correctly in the vanilla base image, with the necessary sanitizer runtime libraries installed.
COPY bin/install-sanitizer-libs.sh /tmp/install-sanitizer-libs.sh
COPY bin/install/sanitizer-libs.sh /tmp/install-sanitizer-libs.sh
COPY nix/docker/test_files/cpp/run-binaries.sh /tmp/test_files/cpp/run-binaries.sh
COPY nix/docker/test_files/rust/run-binaries.sh /tmp/test_files/rust/run-binaries.sh
COPY --from=final /tmp/cpp-bins /tmp/cpp-bins

View File

@@ -52,10 +52,10 @@ work without `ca-certificates` being installed in the base image.
workspace with `cargo` to exercise proc-macro dylib loading.
3. **`tester`** — Start again from a clean `BASE_IMAGE` (no Nix toolchain),
install only the sanitizer runtime libraries
([`install-sanitizer-libs.sh`](./install-sanitizer-libs.sh)), and run the
binaries compiled in `final`. This proves the binaries built with the Nix
toolchain actually run on a vanilla base image. On `nixos/nix` this step is
skipped (the binaries are patched for a conventional FHS loader).
([`bin/install/sanitizer-libs.sh`](../../bin/install/sanitizer-libs.sh)),
and run the binaries compiled in `final`. This proves the binaries built with
the Nix toolchain actually run on a vanilla base image. On `nixos/nix` this
step is skipped (the binaries are patched for a conventional FHS loader).
4. **Output** — The final image is gated on the tester succeeding: it copies a
sentinel file out of `tester`, so a failed test run fails the whole build.
@@ -75,9 +75,10 @@ toolchain being present at runtime. Two pieces make that work:
- **An expected dynamic linker in the image.**
Binaries built in Nix environments reference a dynamic linker from Nix store paths, which won't be present in the base image. However,
[`bin/default-loader-path.sh`](../../bin/default-loader-path.sh) reports the
[`bin/nix/default-loader-path.sh`](../../bin/nix/default-loader-path.sh) reports the
expected loader path for the current architecture, so we can patch the binaries
to use the correct loader.
to use the correct loader. The image ships it as
`/usr/local/bin/default-loader-path.sh`.
The build then verifies all of this end to end, and the C++ and Rust programs
go through the same pipeline: each is compiled in `final`, has its `PT_INTERP`
@@ -91,11 +92,11 @@ whose resulting binary is patched and run like the others.
## Files
| File | Purpose |
| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
| [`./Dockerfile`](./Dockerfile) | Multi-stage build described above. |
| [`./test_files/cpp/`](./test_files/cpp) | C++ sanitizer smoke test: sources + compile/run scripts. |
| [`./test_files/rust/`](./test_files/rust) | Rust smoke test: rustc sources + a cargo proc-macro workspace + compile/run scripts. |
| [`/bin/check-tools.sh`](../../bin/check-tools.sh) | Verify every expected tools are present and runnable. |
| [`/bin/default-loader-path.sh`](../../bin/default-loader-path.sh) | Print the dynamic-linker (`PT_INTERP`) path for the current architecture. |
| [`/bin/install-sanitizer-libs.sh`](../../bin/install-sanitizer-libs.sh) | Install `libasan`/`libtsan`/`libubsan` runtimes on the supported base images. |
| File | Purpose |
| ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
| [`./Dockerfile`](./Dockerfile) | Multi-stage build described above. |
| [`./test_files/cpp/`](./test_files/cpp) | C++ sanitizer smoke test: sources + compile/run scripts. |
| [`./test_files/rust/`](./test_files/rust) | Rust smoke test: rustc sources + a cargo proc-macro workspace + compile/run scripts. |
| [`/bin/check-tools.sh`](../../bin/check-tools.sh) | Verify every expected tools are present and runnable. |
| [`/bin/nix/default-loader-path.sh`](../../bin/nix/default-loader-path.sh) | Print the dynamic-linker (`PT_INTERP`) path for the current architecture. |
| [`/bin/install/sanitizer-libs.sh`](../../bin/install/sanitizer-libs.sh) | Install `libasan`/`libtsan`/`libubsan` runtimes on the supported base images. |

View File

@@ -8,7 +8,7 @@ set -eo pipefail
src_dir="${1:?usage: $0 <src_dir> <dst_dir>}"
dst_dir="${2:?usage: $0 <src_dir> <dst_dir>}"
loader="$(/tmp/loader-path.sh)"
loader="$(/usr/local/bin/default-loader-path.sh)"
mkdir -p "${dst_dir}"

View File

@@ -8,7 +8,7 @@ set -eo pipefail
src_dir="${1:?usage: $0 <src_dir> <dst_dir>}"
dst_dir="${2:?usage: $0 <src_dir> <dst_dir>}"
loader="$(/tmp/loader-path.sh)"
loader="$(/usr/local/bin/default-loader-path.sh)"
mkdir -p "${dst_dir}"

View File

@@ -12,7 +12,7 @@ package/
sign_rpm.py Signs the built RPMs (called by CI when publishing)
images/
packaging/
Dockerfile Packaging image, built by `build-packaging-images.yml`; installs its tooling with `bin/install-packaging-tools.sh`
Dockerfile Packaging image, built by `build-packaging-images.yml`; installs its tooling with `bin/install/packaging-tools.sh`
publish_pkg.py Uploads built packages to the XRPLF Nexus repositories (called by CI, and shipped in that image)
xrpld/
Dockerfile The xrpld Docker images, installing the built DEB on Ubuntu (see "Docker images")

View File

@@ -2,7 +2,7 @@ ARG BASE_IMAGE=debian:trixie
FROM ${BASE_IMAGE}
RUN --mount=type=bind,source=bin/install-packaging-tools.sh,target=/install-packaging-tools.sh \
RUN --mount=type=bind,source=bin/install/packaging-tools.sh,target=/install-packaging-tools.sh \
/install-packaging-tools.sh
# See package/README.md, "Publishing from other repositories".