mirror of
https://github.com/XRPLF/rippled.git
synced 2026-10-09 13:18:09 +00:00
refactor: Group binaries in subdirectories (#8535)
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
2
.github/actions/build-deps/action.yml
vendored
2
.github/actions/build-deps/action.yml
vendored
@@ -52,4 +52,4 @@ runs:
|
||||
- name: Check build-context packages for Nix store dependencies (Linux)
|
||||
if: ${{ runner.os == 'Linux' }}
|
||||
shell: bash
|
||||
run: ./bin/check-build-context-runtime.sh "${RUNNER_TEMP}/conan-graph.json"
|
||||
run: ./bin/nix/check-build-context-runtime.sh "${RUNNER_TEMP}/conan-graph.json"
|
||||
|
||||
6
.github/workflows/build-nix-images.yml
vendored
6
.github/workflows/build-nix-images.yml
vendored
@@ -12,7 +12,7 @@ on:
|
||||
- "!nix/docker/README.md"
|
||||
- "!nix/devshell.nix"
|
||||
- "!nix/check-tools/**"
|
||||
- "bin/default-loader-path.sh"
|
||||
- "bin/nix/default-loader-path.sh"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/build-nix-images.yml"
|
||||
@@ -24,8 +24,8 @@ on:
|
||||
- "!nix/devshell.nix"
|
||||
- "!nix/check-tools/**"
|
||||
- "bin/check-tools.sh"
|
||||
- "bin/default-loader-path.sh"
|
||||
- "bin/install-sanitizer-libs.sh"
|
||||
- "bin/nix/default-loader-path.sh"
|
||||
- "bin/install/sanitizer-libs.sh"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
|
||||
4
.github/workflows/build-packaging-images.yml
vendored
4
.github/workflows/build-packaging-images.yml
vendored
@@ -5,12 +5,12 @@ on:
|
||||
branches:
|
||||
- develop
|
||||
paths:
|
||||
- "bin/install-packaging-tools.sh"
|
||||
- "bin/install/packaging-tools.sh"
|
||||
- "package/images/packaging/**"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/build-packaging-images.yml"
|
||||
- "bin/install-packaging-tools.sh"
|
||||
- "bin/install/packaging-tools.sh"
|
||||
- "package/images/packaging/**"
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
3
.github/workflows/on-pr.yml
vendored
3
.github/workflows/on-pr.yml
vendored
@@ -93,9 +93,8 @@ jobs:
|
||||
.github/workflows/reusable-upload-recipe.yml
|
||||
.clang-tidy
|
||||
.codecov.yml
|
||||
bin/check-nix-store-refs.sh
|
||||
bin/check-tools.sh
|
||||
bin/default-loader-path.sh
|
||||
bin/nix/**
|
||||
cfg/**
|
||||
cmake/**
|
||||
conan/**
|
||||
|
||||
3
.github/workflows/on-trigger.yml
vendored
3
.github/workflows/on-trigger.yml
vendored
@@ -31,9 +31,8 @@ on:
|
||||
- ".github/workflows/reusable-upload-recipe.yml"
|
||||
- ".clang-tidy"
|
||||
- ".codecov.yml"
|
||||
- "bin/check-nix-store-refs.sh"
|
||||
- "bin/check-tools.sh"
|
||||
- "bin/default-loader-path.sh"
|
||||
- "bin/nix/**"
|
||||
- "cfg/**"
|
||||
- "cmake/**"
|
||||
- "conan/**"
|
||||
|
||||
10
.github/workflows/reusable-build-test-config.yml
vendored
10
.github/workflows/reusable-build-test-config.yml
vendored
@@ -257,20 +257,20 @@ jobs:
|
||||
# cache included, since what it holds is what gets uploaded and reused.
|
||||
- name: Check the build output for Nix store references (Nix toolchain)
|
||||
if: ${{ inputs.toolchain == 'nix' }}
|
||||
run: ./bin/check-nix-store-refs.sh "${BUILD_DIR}"
|
||||
run: ./bin/nix/check-nix-store-refs.sh "${BUILD_DIR}"
|
||||
|
||||
- name: Check the Conan cache for Nix store references (Nix toolchain)
|
||||
if: ${{ inputs.toolchain == 'nix' }}
|
||||
run: ./bin/check-nix-store-refs.sh "${CONAN_HOME}"
|
||||
run: ./bin/nix/check-nix-store-refs.sh "${CONAN_HOME}"
|
||||
|
||||
# Only what PatchNixBinary.cmake retargets: the toolchain in the Linux
|
||||
# images always references the store. Same condition it uses.
|
||||
- name: Check for Nix store references (Linux)
|
||||
if: ${{ runner.os == 'Linux' && env.SANITIZERS_ENABLED == 'false' }}
|
||||
run: |
|
||||
./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpld"
|
||||
./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpl_tests"
|
||||
./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpld_tests"
|
||||
./bin/nix/check-nix-store-refs.sh "${BUILD_DIR}/xrpld"
|
||||
./bin/nix/check-nix-store-refs.sh "${BUILD_DIR}/xrpl_tests"
|
||||
./bin/nix/check-nix-store-refs.sh "${BUILD_DIR}/xrpld_tests"
|
||||
|
||||
- name: Show ccache statistics
|
||||
if: ${{ inputs.ccache_enabled }}
|
||||
|
||||
2
.github/workflows/upload-conan-deps.yml
vendored
2
.github/workflows/upload-conan-deps.yml
vendored
@@ -115,7 +115,7 @@ jobs:
|
||||
|
||||
- name: Check the Conan cache for Nix store references (Nix toolchain)
|
||||
if: ${{ matrix.toolchain == 'nix' }}
|
||||
run: ./bin/check-nix-store-refs.sh "${CONAN_HOME}"
|
||||
run: ./bin/nix/check-nix-store-refs.sh "${CONAN_HOME}"
|
||||
|
||||
- name: Log into Conan remote
|
||||
if: ${{ github.repository == 'XRPLF/rippled' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }}
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
# update moves the GCC runtime to a new store path, so a cached binary has to
|
||||
# get by with the pinned glibc alone. See docs/build/nix.md.
|
||||
#
|
||||
# Usage: bin/check-build-context-runtime.sh <graph.json>
|
||||
# Usage: bin/nix/check-build-context-runtime.sh <graph.json>
|
||||
# <graph.json> is the output of `conan install --format=json`.
|
||||
|
||||
set -euo pipefail
|
||||
@@ -10,7 +10,7 @@
|
||||
# alone; the scripts in a Conan cache are all git hook samples and autotools
|
||||
# scratch, 36 false positives to 0 real.
|
||||
#
|
||||
# Usage: bin/check-nix-store-refs.sh <path>
|
||||
# Usage: bin/nix/check-nix-store-refs.sh <path>
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
(runtime libraries resolved through the rpath) are skipped too.
|
||||
Everywhere else `patch_nix_binary` is a no-op.
|
||||
|
||||
The default loader is resolved by bin/default-loader-path.sh.
|
||||
The default loader is resolved by bin/nix/default-loader-path.sh.
|
||||
#]===================================================================]
|
||||
|
||||
include_guard(GLOBAL)
|
||||
@@ -25,7 +25,7 @@ include_guard(GLOBAL)
|
||||
include(CompilationEnv)
|
||||
|
||||
# Resolves the system default ELF loader path for the current architecture.
|
||||
set(_loader_path_script "${CMAKE_SOURCE_DIR}/bin/default-loader-path.sh")
|
||||
set(_loader_path_script "${CMAKE_SOURCE_DIR}/bin/nix/default-loader-path.sh")
|
||||
|
||||
if(
|
||||
is_linux
|
||||
|
||||
@@ -154,7 +154,7 @@ class Xrpl(ConanFile):
|
||||
self.requires("xxhash/0.8.3", transitive_headers=True)
|
||||
|
||||
exports_sources = (
|
||||
"bin/default-loader-path.sh",
|
||||
"bin/nix/default-loader-path.sh",
|
||||
"CMakeLists.txt",
|
||||
"cfg/*",
|
||||
"cmake/*",
|
||||
|
||||
12
docs/build/nix.md
vendored
12
docs/build/nix.md
vendored
@@ -183,14 +183,14 @@ at link or run time.
|
||||
> configuration CI covers, and no dependency binaries are published for it.
|
||||
|
||||
This is checked rather than assumed.
|
||||
[`bin/check-nix-store-refs.sh`](../../bin/check-nix-store-refs.sh) takes one file
|
||||
or directory and fails if a binary under it resolves a store path at run time.
|
||||
[`bin/nix/check-nix-store-refs.sh`](../../bin/nix/check-nix-store-refs.sh) takes one
|
||||
file or directory and fails if a binary under it resolves a store path at run time.
|
||||
CI runs it over the build output and the Conan cache, and again in the upload job
|
||||
before anything is published. You can run it yourself:
|
||||
|
||||
```bash
|
||||
bin/check-nix-store-refs.sh build
|
||||
bin/check-nix-store-refs.sh ~/.conan2-nix
|
||||
bin/nix/check-nix-store-refs.sh build
|
||||
bin/nix/check-nix-store-refs.sh ~/.conan2-nix
|
||||
```
|
||||
|
||||
It works on Linux too, but asserts something narrower there: the toolchain always
|
||||
@@ -204,7 +204,7 @@ instrument them. CI checks that they load nothing from the store but glibc, from
|
||||
the graph `conan install --format=json` writes:
|
||||
|
||||
```bash
|
||||
bin/check-build-context-runtime.sh graph.json
|
||||
bin/nix/check-build-context-runtime.sh graph.json
|
||||
```
|
||||
|
||||
Only the binaries [`PatchNixBinary.cmake`](../../cmake/PatchNixBinary.cmake)
|
||||
@@ -212,7 +212,7 @@ retargets to the system loader have to be fully clean, and those are what CI
|
||||
checks:
|
||||
|
||||
```bash
|
||||
bin/check-nix-store-refs.sh build/xrpld
|
||||
bin/nix/check-nix-store-refs.sh build/xrpld
|
||||
```
|
||||
|
||||
### The libresolv stub
|
||||
|
||||
6
docs/build/nix_troubleshooting.md
vendored
6
docs/build/nix_troubleshooting.md
vendored
@@ -178,11 +178,11 @@ A binary stops starting after a `nix flake update`, or after
|
||||
dyld[57271]: Library not loaded: /nix/store/…-libresolv-93/lib/libresolv.9.dylib
|
||||
```
|
||||
|
||||
[`bin/check-nix-store-refs.sh`](../../bin/check-nix-store-refs.sh) finds the same
|
||||
thing without having to run anything, and names the file:
|
||||
[`bin/nix/check-nix-store-refs.sh`](../../bin/nix/check-nix-store-refs.sh) finds the
|
||||
same thing without having to run anything, and names the file:
|
||||
|
||||
```
|
||||
$ bin/check-nix-store-refs.sh ~/.conan2-nix
|
||||
$ bin/nix/check-nix-store-refs.sh ~/.conan2-nix
|
||||
::error file=/Users/you/.conan2-nix/p/b/c-area24ded30c388c/p/bin/adig::references the Nix store at run time
|
||||
/Users/you/.conan2-nix/p/b/c-area24ded30c388c/p/bin/adig
|
||||
/nix/store/p4lp3xq4imd1qzqh08x8vcq2zfhi7rca-libresolv-93/lib/libresolv.9.dylib
|
||||
|
||||
@@ -60,10 +60,10 @@ ENV GIT_SSL_CAINFO="/nix/ci-env/etc/ssl/certs/ca-bundle.crt"
|
||||
# Externally-built dynamically-linked ELF binaries hard-code the loader path
|
||||
# (e.g. /lib64/ld-linux-x86-64.so.2) in their PT_INTERP header. Install it
|
||||
# from the Nix store when the base image doesn't already provide one.
|
||||
COPY bin/default-loader-path.sh /tmp/loader-path.sh
|
||||
COPY bin/nix/default-loader-path.sh /usr/local/bin/default-loader-path.sh
|
||||
|
||||
RUN <<EOF
|
||||
target="$(/tmp/loader-path.sh)"
|
||||
target="$(/usr/local/bin/default-loader-path.sh)"
|
||||
|
||||
if [ ! -e "${target}" ]; then
|
||||
# Use the loader from the same glibc that gcc links libc against, so
|
||||
@@ -101,7 +101,7 @@ RUN if echo "${BASE_IMAGE}" | grep -qiE 'nixos'; then \
|
||||
SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]
|
||||
|
||||
# Sanity-check that the built binaries run correctly in the vanilla base image, with the necessary sanitizer runtime libraries installed.
|
||||
COPY bin/install-sanitizer-libs.sh /tmp/install-sanitizer-libs.sh
|
||||
COPY bin/install/sanitizer-libs.sh /tmp/install-sanitizer-libs.sh
|
||||
COPY nix/docker/test_files/cpp/run-binaries.sh /tmp/test_files/cpp/run-binaries.sh
|
||||
COPY nix/docker/test_files/rust/run-binaries.sh /tmp/test_files/rust/run-binaries.sh
|
||||
COPY --from=final /tmp/cpp-bins /tmp/cpp-bins
|
||||
|
||||
@@ -52,10 +52,10 @@ work without `ca-certificates` being installed in the base image.
|
||||
workspace with `cargo` to exercise proc-macro dylib loading.
|
||||
3. **`tester`** — Start again from a clean `BASE_IMAGE` (no Nix toolchain),
|
||||
install only the sanitizer runtime libraries
|
||||
([`install-sanitizer-libs.sh`](./install-sanitizer-libs.sh)), and run the
|
||||
binaries compiled in `final`. This proves the binaries built with the Nix
|
||||
toolchain actually run on a vanilla base image. On `nixos/nix` this step is
|
||||
skipped (the binaries are patched for a conventional FHS loader).
|
||||
([`bin/install/sanitizer-libs.sh`](../../bin/install/sanitizer-libs.sh)),
|
||||
and run the binaries compiled in `final`. This proves the binaries built with
|
||||
the Nix toolchain actually run on a vanilla base image. On `nixos/nix` this
|
||||
step is skipped (the binaries are patched for a conventional FHS loader).
|
||||
4. **Output** — The final image is gated on the tester succeeding: it copies a
|
||||
sentinel file out of `tester`, so a failed test run fails the whole build.
|
||||
|
||||
@@ -75,9 +75,10 @@ toolchain being present at runtime. Two pieces make that work:
|
||||
|
||||
- **An expected dynamic linker in the image.**
|
||||
Binaries built in Nix environments reference a dynamic linker from Nix store paths, which won't be present in the base image. However,
|
||||
[`bin/default-loader-path.sh`](../../bin/default-loader-path.sh) reports the
|
||||
[`bin/nix/default-loader-path.sh`](../../bin/nix/default-loader-path.sh) reports the
|
||||
expected loader path for the current architecture, so we can patch the binaries
|
||||
to use the correct loader.
|
||||
to use the correct loader. The image ships it as
|
||||
`/usr/local/bin/default-loader-path.sh`.
|
||||
|
||||
The build then verifies all of this end to end, and the C++ and Rust programs
|
||||
go through the same pipeline: each is compiled in `final`, has its `PT_INTERP`
|
||||
@@ -91,11 +92,11 @@ whose resulting binary is patched and run like the others.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
|
||||
| [`./Dockerfile`](./Dockerfile) | Multi-stage build described above. |
|
||||
| [`./test_files/cpp/`](./test_files/cpp) | C++ sanitizer smoke test: sources + compile/run scripts. |
|
||||
| [`./test_files/rust/`](./test_files/rust) | Rust smoke test: rustc sources + a cargo proc-macro workspace + compile/run scripts. |
|
||||
| [`/bin/check-tools.sh`](../../bin/check-tools.sh) | Verify every expected tools are present and runnable. |
|
||||
| [`/bin/default-loader-path.sh`](../../bin/default-loader-path.sh) | Print the dynamic-linker (`PT_INTERP`) path for the current architecture. |
|
||||
| [`/bin/install-sanitizer-libs.sh`](../../bin/install-sanitizer-libs.sh) | Install `libasan`/`libtsan`/`libubsan` runtimes on the supported base images. |
|
||||
| File | Purpose |
|
||||
| ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
|
||||
| [`./Dockerfile`](./Dockerfile) | Multi-stage build described above. |
|
||||
| [`./test_files/cpp/`](./test_files/cpp) | C++ sanitizer smoke test: sources + compile/run scripts. |
|
||||
| [`./test_files/rust/`](./test_files/rust) | Rust smoke test: rustc sources + a cargo proc-macro workspace + compile/run scripts. |
|
||||
| [`/bin/check-tools.sh`](../../bin/check-tools.sh) | Verify every expected tools are present and runnable. |
|
||||
| [`/bin/nix/default-loader-path.sh`](../../bin/nix/default-loader-path.sh) | Print the dynamic-linker (`PT_INTERP`) path for the current architecture. |
|
||||
| [`/bin/install/sanitizer-libs.sh`](../../bin/install/sanitizer-libs.sh) | Install `libasan`/`libtsan`/`libubsan` runtimes on the supported base images. |
|
||||
|
||||
@@ -8,7 +8,7 @@ set -eo pipefail
|
||||
src_dir="${1:?usage: $0 <src_dir> <dst_dir>}"
|
||||
dst_dir="${2:?usage: $0 <src_dir> <dst_dir>}"
|
||||
|
||||
loader="$(/tmp/loader-path.sh)"
|
||||
loader="$(/usr/local/bin/default-loader-path.sh)"
|
||||
|
||||
mkdir -p "${dst_dir}"
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ set -eo pipefail
|
||||
src_dir="${1:?usage: $0 <src_dir> <dst_dir>}"
|
||||
dst_dir="${2:?usage: $0 <src_dir> <dst_dir>}"
|
||||
|
||||
loader="$(/tmp/loader-path.sh)"
|
||||
loader="$(/usr/local/bin/default-loader-path.sh)"
|
||||
|
||||
mkdir -p "${dst_dir}"
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ package/
|
||||
sign_rpm.py Signs the built RPMs (called by CI when publishing)
|
||||
images/
|
||||
packaging/
|
||||
Dockerfile Packaging image, built by `build-packaging-images.yml`; installs its tooling with `bin/install-packaging-tools.sh`
|
||||
Dockerfile Packaging image, built by `build-packaging-images.yml`; installs its tooling with `bin/install/packaging-tools.sh`
|
||||
publish_pkg.py Uploads built packages to the XRPLF Nexus repositories (called by CI, and shipped in that image)
|
||||
xrpld/
|
||||
Dockerfile The xrpld Docker images, installing the built DEB on Ubuntu (see "Docker images")
|
||||
|
||||
@@ -2,7 +2,7 @@ ARG BASE_IMAGE=debian:trixie
|
||||
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
RUN --mount=type=bind,source=bin/install-packaging-tools.sh,target=/install-packaging-tools.sh \
|
||||
RUN --mount=type=bind,source=bin/install/packaging-tools.sh,target=/install-packaging-tools.sh \
|
||||
/install-packaging-tools.sh
|
||||
|
||||
# See package/README.md, "Publishing from other repositories".
|
||||
|
||||
Reference in New Issue
Block a user