ci: Refactor build-related nix / docker / workflows (#7408)

.github/scripts/strategy-matrix/linux.json

@@ -1,5 +1,5 @@
 {
-  "image_tag": "sha-6c54342",
+  "image_tag": "sha-8abe82e",
   "configs": {
     "ubuntu": [
       {

.github/workflows/build-nix-image.yml

@@ -1,109 +0,0 @@
-name: Build Nix Docker image
-
-on:
-  push:
-    branches:
-      - develop
-    paths:
-      - ".github/workflows/build-nix-image.yml"
-      - ".github/workflows/reusable-build-docker-image.yml"
-      - "docker/**"
-      - "flake.nix"
-      - "flake.lock"
-      - "nix/**"
-  pull_request:
-    paths:
-      - ".github/workflows/build-nix-image.yml"
-      - ".github/workflows/reusable-build-docker-image.yml"
-      - "docker/**"
-      - "flake.nix"
-      - "flake.lock"
-      - "nix/**"
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  build:
-    name: Build ${{ matrix.distro.name }} (${{ matrix.target.platform }})
-    permissions:
-      contents: read
-      packages: write
-    strategy:
-      fail-fast: false
-      matrix:
-        # The base images are the oldest supported version of each distro
-        # that we want to build images for.
-        distro:
-          - name: nixos
-            base_image: nixos/nix:latest
-          - name: ubuntu
-            base_image: ubuntu:20.04
-          - name: rhel
-            base_image: registry.access.redhat.com/ubi9/ubi:latest
-          - name: debian
-            base_image: debian:bookworm
-        target:
-          - platform: linux/amd64
-            runner: ubuntu-latest
-          - platform: linux/arm64
-            runner: ubuntu-24.04-arm
-    uses: ./.github/workflows/reusable-build-docker-image.yml
-    with:
-      image_name: ghcr.io/xrplf/xrpld/nix-${{ matrix.distro.name }}
-      dockerfile: docker/nix.Dockerfile
-      base_image: ${{ matrix.distro.base_image }}
-      platform: ${{ matrix.target.platform }}
-      runner: ${{ matrix.target.runner }}
-      push: ${{ github.repository == 'XRPLF/rippled' && github.event_name == 'push' }}
-
-  merge:
-    name: Merge ${{ matrix.distro }} manifest
-    needs: build
-    if: ${{ github.repository == 'XRPLF/rippled' && github.event_name == 'push' }}
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    strategy:
-      fail-fast: false
-      matrix:
-        distro: [nixos, ubuntu, rhel, debian]
-    env:
-      IMAGE_NAME: ghcr.io/xrplf/xrpld/nix-${{ matrix.distro }}
-
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
-
-      - name: Docker metadata
-        id: meta
-        uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
-        with:
-          images: ${{ env.IMAGE_NAME }}
-          tags: |
-            type=sha,prefix=sha-,format=short
-            type=raw,value=latest
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Create multi-arch manifests
-        run: |
-          for tag in $(jq -cr '.tags[]' <<<"$DOCKER_METADATA_OUTPUT_JSON"); do
-              docker buildx imagetools create -t "$tag" "${tag}-amd64" "${tag}-arm64"
-          done
-
-      - name: Inspect image
-        run: |
-          docker buildx imagetools inspect "${IMAGE_NAME}:${{ steps.meta.outputs.version }}"

.github/workflows/build-nix-images.yml

@@ -0,0 +1,56 @@
+name: Build Nix Docker images
+
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - ".github/workflows/build-nix-images.yml"
+      - ".github/workflows/reusable-build-docker-image.yml"
+      - ".github/workflows/reusable-build-merge-docker-images.yml"
+      - "flake.nix"
+      - "flake.lock"
+      - "nix/**"
+  pull_request:
+    paths:
+      - ".github/workflows/build-nix-images.yml"
+      - ".github/workflows/reusable-build-docker-image.yml"
+      - ".github/workflows/reusable-build-merge-docker-images.yml"
+      - "flake.nix"
+      - "flake.lock"
+      - "nix/**"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build-merge:
+    name: Build and push nix-${{ matrix.distro.name }}
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        # The base images are the oldest supported version of each distro
+        # that we want to build images for.
+        distro:
+          - name: nixos
+            base_image: nixos/nix:latest
+          - name: ubuntu
+            base_image: ubuntu:20.04
+          - name: debian
+            base_image: debian:bookworm
+          - name: rhel
+            base_image: registry.access.redhat.com/ubi9/ubi:latest
+    uses: ./.github/workflows/reusable-build-merge-docker-images.yml
+    with:
+      image_name: ghcr.io/xrplf/xrpld/nix-${{ matrix.distro.name }}
+      dockerfile: nix/docker/Dockerfile
+      base_image: ${{ matrix.distro.base_image }}

.github/workflows/build-packaging-images.yml

@@ -0,0 +1,48 @@
+name: Build packaging Docker images
+
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - ".github/workflows/build-packaging-images.yml"
+      - ".github/workflows/reusable-build-docker-image.yml"
+      - ".github/workflows/reusable-build-merge-docker-images.yml"
+      - "package/Dockerfile"
+      - "package/install-packaging-tools.sh"
+  pull_request:
+    paths:
+      - ".github/workflows/build-packaging-images.yml"
+      - ".github/workflows/reusable-build-docker-image.yml"
+      - ".github/workflows/reusable-build-merge-docker-images.yml"
+      - "package/Dockerfile"
+      - "package/install-packaging-tools.sh"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build-merge:
+    name: Build and push packaging-${{ matrix.distro.name }}
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        distro:
+          - name: debian
+            base_image: debian:bookworm
+          - name: rhel
+            base_image: registry.access.redhat.com/ubi9/ubi:latest
+    uses: ./.github/workflows/reusable-build-merge-docker-images.yml
+    with:
+      image_name: ghcr.io/xrplf/xrpld/packaging-${{ matrix.distro.name }}
+      dockerfile: package/Dockerfile
+      base_image: ${{ matrix.distro.base_image }}

.github/workflows/publish-docs.yml

@@ -41,7 +41,7 @@ env:
 jobs:
   build:
     runs-on: ubuntu-latest
-    container: ghcr.io/xrplf/ci/tools-rippled-documentation:sha-a8c7be1
+    container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-8abe82e
     steps:
       - name: Checkout repository
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -57,19 +57,11 @@ jobs:
         with:
           subtract: ${{ env.NPROC_SUBTRACT }}
 
-      - name: Check configuration
-        run: |
-          echo 'Checking path.'
-          echo ${PATH} | tr ':' '\n'
+      - name: Print build environment
+        uses: XRPLF/actions/print-build-env@59dec886e4afb05a1724443af08baccbc045b574
 
-          echo 'Checking environment variables.'
-          env | sort
-
-          echo 'Checking CMake version.'
-          cmake --version
-
-          echo 'Checking Doxygen version.'
-          doxygen --version
+      - name: Check Doxygen version
+        run: doxygen --version
 
       - name: Build documentation
         env:

.github/workflows/reusable-build-docker-image.yml

@@ -38,7 +38,7 @@ defaults:
 
 jobs:
   build:
-    name: Build (${{ inputs.platform }})
+    name: Build ${{ inputs.platform }}
     runs-on: ${{ inputs.runner }}
     permissions:
       contents: read

.github/workflows/reusable-build-merge-docker-images.yml

@@ -0,0 +1,89 @@
+name: Reusable build and merge Docker image (multi-arch)
+
+on:
+  workflow_call:
+    inputs:
+      image_name:
+        description: "Full image name without tag (e.g. 'ghcr.io/xrplf/xrpld/nix-ubuntu')"
+        required: true
+        type: string
+      dockerfile:
+        description: "Path to the Dockerfile, relative to the repository root"
+        required: true
+        type: string
+      base_image:
+        description: "Value passed to the Dockerfile as the BASE_IMAGE build arg"
+        required: true
+        type: string
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build:
+    name: Build ${{ inputs.image_name }}
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+
+    uses: ./.github/workflows/reusable-build-docker-image.yml
+    with:
+      image_name: ${{ inputs.image_name }}
+      dockerfile: ${{ inputs.dockerfile }}
+      base_image: ${{ inputs.base_image }}
+      platform: ${{ matrix.target.platform }}
+      runner: ${{ matrix.target.runner }}
+      push: ${{ github.repository == 'XRPLF/rippled' && github.event_name == 'push' }}
+
+  merge:
+    name: Merge ${{ inputs.image_name }}
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
+
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
+        with:
+          images: ${{ inputs.image_name }}
+          tags: |
+            type=sha,prefix=sha-,format=short
+            type=raw,value=latest
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create multi-arch manifests
+        if: ${{ github.repository == 'XRPLF/rippled' && github.event_name == 'push' }}
+        run: |
+          for tag in $(jq -cr '.tags[]' <<<"$DOCKER_METADATA_OUTPUT_JSON"); do
+              docker buildx imagetools create -t "$tag" "${tag}-amd64" "${tag}-arm64"
+          done
+
+      - name: Inspect image
+        if: ${{ github.repository == 'XRPLF/rippled' && github.event_name == 'push' }}
+        env:
+          IMAGE_NAME: ${{ inputs.image_name }}
+          IMAGE_VERSION: ${{ steps.meta.outputs.version }}
+        run: |
+          docker buildx imagetools inspect "${IMAGE_NAME}:${IMAGE_VERSION}"

.github/workflows/reusable-upload-recipe.yml

@@ -40,7 +40,7 @@ defaults:
 jobs:
   upload:
     runs-on: ubuntu-latest
-    container: ghcr.io/xrplf/ci/ubuntu-noble:gcc-13-sha-5dd7158
+    container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-8abe82e
     steps:
       - name: Checkout repository
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

nix/docker/Dockerfile

@@ -56,7 +56,7 @@ ENV GIT_SSL_CAINFO="/nix/ci-env/etc/ssl/certs/ca-bundle.crt"
 # Externally-built dynamically-linked ELF binaries hard-code the loader path
 # (e.g. /lib64/ld-linux-x86-64.so.2) in their PT_INTERP header. Install it
 # from the Nix store when the base image doesn't already provide one.
-COPY docker/loader-path.sh /tmp/loader-path.sh
+COPY nix/docker/loader-path.sh /tmp/loader-path.sh
 
 RUN <<EOF
 target="$(/tmp/loader-path.sh)"
@@ -71,12 +71,12 @@ if [ ! -e "${target}" ]; then
 fi
 EOF
 
-COPY docker/check-tools.sh /tmp/check-tools.sh
+COPY nix/docker/check-tools.sh /tmp/check-tools.sh
 RUN /tmp/check-tools.sh
 
 # Sanity-check that the g++/clang++ are able to build binaries, including sanitizer-instrumented ones.
-COPY docker/test_files/cpp_sources/ /tmp/cpp_sources/
-COPY docker/test_files/compile-cpp-sources.sh /tmp/compile-cpp-sources.sh
+COPY nix/docker/test_files/cpp_sources/ /tmp/cpp_sources/
+COPY nix/docker/test_files/compile-cpp-sources.sh /tmp/compile-cpp-sources.sh
 RUN /tmp/compile-cpp-sources.sh /tmp/cpp_sources /tmp/bins
 
 # Tester: start from a clean BASE_IMAGE, install sanitizer runtime libraries,
@@ -93,8 +93,8 @@ RUN if echo "${BASE_IMAGE}" | grep -qiE 'nixos'; then \
 SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]
 
 # Sanity-check that the built binaries run correctly in the vanilla base image, with the necessary sanitizer runtime libraries installed.
-COPY docker/install-sanitizer-libs.sh /tmp/install-sanitizer-libs.sh
-COPY docker/test_files/run-test-binaries.sh /tmp/run-test-binaries.sh
+COPY nix/docker/install-sanitizer-libs.sh /tmp/install-sanitizer-libs.sh
+COPY nix/docker/test_files/run-test-binaries.sh /tmp/run-test-binaries.sh
 COPY --from=final /tmp/bins /tmp/bins
 
 RUN <<EOF

package/Dockerfile

@@ -0,0 +1,14 @@
+ARG BASE_IMAGE=debian:bookworm
+
+FROM ${BASE_IMAGE}
+
+# Packaging runs in a vanilla distro image, so the tooling has to come
+# from the distro's archive: debhelper for deb, rpm-build (and the
+# systemd / find-debuginfo macros it depends on) for rpm.
+# The container also uses git (real history) for
+# build_pkg.sh's SOURCE_DATE_EPOCH; otherwise it falls back to a tarball
+# download and the timestamp comes from wall-clock time.
+
+COPY package/install-packaging-tools.sh /tmp/install-packaging-tools.sh
+
+RUN /tmp/install-packaging-tools.sh

package/README.md

@@ -22,25 +22,24 @@ package/
 
 Packaging targets and their container images are declared in
 [`.github/scripts/strategy-matrix/linux.json`](../.github/scripts/strategy-matrix/linux.json)
-via a `"package": true` field on specific os entries. Today only
-`linux/amd64` is emitted; the architecture is hardcoded in `generate.py`
-and the workflow runner. The package format
+inside `package_configs` configurations. Today only
+`linux/amd64` is emitted. The package format
 (deb or rpm) is inferred at build time from the container's package manager
 (`apt-get` -> deb, `dnf`/`yum` -> rpm). The image tag is composed as
-`ghcr.io/xrplf/ci/{distro}-{version}:{compiler}-{cver}-sha-{image_sha}` —
+`ghcr.io/xrplf/xrpld/packaging-<distro>:sha-<git_sha>` —
 the same scheme used by `reusable-build-test.yml`. Bump `image_sha` in
 `linux.json` and both CI and local builds pick up the new image with no
 workflow edits.
 
 | Package type | Image (derived from `linux.json`)                    | Tool required                                                   |
 | ------------ | ---------------------------------------------------- | --------------------------------------------------------------- |
-| RPM          | `ghcr.io/xrplf/ci/rhel-9:gcc-12-sha-<git_sha>`       | `rpmbuild`                                                      |
-| DEB          | `ghcr.io/xrplf/ci/ubuntu-jammy:gcc-12-sha-<git_sha>` | `dpkg-buildpackage`, `debhelper (>= 13)`, `dh-sequence-systemd` |
+| RPM          | `ghcr.io/xrplf/xrpld/packaging-rhel:sha-<git_sha>`   | `rpmbuild`                                                      |
+| DEB          | `ghcr.io/xrplf/xrpld/packaging-debian:sha-<git_sha>` | `dpkg-buildpackage`, `debhelper (>= 13)`, `dh-sequence-systemd` |
 
 To print the exact image tags for the current `linux.json`:
 
 ```bash
-./.github/scripts/strategy-matrix/generate.py --packaging --config=.github/scripts/strategy-matrix/linux.json
+./.github/scripts/strategy-matrix/generate.py --packaging
 ```
 
 ## Building packages

package/install-packaging-tools.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [ ! -f /etc/os-release ]; then
+    echo "ERROR: /etc/os-release not found; cannot detect OS" >&2
+    exit 1
+fi
+
+# shellcheck source=/dev/null
+. /etc/os-release
+
+echo "Detected OS: ${ID} ${VERSION_ID:-}"
+
+case "${ID}" in
+    ubuntu | debian | rhel | centos | rocky | almalinux)
+        echo "Supported OS detected: ${ID}"
+        ;;
+    *)
+        echo "ERROR: unsupported OS '${ID}'. Supported: debian, ubuntu, rhel-family" >&2
+        exit 1
+        ;;
+esac
+
+function install() {
+    case "${ID}" in
+        debian | ubuntu)
+            apt-get update -y
+            apt-get install -y --no-install-recommends \
+                ca-certificates \
+                debhelper \
+                debhelper-compat \
+                dpkg-dev \
+                git
+            ;;
+
+        rhel | centos | rocky | almalinux)
+            dnf install -y --setopt=install_weak_deps=False \
+                git \
+                rpm-build \
+                redhat-rpm-config \
+                systemd-rpm-macros
+            ;;
+    esac
+}
+
+function postinstall() {
+    # Don't clear cache in non-CI environments
+    if [ -z "${CI:-}" ]; then
+        echo "Not running in CI environment; skipping cache cleanup"
+        return
+    fi
+
+    case "${ID}" in
+        debian | ubuntu)
+            apt-get clean
+            rm -rf /var/lib/apt/lists/*
+            ;;
+
+        rhel | centos | rocky | almalinux)
+            dnf clean -y all
+            rm -rf /var/cache/dnf/*
+            ;;
+    esac
+}
+
+install
+postinstall