From ed7741010019e855d0204144162e863e63c41759 Mon Sep 17 00:00:00 2001 From: Michael Legleux Date: Tue, 21 Apr 2026 13:26:53 -0700 Subject: [PATCH] update matrix to build packages --- .github/scripts/strategy-matrix/generate.py | 69 ++++++++++-- .github/scripts/strategy-matrix/linux.json | 6 +- .github/workflows/on-pr.yml | 25 +++-- .github/workflows/on-tag.yml | 24 ++-- .github/workflows/on-trigger.yml | 23 ++-- .github/workflows/reusable-package.yml | 4 +- .../workflows/reusable-strategy-matrix.yml | 13 ++- cmake/XrplPackaging.cmake | 2 + package/README.md | 103 +++++++++++++----- package/deb/debian/rules | 16 +-- 10 files changed, 197 insertions(+), 88 deletions(-) diff --git a/.github/scripts/strategy-matrix/generate.py b/.github/scripts/strategy-matrix/generate.py index 9b2315af60..ec08674558 100755 --- a/.github/scripts/strategy-matrix/generate.py +++ b/.github/scripts/strategy-matrix/generate.py @@ -32,6 +32,52 @@ We will further set additional CMake arguments as follows: """ +def build_config_name(os_entry: dict, architecture: dict, build_type: str) -> str: + name = os_entry["distro_name"] + if (n := os_entry["distro_version"]) != "": + name += f"-{n}" + if (n := os_entry["compiler_name"]) != "": + name += f"-{n}" + if (n := os_entry["compiler_version"]) != "": + name += f"-{n}" + platform = architecture["platform"] + name += f"-{platform[platform.find('/') + 1:]}" + name += f"-{build_type.lower()}" + return name + + +def build_container_image(os_entry: dict) -> str: + return ( + f"ghcr.io/xrplf/ci/{os_entry['distro_name']}-{os_entry['distro_version']}:" + f"{os_entry['compiler_name']}-{os_entry['compiler_version']}-sha-{os_entry['image_sha']}" + ) + + +def generate_packaging_matrix(config: Config) -> list: + """Emit packaging entries for each os entry with a 'packaging' field. + Packaging always uses Release build on linux/amd64. + """ + architecture = next( + (a for a in config.architecture if a["platform"] == "linux/amd64"), + None, + ) + if architecture is None: + raise Exception("linux/amd64 architecture required for packaging") + + entries = [] + for os_entry in config.os: + for pkg_type in os_entry.get("packaging", []): + config_name = build_config_name(os_entry, architecture, "Release") + entries.append( + { + "pkg_type": pkg_type, + "artifact_name": f"xrpld-{config_name}", + "container_image": build_container_image(os_entry), + } + ) + return entries + + def generate_strategy_matrix(all: bool, config: Config) -> list: configurations = [] for architecture, os, build_type, cmake_args in itertools.product( @@ -223,17 +269,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list: # Generate a unique name for the configuration, e.g. macos-arm64-debug # or debian-bookworm-gcc-12-amd64-release. - config_name = os["distro_name"] - if (n := os["distro_version"]) != "": - config_name += f"-{n}" - if (n := os["compiler_name"]) != "": - config_name += f"-{n}" - if (n := os["compiler_version"]) != "": - config_name += f"-{n}" - config_name += ( - f"-{architecture['platform'][architecture['platform'].find('/')+1:]}" - ) - config_name += f"-{build_type.lower()}" + config_name = build_config_name(os, architecture, build_type) if "-Dcoverage=ON" in cmake_args: config_name += "-coverage" if "-Dunity=ON" in cmake_args: @@ -321,10 +357,19 @@ if __name__ == "__main__": required=False, type=Path, ) + parser.add_argument( + "-p", + "--packaging", + help="Emit the packaging matrix (derived from the 'packaging' field on os entries) instead of the build/test matrix.", + action="store_true", + ) args = parser.parse_args() matrix = [] - if args.config is None or args.config == "": + if args.packaging: + config_path = args.config if args.config else THIS_DIR / "linux.json" + matrix += generate_packaging_matrix(read_config(config_path)) + elif args.config is None or args.config == "": matrix += generate_strategy_matrix( args.all, read_config(THIS_DIR / "linux.json") ) diff --git a/.github/scripts/strategy-matrix/linux.json b/.github/scripts/strategy-matrix/linux.json index 4943579be8..864116474d 100644 --- a/.github/scripts/strategy-matrix/linux.json +++ b/.github/scripts/strategy-matrix/linux.json @@ -120,7 +120,8 @@ "distro_version": "9", "compiler_name": "gcc", "compiler_version": "12", - "image_sha": "ab4d1f0" + "image_sha": "ab4d1f0", + "packaging": ["rpm"] }, { "distro_name": "rhel", @@ -162,7 +163,8 @@ "distro_version": "jammy", "compiler_name": "gcc", "compiler_version": "12", - "image_sha": "ab4d1f0" + "image_sha": "ab4d1f0", + "packaging": ["deb"] }, { "distro_name": "ubuntu", diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index 30bd6847ed..f62854f46b 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -157,23 +157,24 @@ jobs: id: version uses: ./.github/actions/generate-version - package-deb: - needs: [should-run, build-test, generate-version] - uses: ./.github/workflows/reusable-package.yml + generate-packaging-matrix: + needs: should-run + if: ${{ needs.should-run.outputs.go == 'true' }} + uses: ./.github/workflows/reusable-strategy-matrix.yml with: - pkg_type: deb - artifact_name: xrpld-ubuntu-jammy-gcc-12-amd64-release - version: ${{ needs.generate-version.outputs.version }} - container_image: ghcr.io/xrplf/ci/ubuntu-jammy:gcc-12@sha256:14e463a9a88060b5a1e579204525c4a6f23afdd0255446f8e4f5415ae0cedac6 + mode: packaging - package-rpm: - needs: [should-run, build-test, generate-version] + package: + needs: [should-run, build-test, generate-version, generate-packaging-matrix] + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-packaging-matrix.outputs.matrix) }} uses: ./.github/workflows/reusable-package.yml with: - pkg_type: rpm - artifact_name: xrpld-rhel-9-gcc-12-amd64-release + pkg_type: ${{ matrix.pkg_type }} + artifact_name: ${{ matrix.artifact_name }} version: ${{ needs.generate-version.outputs.version }} - container_image: ghcr.io/xrplf/ci/rhel-9:gcc-12@sha256:196f97b62b8352da1b7cfa91c59d442cc4940eae89a05b547408f62cbddcb8b3 + container_image: ${{ matrix.container_image }} upload-recipe: needs: diff --git a/.github/workflows/on-tag.yml b/.github/workflows/on-tag.yml index d508f19847..b49af79955 100644 --- a/.github/workflows/on-tag.yml +++ b/.github/workflows/on-tag.yml @@ -53,20 +53,20 @@ jobs: id: version uses: ./.github/actions/generate-version - package-deb: - needs: [build-test, generate-version] - uses: ./.github/workflows/reusable-package.yml + generate-packaging-matrix: + if: ${{ github.repository == 'XRPLF/rippled' }} + uses: ./.github/workflows/reusable-strategy-matrix.yml with: - pkg_type: deb - artifact_name: xrpld-ubuntu-jammy-gcc-12-amd64-release - version: ${{ needs.generate-version.outputs.version }} - container_image: ghcr.io/xrplf/ci/ubuntu-jammy:gcc-12@sha256:14e463a9a88060b5a1e579204525c4a6f23afdd0255446f8e4f5415ae0cedac6 + mode: packaging - package-rpm: - needs: [build-test, generate-version] + package: + needs: [build-test, generate-version, generate-packaging-matrix] + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-packaging-matrix.outputs.matrix) }} uses: ./.github/workflows/reusable-package.yml with: - pkg_type: rpm - artifact_name: xrpld-rhel-9-gcc-12-amd64-release + pkg_type: ${{ matrix.pkg_type }} + artifact_name: ${{ matrix.artifact_name }} version: ${{ needs.generate-version.outputs.version }} - container_image: ghcr.io/xrplf/ci/rhel-9:gcc-12@sha256:196f97b62b8352da1b7cfa91c59d442cc4940eae89a05b547408f62cbddcb8b3 + container_image: ${{ matrix.container_image }} diff --git a/.github/workflows/on-trigger.yml b/.github/workflows/on-trigger.yml index 0962780f7a..f04d20fc4b 100644 --- a/.github/workflows/on-trigger.yml +++ b/.github/workflows/on-trigger.yml @@ -116,20 +116,19 @@ jobs: id: version uses: ./.github/actions/generate-version - package-deb: - needs: [build-test, generate-version] - uses: ./.github/workflows/reusable-package.yml + generate-packaging-matrix: + uses: ./.github/workflows/reusable-strategy-matrix.yml with: - pkg_type: deb - artifact_name: xrpld-ubuntu-jammy-gcc-12-amd64-release - version: ${{ needs.generate-version.outputs.version }} - container_image: ghcr.io/xrplf/ci/ubuntu-jammy:gcc-12@sha256:14e463a9a88060b5a1e579204525c4a6f23afdd0255446f8e4f5415ae0cedac6 + mode: packaging - package-rpm: - needs: [build-test, generate-version] + package: + needs: [build-test, generate-version, generate-packaging-matrix] + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-packaging-matrix.outputs.matrix) }} uses: ./.github/workflows/reusable-package.yml with: - pkg_type: rpm - artifact_name: xrpld-rhel-9-gcc-12-amd64-release + pkg_type: ${{ matrix.pkg_type }} + artifact_name: ${{ matrix.artifact_name }} version: ${{ needs.generate-version.outputs.version }} - container_image: ghcr.io/xrplf/ci/rhel-9:gcc-12@sha256:196f97b62b8352da1b7cfa91c59d442cc4940eae89a05b547408f62cbddcb8b3 + container_image: ${{ matrix.container_image }} diff --git a/.github/workflows/reusable-package.yml b/.github/workflows/reusable-package.yml index b166f85ed7..2a51782bc9 100644 --- a/.github/workflows/reusable-package.yml +++ b/.github/workflows/reusable-package.yml @@ -62,8 +62,8 @@ jobs: PKG_RELEASE: ${{ inputs.pkg_release }} run: | mkdir -p "${BUILD_DIR}/package/rpm" - sed -e "s/@xrpld_version@/${PKG_VERSION}/" \ - -e "s/@pkg_release@/${PKG_RELEASE}/" \ + sed -e "s|@xrpld_version@|${PKG_VERSION}|" \ + -e "s|@pkg_release@|${PKG_RELEASE}|" \ package/rpm/xrpld.spec.in > "${BUILD_DIR}/package/rpm/xrpld.spec" - name: Build package diff --git a/.github/workflows/reusable-strategy-matrix.yml b/.github/workflows/reusable-strategy-matrix.yml index b1232a138f..cff51ee897 100644 --- a/.github/workflows/reusable-strategy-matrix.yml +++ b/.github/workflows/reusable-strategy-matrix.yml @@ -3,6 +3,11 @@ name: Generate strategy matrix on: workflow_call: inputs: + mode: + description: 'What matrix to emit: "build-test" (default) or "packaging".' + required: false + type: string + default: "build-test" os: description: 'The operating system to use for the build ("linux", "macos", "windows").' required: false @@ -41,5 +46,11 @@ jobs: id: generate env: GENERATE_CONFIG: ${{ inputs.os != '' && format('--config={0}.json', inputs.os) || '' }} + GENERATE_MODE: ${{ inputs.mode == 'packaging' && '--packaging --config=linux.json' || '' }} GENERATE_OPTION: ${{ inputs.strategy_matrix == 'all' && '--all' || '' }} - run: ./generate.py ${GENERATE_OPTION} ${GENERATE_CONFIG} >> "${GITHUB_OUTPUT}" + run: | + if [ -n "${GENERATE_MODE}" ]; then + ./generate.py ${GENERATE_MODE} >> "${GITHUB_OUTPUT}" + else + ./generate.py ${GENERATE_OPTION} ${GENERATE_CONFIG} >> "${GITHUB_OUTPUT}" + fi diff --git a/cmake/XrplPackaging.cmake b/cmake/XrplPackaging.cmake index a95ec7da7d..1a7075f641 100644 --- a/cmake/XrplPackaging.cmake +++ b/cmake/XrplPackaging.cmake @@ -34,6 +34,7 @@ if(RPMBUILD_EXECUTABLE) ${CMAKE_SOURCE_DIR}/package/build_pkg.sh rpm ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + DEPENDS xrpld COMMENT "Building RPM package" VERBATIM ) @@ -49,6 +50,7 @@ if(DPKG_BUILDPACKAGE_EXECUTABLE) ${CMAKE_SOURCE_DIR}/package/build_pkg.sh deb ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} ${xrpld_version} ${pkg_release} WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + DEPENDS xrpld COMMENT "Building Debian package" VERBATIM ) diff --git a/package/README.md b/package/README.md index 11d1e12121..4b71d8e417 100644 --- a/package/README.md +++ b/package/README.md @@ -25,43 +25,98 @@ package/ ## Prerequisites -| Package type | Container | Tool required | -| ------------ | -------------------------------------- | --------------------------------------------------------------- | -| RPM | `ghcr.io/xrplf/ci/rhel-9:gcc-12` | `rpmbuild` | -| DEB | `ghcr.io/xrplf/ci/ubuntu-jammy:gcc-12` | `dpkg-buildpackage`, `debhelper (>= 13)`, `dh-sequence-systemd` | +Packaging targets and their container images are declared in +[`.github/scripts/strategy-matrix/linux.json`](../.github/scripts/strategy-matrix/linux.json) +via a `"packaging"` field on specific os entries. The image tag is composed +as `ghcr.io/xrplf/ci/{distro}-{version}:{compiler}-{cver}-sha-{image_sha}` — +the same scheme used by `reusable-build-test.yml`. Bump `image_sha` in +`linux.json` and both CI and local builds pick up the new image with no +workflow edits. + +| Package type | Image (derived from `linux.json`) | Tool required | +| ------------ | ---------------------------------------------------- | --------------------------------------------------------------- | +| RPM | `ghcr.io/xrplf/ci/rhel-9:gcc-12-sha-` | `rpmbuild` | +| DEB | `ghcr.io/xrplf/ci/ubuntu-jammy:gcc-12-sha-` | `dpkg-buildpackage`, `debhelper (>= 13)`, `dh-sequence-systemd` | + +To print the exact image tags for the current `linux.json`: + +```bash +./.github/scripts/strategy-matrix/generate.py --packaging --config=.github/scripts/strategy-matrix/linux.json +``` ## Building packages -### Via CI (recommended) +### Via CI -The `reusable-package.yml` workflow downloads a pre-built `xrpld` binary artifact -and calls `build_pkg.sh` directly. No CMake configure or build step is needed in -the packaging job. +Caller workflows (`on-pr.yml`, `on-tag.yml`, `on-trigger.yml`) call +`reusable-strategy-matrix.yml` with `mode: packaging` to generate the matrix of +`{pkg_type, artifact_name, container_image}` entries, then fan out to +`reusable-package.yml` per entry. That workflow downloads the pre-built `xrpld` +binary artifact and calls `build_pkg.sh` directly — no CMake configure or +build step is needed inside the packaging job. -### Via CMake (local development) +### Locally (mirrors CI) -Configure with the required install prefix, then invoke the target: +With an `xrpld` binary already built at `build/xrpld`, run the packaging step +inside the same container CI uses. The image tag is derived from `linux.json` +so you don't need to hardcode a SHA. + +```bash +# From the repo root. +PKG_TYPE=deb # or rpm +VERSION=2.4.0-local +PKG_RELEASE=1 + +# Derive the correct image for this package type from linux.json. +IMAGE=$(jq -r --arg pkg "$PKG_TYPE" ' + .os[] | select((.packaging // []) | index($pkg)) | + "ghcr.io/xrplf/ci/\(.distro_name)-\(.distro_version):\(.compiler_name)-\(.compiler_version)-sha-\(.image_sha)" +' .github/scripts/strategy-matrix/linux.json) + +# RPM only: generate the spec from the template (CMake does this automatically +# during configure; this mirrors the CI step for direct invocations). +if [ "$PKG_TYPE" = "rpm" ] && [ ! -f build/package/rpm/xrpld.spec ]; then + mkdir -p build/package/rpm + sed -e "s|@xrpld_version@|$VERSION|" \ + -e "s|@pkg_release@|$PKG_RELEASE|" \ + package/rpm/xrpld.spec.in > build/package/rpm/xrpld.spec +fi + +# Run the packaging in the container. +docker run --rm \ + -v "$(pwd):/src" \ + -w /src \ + "$IMAGE" \ + ./package/build_pkg.sh "$PKG_TYPE" . build "$VERSION" "$PKG_RELEASE" + +# Output: +# build/debbuild/*.deb (DEB + dbgsym .ddeb) +# build/rpmbuild/RPMS/x86_64/*.rpm +``` + +### Via CMake (host-side target) + +If you run CMake configure on a host that has `rpmbuild` or `dpkg-buildpackage` +installed natively, you can use the CMake targets directly — no container +needed, but the host toolchain replaces the pinned CI image: ```bash cmake \ -DCMAKE_INSTALL_PREFIX=/opt/xrpld \ -Dxrpld=ON \ + -Dxrpld_version=2.4.0-local \ -Dtests=OFF \ .. -# RPM (in RHEL container): -cmake --build . --target package-rpm - -# DEB (in Debian/Ubuntu container): -cmake --build . --target package-deb +cmake --build . --target package-rpm # requires rpmbuild +cmake --build . --target package-deb # requires dpkg-buildpackage ``` -The `cmake/XrplPackaging.cmake` module gates each target on whether the required -tool (`rpmbuild` / `dpkg-buildpackage`) is present at configure time, so -configuring on a host that lacks one simply omits the corresponding target. - -`CMAKE_INSTALL_PREFIX` must be `/opt/xrpld`; if it is not, both targets are -skipped with a `STATUS` message. +The `cmake/XrplPackaging.cmake` module gates each target on whether the +required tool is present at configure time, so configuring on a host that +lacks one simply omits the corresponding target. `CMAKE_INSTALL_PREFIX` must +be `/opt/xrpld`; if it is not, both targets are skipped with a `STATUS` +message. ## How `build_pkg.sh` works @@ -110,9 +165,3 @@ export LC_ALL=C.UTF-8 export GZIP=-n export DEB_BUILD_OPTIONS="noautodbgsym reproducible=+fixfilepath" ``` - -## TODO - -- Port debsigs signing instructions and integrate into CI. -- Port RPM GPG signing setup (key import + `%{?_gpg_sign}` in spec). -- Introduce a virtual package for key rotation. diff --git a/package/deb/debian/rules b/package/deb/debian/rules index 3fd5e89f97..500966e704 100644 --- a/package/deb/debian/rules +++ b/package/deb/debian/rules @@ -10,14 +10,14 @@ override_dh_auto_configure override_dh_auto_build override_dh_auto_test: @: override_dh_auto_install: - install -Dm0755 xrpld debian/tmp/opt/xrpld/bin/xrpld - install -Dm0644 xrpld.cfg debian/tmp/opt/xrpld/etc/xrpld.cfg - install -Dm0644 validators.txt debian/tmp/opt/xrpld/etc/validators.txt - install -Dm0644 xrpld.logrotate debian/tmp/opt/xrpld/bin/xrpld.logrotate - install -Dm0755 update-xrpld.sh debian/tmp/opt/xrpld/bin/update-xrpld.sh - install -Dm0644 update-xrpld-cron debian/tmp/opt/xrpld/bin/update-xrpld-cron - install -Dm0644 README.md debian/tmp/usr/share/doc/xrpld/README.md - install -Dm0644 LICENSE.md debian/tmp/usr/share/doc/xrpld/LICENSE.md + install -Dm0755 xrpld debian/tmp/opt/xrpld/bin/xrpld + install -Dm0644 xrpld.cfg debian/tmp/opt/xrpld/etc/xrpld.cfg + install -Dm0644 validators.txt debian/tmp/opt/xrpld/etc/validators.txt + install -Dm0644 xrpld.logrotate debian/tmp/opt/xrpld/bin/xrpld.logrotate + install -Dm0755 update-xrpld.sh debian/tmp/opt/xrpld/bin/update-xrpld.sh + install -Dm0644 update-xrpld-cron debian/tmp/opt/xrpld/bin/update-xrpld-cron + install -Dm0644 README.md debian/tmp/usr/share/doc/xrpld/README.md + install -Dm0644 LICENSE.md debian/tmp/usr/share/doc/xrpld/LICENSE.md override_dh_installsystemd: dh_installsystemd