From 05a76895ad13da932043693cc313284dfe729cd5 Mon Sep 17 00:00:00 2001 From: Bart <11445373+bthomee@users.noreply.github.com> Date: Mon, 22 Dec 2025 16:52:45 -0500 Subject: [PATCH] ci: Support more flexible strategy matrix generation --- .github/scripts/strategy-matrix/README.md | 90 +++ .github/scripts/strategy-matrix/__init__.py | 0 .github/scripts/strategy-matrix/generate.py | 446 +++++------ .../scripts/strategy-matrix/generate_test.py | 468 +++++++++++ .../strategy-matrix/helpers/__init__.py | 0 .../scripts/strategy-matrix/helpers/defs.py | 190 +++++ .../strategy-matrix/helpers/defs_test.py | 743 ++++++++++++++++++ .../scripts/strategy-matrix/helpers/enums.py | 75 ++ .../scripts/strategy-matrix/helpers/funcs.py | 235 ++++++ .../strategy-matrix/helpers/funcs_test.py | 419 ++++++++++ .../scripts/strategy-matrix/helpers/unique.py | 30 + .../strategy-matrix/helpers/unique_test.py | 39 + .github/scripts/strategy-matrix/linux.json | 212 ----- .github/scripts/strategy-matrix/linux.py | 490 ++++++++++++ .github/scripts/strategy-matrix/macos.json | 22 - .github/scripts/strategy-matrix/macos.py | 9 + .github/scripts/strategy-matrix/windows.json | 19 - .github/scripts/strategy-matrix/windows.py | 9 + .github/workflows/on-pr.yml | 5 +- .github/workflows/on-trigger.yml | 7 +- .../workflows/reusable-build-test-config.yml | 30 +- .github/workflows/reusable-build-test.yml | 37 +- .../workflows/reusable-strategy-matrix.yml | 31 +- .github/workflows/upload-conan-deps.yml | 21 +- .gitignore | 1 + 25 files changed, 3056 insertions(+), 572 deletions(-) create mode 100644 .github/scripts/strategy-matrix/README.md create mode 100644 .github/scripts/strategy-matrix/__init__.py create mode 100644 .github/scripts/strategy-matrix/generate_test.py create mode 100644 .github/scripts/strategy-matrix/helpers/__init__.py create mode 100755 .github/scripts/strategy-matrix/helpers/defs.py create mode 100644 .github/scripts/strategy-matrix/helpers/defs_test.py create mode 100755 .github/scripts/strategy-matrix/helpers/enums.py create mode 100755 .github/scripts/strategy-matrix/helpers/funcs.py create mode 100644 .github/scripts/strategy-matrix/helpers/funcs_test.py create mode 100644 .github/scripts/strategy-matrix/helpers/unique.py create mode 100644 .github/scripts/strategy-matrix/helpers/unique_test.py delete mode 100644 .github/scripts/strategy-matrix/linux.json create mode 100755 .github/scripts/strategy-matrix/linux.py delete mode 100644 .github/scripts/strategy-matrix/macos.json create mode 100755 .github/scripts/strategy-matrix/macos.py delete mode 100644 .github/scripts/strategy-matrix/windows.json create mode 100755 .github/scripts/strategy-matrix/windows.py diff --git a/.github/scripts/strategy-matrix/README.md b/.github/scripts/strategy-matrix/README.md new file mode 100644 index 0000000000..ae521b98f4 --- /dev/null +++ b/.github/scripts/strategy-matrix/README.md @@ -0,0 +1,90 @@ +# Strategy Matrix + +The scripts in this directory will generate a strategy matrix for GitHub Actions +CI, depending on the trigger that caused the workflow to run and the platform +specified. + +There are several build, test, and publish settings that can be enabled for each +configuration. The settings are combined in a Cartesian product to generate the +full matrix, while filtering out any combinations not applicable to the trigger. + +## Platforms + +We support three platforms: Linux, macOS, and Windows. + +### Linux + +We support a variety of distributions (Debian, RHEL, and Ubuntu) and compilers +(GCC and Clang) on Linux. As there are so many combinations, we don't run them +all. Instead, we focus on a few key ones for PR commits and merges, while we run +most of them on a scheduled or ad hoc basis. + +Some noteworthy configurations are: + +- The official release build is GCC 14 on Debian Bullseye. + - Although we generally enable assertions in release builds, we disable them + for the official release build. + - We publish .deb and .rpm packages for this build, as well as a Docker image. + - For PR commits we also publish packages and images for testing purposes. +- Antithesis instrumentation is only supported on Clang 16+ on AMD64. + - We publish a Docker image for this build, but no packages. +- Coverage reports are generated on Bullseye with GCC 15. + - It must be enabled for both commits (to show PR coverage) and merges (to + show default branch coverage). + +Note that we try to run pipelines equally across both AMD64 and ARM64, but in +some cases we cannot build on ARM64: + +- All Clang 20+ builds on ARM64 are currently skipped due to a Boost build + error. +- All RHEL builds on AMD64 are currently skipped due to a build failure that + needs further investigation. + +Also note that to create a Docker image we ideally build on both AMD64 and +ARM64 to create a multi-arch image. Both configs should therefore be triggered +by the same event. However, as the script outputs individual configs, the +workflow must be able to run both builds separately and then merge the +single-arch images afterwards into a multi-arch image. + +### MacOS + +We support building on macOS, which uses the Apple Clang compiler and the ARM64 +architecture. We use default settings for all builds, and don't publish any +packages or images. + +### Windows + +We also support building on Windows, which uses the MSVC compiler and the AMD64 +architecture. While we could build on ARM64, we have not yet found a suitable +cloud machine to use as a GitHub runner. We use default settings for all builds, +and don't publish any packages or images. + +## Triggers + +We have four triggers that can cause the workflow to run: + +- `commit`: A commit is pushed to a pull request. +- `merge`: A pull request is merged. +- `label`: A label is added to a pull request. +- `schedule`: The workflow is run on a scheduled basis. + +The `label` trigger is currently not used, but it is reserved for future use. +The `schedule` trigger is used to run the workflow each weekday, and is also +used for ad hoc testing via the `workflow_dispatch` event. + +## Usage + +Our GitHub CI pipeline uses the `generate.py` script to generate the matrix for +the current workflow invocation. Naturally, the script can be run locally to +generate the matrix for testing purposes, e.g.: + +```bash +python3 generate.py --platform=linux --trigger=commit +``` + +If you want to pretty-print the output, you can pipe it to `jq` after stripping +off the `matrix=` prefix, e.g.: + +```bash +python3 generate.py --platform=linux --trigger=commit | cut -d= -f2- | jq +``` diff --git a/.github/scripts/strategy-matrix/__init__.py b/.github/scripts/strategy-matrix/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/scripts/strategy-matrix/generate.py b/.github/scripts/strategy-matrix/generate.py index 79530a1d75..47c38f6d0d 100755 --- a/.github/scripts/strategy-matrix/generate.py +++ b/.github/scripts/strategy-matrix/generate.py @@ -1,301 +1,211 @@ #!/usr/bin/env python3 import argparse +import dataclasses import itertools -import json -from dataclasses import dataclass -from pathlib import Path +from collections.abc import Iterator -THIS_DIR = Path(__file__).parent.resolve() +import linux +import macos +import windows +from helpers.defs import * +from helpers.enums import * +from helpers.funcs import * +from helpers.unique import * + +# The GitHub runner tags to use for the different architectures. +RUNNER_TAGS = { + Arch.LINUX_AMD64: ["self-hosted", "Linux", "X64", "heavy"], + Arch.LINUX_ARM64: ["self-hosted", "Linux", "ARM64", "heavy-arm64"], + Arch.MACOS_ARM64: ["self-hosted", "macOS", "ARM64", "mac-runner-m1"], + Arch.WINDOWS_AMD64: ["self-hosted", "Windows", "devbox"], +} -@dataclass -class Config: - architecture: list[dict] - os: list[dict] - build_type: list[str] - cmake_args: list[str] +def generate_configs(distros: list[Distro], trigger: Trigger) -> list[Config]: + """Generate a strategy matrix for GitHub Actions CI. + + Args: + distros: The distros to generate the matrix for. + trigger: The trigger that caused the workflow to run. + + Returns: + list[Config]: The generated configurations. + + Raises: + ValueError: If any of the required fields are empty or invalid. + TypeError: If any of the required fields are of the wrong type. + + """ + + configs = [] + for distro in distros: + for config in generate_config_for_distro(distro, trigger): + configs.append(config) + + if not is_unique(configs): + raise ValueError("configs must be a list of unique Config") + + return configs -""" -Generate a strategy matrix for GitHub Actions CI. +def generate_config_for_distro(distro: Distro, trigger: Trigger) -> Iterator[Config]: + """Generate a strategy matrix for a specific distro. -On each PR commit we will build a selection of Debian, RHEL, Ubuntu, MacOS, and -Windows configurations, while upon merge into the develop, release, or master -branches, we will build all configurations, and test most of them. + Args: + distro: The distro to generate the matrix for. + trigger: The trigger that caused the workflow to run. -We will further set additional CMake arguments as follows: -- All builds will have the `tests`, `werr`, and `xrpld` options. -- All builds will have the `wextra` option except for GCC 12 and Clang 16. -- All release builds will have the `assert` option. -- Certain Debian Bookworm configurations will change the reference fee, enable - codecov, and enable voidstar in PRs. -""" + Yields: + Config: The next configuration to build. + Raises: + ValueError: If any of the required fields are empty or invalid. + TypeError: If any of the required fields are of the wrong type. -def generate_strategy_matrix(all: bool, config: Config) -> list: - configurations = [] - for architecture, os, build_type, cmake_args in itertools.product( - config.architecture, config.os, config.build_type, config.cmake_args - ): - # The default CMake target is 'all' for Linux and MacOS and 'install' - # for Windows, but it can get overridden for certain configurations. - cmake_target = "install" if os["distro_name"] == "windows" else "all" - - # We build and test all configurations by default, except for Windows in - # Debug, because it is too slow, as well as when code coverage is - # enabled as that mode already runs the tests. - build_only = False - if os["distro_name"] == "windows" and build_type == "Debug": - build_only = True - - # Only generate a subset of configurations in PRs. - if not all: - # Debian: - # - Bookworm using GCC 13: Release and Unity on linux/amd64, set - # the reference fee to 500. - # - Bookworm using GCC 15: Debug and no Unity on linux/amd64, enable - # code coverage (which will be done below). - # - Bookworm using Clang 16: Debug and no Unity on linux/arm64, - # enable voidstar. - # - Bookworm using Clang 17: Release and no Unity on linux/amd64, - # set the reference fee to 1000. - # - Bookworm using Clang 20: Debug and Unity on linux/amd64. - if os["distro_name"] == "debian": - skip = True - if os["distro_version"] == "bookworm": - if ( - f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-13" - and build_type == "Release" - and "-Dunity=ON" in cmake_args - and architecture["platform"] == "linux/amd64" - ): - cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=500 {cmake_args}" - skip = False - if ( - f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15" - and build_type == "Debug" - and "-Dunity=OFF" in cmake_args - and architecture["platform"] == "linux/amd64" - ): - skip = False - if ( - f"{os['compiler_name']}-{os['compiler_version']}" == "clang-16" - and build_type == "Debug" - and "-Dunity=OFF" in cmake_args - and architecture["platform"] == "linux/arm64" - ): - cmake_args = f"-Dvoidstar=ON {cmake_args}" - skip = False - if ( - f"{os['compiler_name']}-{os['compiler_version']}" == "clang-17" - and build_type == "Release" - and "-Dunity=ON" in cmake_args - and architecture["platform"] == "linux/amd64" - ): - cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=1000 {cmake_args}" - skip = False - if ( - f"{os['compiler_name']}-{os['compiler_version']}" == "clang-20" - and build_type == "Debug" - and "-Dunity=ON" in cmake_args - and architecture["platform"] == "linux/amd64" - ): - skip = False - if skip: - continue - - # RHEL: - # - 9 using GCC 12: Debug and Unity on linux/amd64. - # - 10 using Clang: Release and no Unity on linux/amd64. - if os["distro_name"] == "rhel": - skip = True - if os["distro_version"] == "9": - if ( - f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12" - and build_type == "Debug" - and "-Dunity=ON" in cmake_args - and architecture["platform"] == "linux/amd64" - ): - skip = False - elif os["distro_version"] == "10": - if ( - f"{os['compiler_name']}-{os['compiler_version']}" == "clang-any" - and build_type == "Release" - and "-Dunity=OFF" in cmake_args - and architecture["platform"] == "linux/amd64" - ): - skip = False - if skip: - continue - - # Ubuntu: - # - Jammy using GCC 12: Debug and no Unity on linux/arm64. - # - Noble using GCC 14: Release and Unity on linux/amd64. - # - Noble using Clang 18: Debug and no Unity on linux/amd64. - # - Noble using Clang 19: Release and Unity on linux/arm64. - if os["distro_name"] == "ubuntu": - skip = True - if os["distro_version"] == "jammy": - if ( - f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12" - and build_type == "Debug" - and "-Dunity=OFF" in cmake_args - and architecture["platform"] == "linux/arm64" - ): - skip = False - elif os["distro_version"] == "noble": - if ( - f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-14" - and build_type == "Release" - and "-Dunity=ON" in cmake_args - and architecture["platform"] == "linux/amd64" - ): - skip = False - if ( - f"{os['compiler_name']}-{os['compiler_version']}" == "clang-18" - and build_type == "Debug" - and "-Dunity=OFF" in cmake_args - and architecture["platform"] == "linux/amd64" - ): - skip = False - if ( - f"{os['compiler_name']}-{os['compiler_version']}" == "clang-19" - and build_type == "Release" - and "-Dunity=ON" in cmake_args - and architecture["platform"] == "linux/arm64" - ): - skip = False - if skip: - continue - - # MacOS: - # - Debug and no Unity on macos/arm64. - if os["distro_name"] == "macos" and not ( - build_type == "Debug" - and "-Dunity=OFF" in cmake_args - and architecture["platform"] == "macos/arm64" - ): - continue - - # Windows: - # - Release and Unity on windows/amd64. - if os["distro_name"] == "windows" and not ( - build_type == "Release" - and "-Dunity=ON" in cmake_args - and architecture["platform"] == "windows/amd64" - ): - continue - - # Additional CMake arguments. - cmake_args = f"{cmake_args} -Dtests=ON -Dwerr=ON -Dxrpld=ON" - if not f"{os['compiler_name']}-{os['compiler_version']}" in [ - "gcc-12", - "clang-16", - ]: - cmake_args = f"{cmake_args} -Dwextra=ON" - if build_type == "Release": - cmake_args = f"{cmake_args} -Dassert=ON" - - # We skip all RHEL on arm64 due to a build failure that needs further - # investigation. - if os["distro_name"] == "rhel" and architecture["platform"] == "linux/arm64": + """ + for spec in distro.specs: + if trigger not in spec.triggers: continue - # We skip all clang 20+ on arm64 due to Boost build error. - if ( - f"{os['compiler_name']}-{os['compiler_version']}" - in ["clang-20", "clang-21"] - and architecture["platform"] == "linux/arm64" - ): + os_name = distro.os_name + os_version = distro.os_version + compiler_name = distro.compiler_name + compiler_version = distro.compiler_version + image_sha = distro.image_sha + yield from generate_config_for_distro_spec( + os_name, + os_version, + compiler_name, + compiler_version, + image_sha, + spec, + trigger, + ) + + +def generate_config_for_distro_spec( + os_name: str, + os_version: str, + compiler_name: str, + compiler_version: str, + image_sha: str, + spec: Spec, + trigger: Trigger, +) -> Iterator[Config]: + """Generate a strategy matrix for a specific distro and spec. + + Args: + os_name: The OS name. + os_version: The OS version. + compiler_name: The compiler name. + compiler_version: The compiler version. + image_sha: The image SHA. + spec: The spec to generate the matrix for. + trigger: The trigger that caused the workflow to run. + + Yields: + Config: The next configuration to build. + + """ + + for trigger_, arch, build_mode, build_type in itertools.product( + spec.triggers, spec.archs, spec.build_modes, spec.build_types + ): + if trigger_ != trigger: continue - # Enable code coverage for Debian Bookworm using GCC 15 in Debug and no - # Unity on linux/amd64 - if ( - f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15" - and build_type == "Debug" - and "-Dunity=OFF" in cmake_args - and architecture["platform"] == "linux/amd64" - ): - cmake_args = f"-Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0 {cmake_args}" + build_option = spec.build_option + test_option = spec.test_option + publish_option = spec.publish_option - # Generate a unique name for the configuration, e.g. macos-arm64-debug - # or debian-bookworm-gcc-12-amd64-release-unity. - config_name = os["distro_name"] - if (n := os["distro_version"]) != "": - config_name += f"-{n}" - if (n := os["compiler_name"]) != "": - config_name += f"-{n}" - if (n := os["compiler_version"]) != "": - config_name += f"-{n}" - config_name += ( - f"-{architecture['platform'][architecture['platform'].find('/') + 1 :]}" - ) - config_name += f"-{build_type.lower()}" - if "-Dunity=ON" in cmake_args: - config_name += "-unity" - - # Add the configuration to the list, with the most unique fields first, - # so that they are easier to identify in the GitHub Actions UI, as long - # names get truncated. - configurations.append( - { - "config_name": config_name, - "cmake_args": cmake_args, - "cmake_target": cmake_target, - "build_only": build_only, - "build_type": build_type, - "os": os, - "architecture": architecture, - } + # Determine the configuration name. + config_name = generate_config_name( + os_name, + os_version, + compiler_name, + compiler_version, + arch, + build_type, + build_mode, + build_option, ) - return configurations + # Determine the CMake arguments. + cmake_args = generate_cmake_args( + compiler_name, + compiler_version, + build_type, + build_mode, + build_option, + test_option, + ) + # Determine the CMake target. + cmake_target = generate_cmake_target(os_name, build_type) -def read_config(file: Path) -> Config: - config = json.loads(file.read_text()) - if ( - config["architecture"] is None - or config["os"] is None - or config["build_type"] is None - or config["cmake_args"] is None - ): - raise Exception("Invalid configuration file.") + # Determine whether to enable running tests, and to create a package + # and/or image. + enable_tests, enable_package, enable_image = generate_enable_options( + os_name, build_type, publish_option + ) - return Config(**config) + # Determine the image to run in, if applicable. + image = generate_image_name( + os_name, + os_version, + compiler_name, + compiler_version, + image_sha, + ) + + # Generate the configuration. + yield Config( + config_name=config_name, + cmake_args=cmake_args, + cmake_target=cmake_target, + build_type=("Debug" if build_type == BuildType.DEBUG else "Release"), + enable_tests=enable_tests, + enable_package=enable_package, + enable_image=enable_image, + runs_on=RUNNER_TAGS[arch], + image=image, + ) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( - "-a", - "--all", - help="Set to generate all configurations (generally used when merging a PR) or leave unset to generate a subset of configurations (generally used when committing to a PR).", - action="store_true", + "--platform", + "-p", + required=False, + type=Platform, + choices=list(Platform), + help="The platform to run on.", ) parser.add_argument( - "-c", - "--config", - help="Path to the JSON file containing the strategy matrix configurations.", - required=False, - type=Path, + "--trigger", + "-t", + required=True, + type=Trigger, + choices=list(Trigger), + help="The trigger that caused the workflow to run.", ) args = parser.parse_args() - matrix = [] - if args.config is None or args.config == "": - matrix += generate_strategy_matrix( - args.all, read_config(THIS_DIR / "linux.json") - ) - matrix += generate_strategy_matrix( - args.all, read_config(THIS_DIR / "macos.json") - ) - matrix += generate_strategy_matrix( - args.all, read_config(THIS_DIR / "windows.json") - ) - else: - matrix += generate_strategy_matrix(args.all, read_config(args.config)) + # Collect the distros to generate configs for. + distros = [] + if args.platform in [None, Platform.LINUX]: + distros += linux.DEBIAN_DISTROS + linux.RHEL_DISTROS + linux.UBUNTU_DISTROS + if args.platform in [None, Platform.MACOS]: + distros += macos.DISTROS + if args.platform in [None, Platform.WINDOWS]: + distros += windows.DISTROS - # Generate the strategy matrix. - print(f"matrix={json.dumps({'include': matrix})}") + # Generate the configs. + configs = generate_configs(distros, args.trigger) + + # Convert the configs into the format expected by GitHub Actions. + include = [] + for config in configs: + include.append(dataclasses.asdict(config)) + print(f"matrix={json.dumps({'include': include})}") diff --git a/.github/scripts/strategy-matrix/generate_test.py b/.github/scripts/strategy-matrix/generate_test.py new file mode 100644 index 0000000000..7746c41816 --- /dev/null +++ b/.github/scripts/strategy-matrix/generate_test.py @@ -0,0 +1,468 @@ +import pytest + +from generate import * + + +@pytest.fixture +def macos_distro(): + return Distro( + os_name="macos", + specs=[ + Spec( + archs=[Arch.MACOS_ARM64], + build_modes=[BuildMode.UNITY_OFF], + build_option=BuildOption.COVERAGE, + build_types=[BuildType.RELEASE], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + ], + ) + + +@pytest.fixture +def windows_distro(): + return Distro( + os_name="windows", + specs=[ + Spec( + archs=[Arch.WINDOWS_AMD64], + build_modes=[BuildMode.UNITY_ON], + build_option=BuildOption.SANITIZE_ASAN, + build_types=[BuildType.DEBUG], + publish_option=PublishOption.IMAGE_ONLY, + test_option=TestOption.REFERENCE_FEE_500, + triggers=[Trigger.COMMIT, Trigger.SCHEDULE], + ) + ], + ) + + +@pytest.fixture +def linux_distro(): + return Distro( + os_name="debian", + os_version="bookworm", + compiler_name="clang", + compiler_version="16", + image_sha="a1b2c3d4", + specs=[ + Spec( + archs=[Arch.LINUX_AMD64], + build_modes=[BuildMode.UNITY_OFF], + build_option=BuildOption.SANITIZE_TSAN, + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.LABEL], + ), + Spec( + archs=[Arch.LINUX_AMD64, Arch.LINUX_ARM64], + build_modes=[BuildMode.UNITY_OFF, BuildMode.UNITY_ON], + build_option=BuildOption.VOIDSTAR, + build_types=[BuildType.PUBLISH], + publish_option=PublishOption.PACKAGE_AND_IMAGE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT, Trigger.LABEL], + ), + ], + ) + + +def test_macos_generate_config_for_distro_spec_matches_trigger(macos_distro): + trigger = Trigger.COMMIT + + distro = macos_distro + result = list( + generate_config_for_distro_spec( + distro.os_name, + distro.os_version, + distro.compiler_name, + distro.compiler_version, + distro.image_sha, + distro.specs[0], + trigger, + ) + ) + assert result == [ + Config( + config_name="macos-coverage-release-arm64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -Dassert=ON -Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0", + cmake_target="all", + build_type="Release", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["self-hosted", "macOS", "ARM64", "mac-runner-m1"], + image=None, + ) + ] + + +def test_macos_generate_config_for_distro_spec_no_match_trigger(macos_distro): + trigger = Trigger.MERGE + + distro = macos_distro + result = list( + generate_config_for_distro_spec( + distro.os_name, + distro.os_version, + distro.compiler_name, + distro.compiler_version, + distro.image_sha, + distro.specs[0], + trigger, + ) + ) + assert result == [] + + +def test_macos_generate_config_for_distro_matches_trigger(macos_distro): + trigger = Trigger.COMMIT + + distro = macos_distro + result = list(generate_config_for_distro(distro, trigger)) + assert result == [ + Config( + config_name="macos-coverage-release-arm64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -Dassert=ON -Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0", + cmake_target="all", + build_type="Release", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["self-hosted", "macOS", "ARM64", "mac-runner-m1"], + image=None, + ) + ] + + +def test_macos_generate_config_for_distro_no_match_trigger(macos_distro): + trigger = Trigger.MERGE + + distro = macos_distro + result = list(generate_config_for_distro(distro, trigger)) + assert result == [] + + +def test_windows_generate_config_for_distro_spec_matches_trigger( + windows_distro, +): + trigger = Trigger.COMMIT + + distro = windows_distro + result = list( + generate_config_for_distro_spec( + distro.os_name, + distro.os_version, + distro.compiler_name, + distro.compiler_version, + distro.image_sha, + distro.specs[0], + trigger, + ) + ) + assert result == [ + Config( + config_name="windows-asan-debug-unity-amd64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -Dunity=ON -DUNIT_TEST_REFERENCE_FEE=500", + cmake_target="install", + build_type="Debug", + enable_tests=False, + enable_package=False, + enable_image=True, + runs_on=["self-hosted", "Windows", "devbox"], + image=None, + ) + ] + + +def test_windows_generate_config_for_distro_spec_no_match_trigger( + windows_distro, +): + trigger = Trigger.MERGE + + distro = windows_distro + result = list( + generate_config_for_distro_spec( + distro.os_name, + distro.os_version, + distro.compiler_name, + distro.compiler_version, + distro.image_sha, + distro.specs[0], + trigger, + ) + ) + assert result == [] + + +def test_windows_generate_config_for_distro_matches_trigger( + windows_distro, +): + trigger = Trigger.COMMIT + + distro = windows_distro + result = list(generate_config_for_distro(distro, trigger)) + assert result == [ + Config( + config_name="windows-asan-debug-unity-amd64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -Dunity=ON -DUNIT_TEST_REFERENCE_FEE=500", + cmake_target="install", + build_type="Debug", + enable_tests=False, + enable_package=False, + enable_image=True, + runs_on=["self-hosted", "Windows", "devbox"], + image=None, + ) + ] + + +def test_windows_generate_config_for_distro_no_match_trigger( + windows_distro, +): + trigger = Trigger.MERGE + + distro = windows_distro + result = list(generate_config_for_distro(distro, trigger)) + assert result == [] + + +def test_linux_generate_config_for_distro_spec_matches_trigger(linux_distro): + trigger = Trigger.LABEL + + distro = linux_distro + result = list( + generate_config_for_distro_spec( + distro.os_name, + distro.os_version, + distro.compiler_name, + distro.compiler_version, + distro.image_sha, + distro.specs[1], + trigger, + ) + ) + assert result == [ + Config( + config_name="debian-bookworm-clang-16-voidstar-publish-amd64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dvoidstar=ON", + cmake_target="install", + build_type="Release", + enable_tests=True, + enable_package=True, + enable_image=True, + runs_on=["self-hosted", "Linux", "X64", "heavy"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + Config( + config_name="debian-bookworm-clang-16-voidstar-publish-unity-amd64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dunity=ON -Dvoidstar=ON", + cmake_target="install", + build_type="Release", + enable_tests=True, + enable_package=True, + enable_image=True, + runs_on=["self-hosted", "Linux", "X64", "heavy"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + Config( + config_name="debian-bookworm-clang-16-voidstar-publish-arm64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dvoidstar=ON", + cmake_target="install", + build_type="Release", + enable_tests=True, + enable_package=True, + enable_image=True, + runs_on=["self-hosted", "Linux", "ARM64", "heavy-arm64"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + Config( + config_name="debian-bookworm-clang-16-voidstar-publish-unity-arm64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dunity=ON -Dvoidstar=ON", + cmake_target="install", + build_type="Release", + enable_tests=True, + enable_package=True, + enable_image=True, + runs_on=["self-hosted", "Linux", "ARM64", "heavy-arm64"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + ] + + +def test_linux_generate_config_for_distro_spec_no_match_trigger(linux_distro): + trigger = Trigger.MERGE + + distro = linux_distro + result = list( + generate_config_for_distro_spec( + distro.os_name, + distro.os_version, + distro.compiler_name, + distro.compiler_version, + distro.image_sha, + distro.specs[1], + trigger, + ) + ) + assert result == [] + + +def test_linux_generate_config_for_distro_matches_trigger(linux_distro): + trigger = Trigger.LABEL + + distro = linux_distro + result = list(generate_config_for_distro(distro, trigger)) + assert result == [ + Config( + config_name="debian-bookworm-clang-16-tsan-debug-amd64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["self-hosted", "Linux", "X64", "heavy"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + Config( + config_name="debian-bookworm-clang-16-voidstar-publish-amd64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dvoidstar=ON", + cmake_target="install", + build_type="Release", + enable_tests=True, + enable_package=True, + enable_image=True, + runs_on=["self-hosted", "Linux", "X64", "heavy"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + Config( + config_name="debian-bookworm-clang-16-voidstar-publish-unity-amd64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dunity=ON -Dvoidstar=ON", + cmake_target="install", + build_type="Release", + enable_tests=True, + enable_package=True, + enable_image=True, + runs_on=["self-hosted", "Linux", "X64", "heavy"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + Config( + config_name="debian-bookworm-clang-16-voidstar-publish-arm64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dvoidstar=ON", + cmake_target="install", + build_type="Release", + enable_tests=True, + enable_package=True, + enable_image=True, + runs_on=["self-hosted", "Linux", "ARM64", "heavy-arm64"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + Config( + config_name="debian-bookworm-clang-16-voidstar-publish-unity-arm64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dunity=ON -Dvoidstar=ON", + cmake_target="install", + build_type="Release", + enable_tests=True, + enable_package=True, + enable_image=True, + runs_on=["self-hosted", "Linux", "ARM64", "heavy-arm64"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + ] + + +def test_linux_generate_config_for_distro_no_match_trigger(linux_distro): + trigger = Trigger.MERGE + + distro = linux_distro + result = list(generate_config_for_distro(distro, trigger)) + assert result == [] + + +def test_generate_configs(macos_distro, windows_distro, linux_distro): + trigger = Trigger.COMMIT + + distros = [macos_distro, windows_distro, linux_distro] + result = generate_configs(distros, trigger) + assert result == [ + Config( + config_name="macos-coverage-release-arm64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -Dassert=ON -Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0", + cmake_target="all", + build_type="Release", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["self-hosted", "macOS", "ARM64", "mac-runner-m1"], + image=None, + ), + Config( + config_name="windows-asan-debug-unity-amd64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -Dunity=ON -DUNIT_TEST_REFERENCE_FEE=500", + cmake_target="install", + build_type="Debug", + enable_tests=False, + enable_package=False, + enable_image=True, + runs_on=["self-hosted", "Windows", "devbox"], + image=None, + ), + Config( + config_name="debian-bookworm-clang-16-voidstar-publish-amd64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dvoidstar=ON", + cmake_target="install", + build_type="Release", + enable_tests=True, + enable_package=True, + enable_image=True, + runs_on=["self-hosted", "Linux", "X64", "heavy"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + Config( + config_name="debian-bookworm-clang-16-voidstar-publish-unity-amd64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dunity=ON -Dvoidstar=ON", + cmake_target="install", + build_type="Release", + enable_tests=True, + enable_package=True, + enable_image=True, + runs_on=["self-hosted", "Linux", "X64", "heavy"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + Config( + config_name="debian-bookworm-clang-16-voidstar-publish-arm64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dvoidstar=ON", + cmake_target="install", + build_type="Release", + enable_tests=True, + enable_package=True, + enable_image=True, + runs_on=["self-hosted", "Linux", "ARM64", "heavy-arm64"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + Config( + config_name="debian-bookworm-clang-16-voidstar-publish-unity-arm64", + cmake_args="-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dunity=ON -Dvoidstar=ON", + cmake_target="install", + build_type="Release", + enable_tests=True, + enable_package=True, + enable_image=True, + runs_on=["self-hosted", "Linux", "ARM64", "heavy-arm64"], + image="ghcr.io/xrplf/ci/debian-bookworm:clang-16-a1b2c3d4", + ), + ] + + +def test_generate_configs_raises_on_duplicate_configs( + macos_distro, windows_distro, linux_distro +): + trigger = Trigger.COMMIT + + distros = [macos_distro, macos_distro] + with pytest.raises(ValueError): + generate_configs(distros, trigger) diff --git a/.github/scripts/strategy-matrix/helpers/__init__.py b/.github/scripts/strategy-matrix/helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/scripts/strategy-matrix/helpers/defs.py b/.github/scripts/strategy-matrix/helpers/defs.py new file mode 100755 index 0000000000..63a7e6f96b --- /dev/null +++ b/.github/scripts/strategy-matrix/helpers/defs.py @@ -0,0 +1,190 @@ +from dataclasses import dataclass, field + +from helpers.enums import * +from helpers.unique import * + + +@dataclass +class Config: + """Represents a configuration to include in the strategy matrix. + + Raises: + ValueError: If any of the required fields are empty or invalid. + TypeError: If any of the required fields are of the wrong type. + """ + + config_name: str + cmake_args: str + cmake_target: str + build_type: str + enable_tests: bool + enable_package: bool + enable_image: bool + runs_on: list[str] + image: str | None = None + + def __post_init__(self): + if not self.config_name: + raise ValueError("config_name cannot be empty") + if not isinstance(self.config_name, str): + raise TypeError("config_name must be a string") + + if not self.cmake_args: + raise ValueError("cmake_args cannot be empty") + if not isinstance(self.cmake_args, str): + raise TypeError("cmake_args must be a string") + + if not self.cmake_target: + raise ValueError("cmake_target cannot be empty") + if not isinstance(self.cmake_target, str): + raise TypeError("cmake_target must be a string") + if self.cmake_target not in ["all", "install"]: + raise ValueError("cmake_target must be 'all' or 'install'") + + if not self.build_type: + raise ValueError("build_type cannot be empty") + if not isinstance(self.build_type, str): + raise TypeError("build_type must be a string") + if self.build_type not in ["Debug", "Release"]: + raise ValueError("build_type must be 'Debug' or 'Release'") + + if not isinstance(self.enable_tests, bool): + raise TypeError("enable_tests must be a boolean") + if not isinstance(self.enable_package, bool): + raise TypeError("enable_package must be a boolean") + if not isinstance(self.enable_image, bool): + raise TypeError("enable_image must be a boolean") + + if not self.runs_on: + raise ValueError("runs_on cannot be empty") + if not isinstance(self.runs_on, list): + raise TypeError("runs_on must be a list") + if not all(isinstance(runner, str) for runner in self.runs_on): + raise TypeError("runs_on must be a list of strings") + if not all(self.runs_on): + raise ValueError("runs_on must be a list of non-empty strings") + if len(self.runs_on) != len(set(self.runs_on)): + raise ValueError("runs_on must be a list of unique strings") + + if self.image and not isinstance(self.image, str): + raise TypeError("image must be a string") + + +@dataclass +class Spec: + """Represents a specification used by a configuration. + + Raises: + ValueError: If any of the required fields are empty. + TypeError: If any of the required fields are of the wrong type. + """ + + archs: list[Arch] = field( + default_factory=lambda: [Arch.LINUX_AMD64, Arch.LINUX_ARM64] + ) + build_option: BuildOption = BuildOption.NONE + build_modes: list[BuildMode] = field( + default_factory=lambda: [BuildMode.UNITY_OFF, BuildMode.UNITY_ON] + ) + build_types: list[BuildType] = field( + default_factory=lambda: [BuildType.DEBUG, BuildType.RELEASE] + ) + publish_option: PublishOption = PublishOption.NONE + test_option: TestOption = TestOption.NONE + triggers: list[Trigger] = field( + default_factory=lambda: [Trigger.COMMIT, Trigger.MERGE, Trigger.SCHEDULE] + ) + + def __post_init__(self): + if not self.archs: + raise ValueError("archs cannot be empty") + if not isinstance(self.archs, list): + raise TypeError("archs must be a list") + if not all(isinstance(arch, str) for arch in self.archs): + raise TypeError("archs must be a list of Arch") + if len(self.archs) != len(set(self.archs)): + raise ValueError("archs must be a list of unique Arch") + + if not isinstance(self.build_option, BuildOption): + raise TypeError("build_option must be a BuildOption") + + if not self.build_modes: + raise ValueError("build_modes cannot be empty") + if not isinstance(self.build_modes, list): + raise TypeError("build_modes must be a list") + if not all( + isinstance(build_mode, BuildMode) for build_mode in self.build_modes + ): + raise TypeError("build_modes must be a list of BuildMode") + if len(self.build_modes) != len(set(self.build_modes)): + raise ValueError("build_modes must be a list of unique BuildMode") + + if not self.build_types: + raise ValueError("build_types cannot be empty") + if not isinstance(self.build_types, list): + raise TypeError("build_types must be a list") + if not all( + isinstance(build_type, BuildType) for build_type in self.build_types + ): + raise TypeError("build_types must be a list of BuildType") + if len(self.build_types) != len(set(self.build_types)): + raise ValueError("build_types must be a list of unique BuildType") + + if not isinstance(self.publish_option, PublishOption): + raise TypeError("publish_option must be a PublishOption") + + if not isinstance(self.test_option, TestOption): + raise TypeError("test_option must be a TestOption") + + if not self.triggers: + raise ValueError("triggers cannot be empty") + if not isinstance(self.triggers, list): + raise TypeError("triggers must be a list") + if not all(isinstance(trigger, Trigger) for trigger in self.triggers): + raise TypeError("triggers must be a list of Trigger") + if len(self.triggers) != len(set(self.triggers)): + raise ValueError("triggers must be a list of unique Trigger") + + +@dataclass +class Distro: + """Represents a Linux, Windows or macOS distribution with specifications. + + Raises: + ValueError: If any of the required fields are empty. + TypeError: If any of the required fields are of the wrong type. + """ + + os_name: str + os_version: str = "" + compiler_name: str = "" + compiler_version: str = "" + image_sha: str = "" + specs: list[Spec] = field(default_factory=list) + + def __post_init__(self): + if not self.os_name: + raise ValueError("os_name cannot be empty") + if not isinstance(self.os_name, str): + raise TypeError("os_name must be a string") + + if self.os_version and not isinstance(self.os_version, str): + raise TypeError("os_version must be a string") + + if self.compiler_name and not isinstance(self.compiler_name, str): + raise TypeError("compiler_name must be a string") + + if self.compiler_version and not isinstance(self.compiler_version, str): + raise TypeError("compiler_version must be a string") + + if self.image_sha and not isinstance(self.image_sha, str): + raise TypeError("image_sha must be a string") + + if not self.specs: + raise ValueError("specs cannot be empty") + if not isinstance(self.specs, list): + raise TypeError("specs must be a list") + if not all(isinstance(spec, Spec) for spec in self.specs): + raise TypeError("specs must be a list of Spec") + if not is_unique(self.specs): + raise ValueError("specs must be a list of unique Spec") diff --git a/.github/scripts/strategy-matrix/helpers/defs_test.py b/.github/scripts/strategy-matrix/helpers/defs_test.py new file mode 100644 index 0000000000..922d2fdc5c --- /dev/null +++ b/.github/scripts/strategy-matrix/helpers/defs_test.py @@ -0,0 +1,743 @@ +import pytest + +from helpers.defs import * +from helpers.enums import * +from helpers.funcs import * + + +def test_config_valid_none_image(): + assert Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label"], + image=None, + ) + + +def test_config_valid_empty_image(): + assert Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="install", + build_type="Debug", + enable_tests=False, + enable_package=True, + enable_image=False, + runs_on=["label"], + image="", + ) + + +def test_config_valid_with_image(): + assert Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="install", + build_type="Release", + enable_tests=False, + enable_package=True, + enable_image=True, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_empty_config_name(): + with pytest.raises(ValueError): + Config( + config_name="", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_wrong_config_name(): + with pytest.raises(TypeError): + Config( + config_name=123, + cmake_args="-Doption=ON", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_empty_cmake_args(): + with pytest.raises(ValueError): + Config( + config_name="config", + cmake_args="", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_wrong_cmake_args(): + with pytest.raises(TypeError): + Config( + config_name="config", + cmake_args=123, + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_empty_cmake_target(): + with pytest.raises(ValueError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_invalid_cmake_target(): + with pytest.raises(ValueError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="invalid", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_wrong_cmake_target(): + with pytest.raises(TypeError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target=123, + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_empty_build_type(): + with pytest.raises(ValueError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_invalid_build_type(): + with pytest.raises(ValueError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="invalid", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_wrong_build_type(): + with pytest.raises(TypeError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type=123, + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_wrong_enable_tests(): + with pytest.raises(TypeError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="Debug", + enable_tests=123, + enable_package=False, + enable_image=False, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_wrong_enable_package(): + with pytest.raises(TypeError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=123, + enable_image=False, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_wrong_enable_image(): + with pytest.raises(TypeError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=True, + enable_image=123, + runs_on=["label"], + image="image", + ) + + +def test_config_raises_on_none_runs_on(): + with pytest.raises(ValueError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=None, + image="image", + ) + + +def test_config_raises_on_empty_runs_on(): + with pytest.raises(ValueError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=[], + image="image", + ) + + +def test_config_raises_on_invalid_runs_on(): + with pytest.raises(ValueError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=[""], + image="image", + ) + + +def test_config_raises_on_wrong_runs_on(): + with pytest.raises(TypeError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=[123], + image="image", + ) + + +def test_config_raises_on_duplicate_runs_on(): + with pytest.raises(ValueError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label", "label"], + image="image", + ) + + +def test_config_raises_on_wrong_image(): + with pytest.raises(TypeError): + Config( + config_name="config", + cmake_args="-Doption=ON", + cmake_target="all", + build_type="Debug", + enable_tests=True, + enable_package=False, + enable_image=False, + runs_on=["label"], + image=123, + ) + + +def test_spec_valid(): + assert Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_none_archs(): + with pytest.raises(ValueError): + Spec( + archs=None, + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_empty_archs(): + with pytest.raises(ValueError): + Spec( + archs=[], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_wrong_archs(): + with pytest.raises(TypeError): + Spec( + archs=[123], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_duplicate_archs(): + with pytest.raises(ValueError): + Spec( + archs=[Arch.LINUX_AMD64, Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_wrong_build_option(): + with pytest.raises(TypeError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=123, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_none_build_modes(): + with pytest.raises(ValueError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=None, + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_empty_build_modes(): + with pytest.raises(ValueError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_wrong_build_modes(): + with pytest.raises(TypeError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[123], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_none_build_types(): + with pytest.raises(ValueError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=None, + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_empty_build_types(): + with pytest.raises(ValueError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_wrong_build_types(): + with pytest.raises(TypeError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[123], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_duplicate_build_types(): + with pytest.raises(ValueError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG, BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_wrong_publish_option(): + with pytest.raises(TypeError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=123, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_wrong_test_option(): + with pytest.raises(TypeError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=123, + triggers=[Trigger.COMMIT], + ) + + +def test_spec_raises_on_none_triggers(): + with pytest.raises(ValueError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=None, + ) + + +def test_spec_raises_on_empty_triggers(): + with pytest.raises(ValueError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[], + ) + + +def test_spec_raises_on_wrong_triggers(): + with pytest.raises(TypeError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[123], + ) + + +def test_spec_raises_on_duplicate_triggers(): + with pytest.raises(ValueError): + Spec( + archs=[Arch.LINUX_AMD64], + build_option=BuildOption.NONE, + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.NONE, + test_option=TestOption.NONE, + triggers=[Trigger.COMMIT, Trigger.COMMIT], + ) + + +def test_distro_valid_none_image_sha(): + assert Distro( + os_name="os_name", + os_version="os_version", + compiler_name="compiler_name", + compiler_version="compiler_version", + image_sha=None, + specs=[Spec()], # This is valid due to the default values. + ) + + +def test_distro_valid_empty_os_compiler_image_sha(): + assert Distro( + os_name="os_name", + os_version="", + compiler_name="", + compiler_version="", + image_sha="", + specs=[Spec()], + ) + + +def test_distro_valid_with_image(): + assert Distro( + os_name="os_name", + os_version="os_version", + compiler_name="compiler_name", + compiler_version="compiler_version", + image_sha="image_sha", + specs=[Spec()], + ) + + +def test_distro_raises_on_empty_os_name(): + with pytest.raises(ValueError): + Distro( + os_name="", + os_version="os_version", + compiler_name="compiler_name", + compiler_version="compiler_version", + image_sha="image_sha", + specs=[Spec()], + ) + + +def test_distro_raises_on_wrong_os_name(): + with pytest.raises(TypeError): + Distro( + os_name=123, + os_version="os_version", + compiler_name="compiler_name", + compiler_version="compiler_version", + image_sha="image_sha", + specs=[Spec()], + ) + + +def test_distro_raises_on_wrong_os_version(): + with pytest.raises(TypeError): + Distro( + os_name="os_name", + os_version=123, + compiler_name="compiler_name", + compiler_version="compiler_version", + image_sha="image_sha", + specs=[Spec()], + ) + + +def test_distro_raises_on_wrong_compiler_name(): + with pytest.raises(TypeError): + Distro( + os_name="os_name", + os_version="os_version", + compiler_name=123, + compiler_version="compiler_version", + image_sha="image_sha", + specs=[Spec()], + ) + + +def test_distro_raises_on_wrong_compiler_version(): + with pytest.raises(TypeError): + Distro( + os_name="os_name", + os_version="os_version", + compiler_name="compiler_name", + compiler_version=123, + image_sha="image_sha", + specs=[Spec()], + ) + + +def test_distro_raises_on_wrong_image_sha(): + with pytest.raises(TypeError): + Distro( + os_name="os_name", + os_version="os_version", + compiler_name="compiler_name", + compiler_version="compiler_version", + image_sha=123, + specs=[Spec()], + ) + + +def test_distro_raises_on_none_specs(): + with pytest.raises(ValueError): + Distro( + os_name="os_name", + os_version="os_version", + compiler_name="compiler_name", + compiler_version="compiler_version", + image_sha="image_sha", + specs=None, + ) + + +def test_distro_raises_on_empty_specs(): + with pytest.raises(ValueError): + Distro( + os_name="os_name", + os_version="os_version", + compiler_name="compiler_name", + compiler_version="compiler_version", + image_sha="image_sha", + specs=[], + ) + + +def test_distro_raises_on_invalid_specs(): + with pytest.raises(ValueError): + Distro( + os_name="os_name", + os_version="os_version", + compiler_name="compiler_name", + compiler_version="compiler_version", + image_sha="image_sha", + specs=[Spec(triggers=[])], + ) + + +def test_distro_raises_on_duplicate_specs(): + with pytest.raises(ValueError): + Distro( + os_name="os_name", + os_version="os_version", + compiler_name="compiler_name", + compiler_version="compiler_version", + image_sha="image_sha", + specs=[Spec(), Spec()], + ) + + +def test_distro_raises_on_wrong_specs(): + with pytest.raises(TypeError): + Distro( + os_name="os_name", + os_version="os_version", + compiler_name="compiler_name", + compiler_version="compiler_version", + image_sha="image_sha", + specs=[123], + ) diff --git a/.github/scripts/strategy-matrix/helpers/enums.py b/.github/scripts/strategy-matrix/helpers/enums.py new file mode 100755 index 0000000000..73822219b3 --- /dev/null +++ b/.github/scripts/strategy-matrix/helpers/enums.py @@ -0,0 +1,75 @@ +from enum import StrEnum, auto + + +class Arch(StrEnum): + """Represents architectures to build for.""" + + LINUX_AMD64 = "linux/amd64" + LINUX_ARM64 = "linux/arm64" + MACOS_ARM64 = "macos/arm64" + WINDOWS_AMD64 = "windows/amd64" + + +class BuildMode(StrEnum): + """Represents whether to perform a unity or non-unity build.""" + + UNITY_OFF = auto() + UNITY_ON = auto() + + +class BuildOption(StrEnum): + """Represents build options to enable.""" + + NONE = auto() + COVERAGE = auto() + SANITIZE_ASAN = ( + auto() + ) # Address Sanitizer, also includes Undefined Behavior Sanitizer. + SANITIZE_TSAN = ( + auto() + ) # Thread Sanitizer, also includes Undefined Behavior Sanitizer. + VOIDSTAR = auto() + + +class TestOption(StrEnum): + """Represents test options to enable, specifically the reference fee to use.""" + + __test__ = False # Tell pytest to not consider this as a test class. + + NONE = "" # Use the default reference fee of 10. + REFERENCE_FEE_500 = "500" + REFERENCE_FEE_1000 = "1000" + + +class PublishOption(StrEnum): + """Represents whether to publish a package, an image, or both.""" + + NONE = auto() + PACKAGE_ONLY = auto() + IMAGE_ONLY = auto() + PACKAGE_AND_IMAGE = auto() + + +class BuildType(StrEnum): + """Represents the build type to use.""" + + DEBUG = auto() + RELEASE = auto() + PUBLISH = auto() # Release build without assertions. + + +class Platform(StrEnum): + """Represents the platform to use.""" + + LINUX = "linux" + MACOS = "macos" + WINDOWS = "windows" + + +class Trigger(StrEnum): + """Represents the trigger that caused the workflow to run.""" + + COMMIT = "commit" + LABEL = "label" + MERGE = "merge" + SCHEDULE = "schedule" diff --git a/.github/scripts/strategy-matrix/helpers/funcs.py b/.github/scripts/strategy-matrix/helpers/funcs.py new file mode 100755 index 0000000000..a33788fd07 --- /dev/null +++ b/.github/scripts/strategy-matrix/helpers/funcs.py @@ -0,0 +1,235 @@ +from helpers.defs import * +from helpers.enums import * + + +def generate_config_name( + os_name: str, + os_version: str | None, + compiler_name: str | None, + compiler_version: str | None, + arch: Arch, + build_type: BuildType, + build_mode: BuildMode, + build_option: BuildOption, +) -> str: + """Create a configuration name based on the distro details and build + attributes. + + The configuration name is used as the display name in the GitHub Actions + UI, and since GitHub truncates long names we have to make sure the most + important information is at the beginning of the name. + + Args: + os_name (str): The OS name. + os_version (str): The OS version. + compiler_name (str): The compiler name. + compiler_version (str): The compiler version. + arch (Arch): The architecture. + build_type (BuildType): The build type. + build_mode (BuildMode): The build mode. + build_option (BuildOption): The build option. + + Returns: + str: The configuration name. + + Raises: + ValueError: If the OS name is empty. + """ + + if not os_name: + raise ValueError("os_name cannot be empty") + + config_name = os_name + if os_version: + config_name += f"-{os_version}" + if compiler_name: + config_name += f"-{compiler_name}" + if compiler_version: + config_name += f"-{compiler_version}" + + if build_option == BuildOption.COVERAGE: + config_name += "-coverage" + elif build_option == BuildOption.VOIDSTAR: + config_name += "-voidstar" + elif build_option == BuildOption.SANITIZE_ASAN: + config_name += "-asan" + elif build_option == BuildOption.SANITIZE_TSAN: + config_name += "-tsan" + + if build_type == BuildType.DEBUG: + config_name += "-debug" + elif build_type == BuildType.RELEASE: + config_name += "-release" + elif build_type == BuildType.PUBLISH: + config_name += "-publish" + + if build_mode == BuildMode.UNITY_ON: + config_name += "-unity" + + config_name += f"-{arch.value.split('/')[1]}" + + return config_name + + +def generate_cmake_args( + compiler_name: str | None, + compiler_version: str | None, + build_type: BuildType, + build_mode: BuildMode, + build_option: BuildOption, + test_option: TestOption, +) -> str: + """Create the CMake arguments based on the build type and enabled build + options. + + - All builds will have the `tests`, `werr`, and `xrpld` options. + - All builds will have the `wextra` option except for GCC 12 and Clang 16. + - All release builds will have the `assert` option. + - Set the unity option if specified. + - Set the coverage option if specified. + - Set the voidstar option if specified. + - Set the reference fee if specified. + + Args: + compiler_name (str): The compiler name. + compiler_version (str): The compiler version. + build_type (BuildType): The build type. + build_mode (BuildMode): The build mode. + build_option (BuildOption): The build option. + test_option (TestOption): The test option. + + Returns: + str: The CMake arguments. + + """ + + cmake_args = "-Dtests=ON -Dwerr=ON -Dxrpld=ON" + if not f"{compiler_name}-{compiler_version}" in [ + "gcc-12", + "clang-16", + ]: + cmake_args += " -Dwextra=ON" + + if build_type == BuildType.RELEASE: + cmake_args += " -Dassert=ON" + + if build_mode == BuildMode.UNITY_ON: + cmake_args += " -Dunity=ON" + + if build_option == BuildOption.COVERAGE: + cmake_args += " -Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0" + elif build_option == BuildOption.SANITIZE_ASAN: + pass # TODO: Add ASAN-UBSAN flags. + elif build_option == BuildOption.SANITIZE_TSAN: + pass # TODO: Add TSAN-UBSAN flags. + elif build_option == BuildOption.VOIDSTAR: + cmake_args += " -Dvoidstar=ON" + + if test_option != TestOption.NONE: + cmake_args += f" -DUNIT_TEST_REFERENCE_FEE={test_option.value}" + + return cmake_args + + +def generate_cmake_target(os_name: str, build_type: BuildType) -> str: + """Create the CMake target based on the build type. + + The `install` target is used for Windows and for publishing a package, while + the `all` target is used for all other configurations. + + Args: + os_name (str): The OS name. + build_type (BuildType): The build type. + + Returns: + str: The CMake target. + """ + if os_name == "windows" or build_type == BuildType.PUBLISH: + return "install" + return "all" + + +def generate_enable_options( + os_name: str, + build_type: BuildType, + publish_option: PublishOption, +) -> tuple[bool, bool, bool]: + """Create the enable flags based on the OS name, build option, and publish + option. + + We build and test all configurations by default, except for Windows in + Debug, because it is too slow. + + Args: + os_name (str): The OS name. + build_type (BuildType): The build type. + publish_option (PublishOption): The publish option. + + Returns: + tuple: A tuple containing the enable test, enable package, and enable image flags. + """ + enable_tests = ( + False if os_name == "windows" and build_type == BuildType.DEBUG else True + ) + + enable_package = ( + True + if publish_option + in [ + PublishOption.PACKAGE_ONLY, + PublishOption.PACKAGE_AND_IMAGE, + ] + else False + ) + + enable_image = ( + True + if publish_option + in [ + PublishOption.IMAGE_ONLY, + PublishOption.PACKAGE_AND_IMAGE, + ] + else False + ) + + return enable_tests, enable_package, enable_image + + +def generate_image_name( + os_name: str, + os_version: str, + compiler_name: str, + compiler_version: str, + image_sha: str, +) -> str | None: + """Create the Docker image name based on the distro details. + + Args: + os_name (str): The OS name. + os_version (str): The OS version. + compiler_name (str): The compiler name. + compiler_version (str): The compiler version. + image_sha (str): The image SHA. + + Returns: + str: The Docker image name or None if not applicable. + + Raises: + ValueError: If any of the arguments is empty for Linux. + """ + + if os_name == "windows" or os_name == "macos": + return None + + if not os_name: + raise ValueError("os_name cannot be empty") + if not os_version: + raise ValueError("os_version cannot be empty") + if not compiler_name: + raise ValueError("compiler_name cannot be empty") + if not compiler_version: + raise ValueError("compiler_version cannot be empty") + if not image_sha: + raise ValueError("image_sha cannot be empty") + + return f"ghcr.io/xrplf/ci/{os_name}-{os_version}:{compiler_name}-{compiler_version}-{image_sha}" diff --git a/.github/scripts/strategy-matrix/helpers/funcs_test.py b/.github/scripts/strategy-matrix/helpers/funcs_test.py new file mode 100644 index 0000000000..0a11f6e133 --- /dev/null +++ b/.github/scripts/strategy-matrix/helpers/funcs_test.py @@ -0,0 +1,419 @@ +import pytest + +from helpers.enums import * +from helpers.funcs import * + + +def test_generate_config_name_a_b_c_d_debug_amd64(): + assert ( + generate_config_name( + "a", + "b", + "c", + "d", + Arch.LINUX_AMD64, + BuildType.DEBUG, + BuildMode.UNITY_OFF, + BuildOption.NONE, + ) + == "a-b-c-d-debug-amd64" + ) + + +def test_generate_config_name_a_b_c_release_unity_arm64(): + assert ( + generate_config_name( + "a", + "b", + "c", + "", + Arch.LINUX_ARM64, + BuildType.RELEASE, + BuildMode.UNITY_ON, + BuildOption.NONE, + ) + == "a-b-c-release-unity-arm64" + ) + + +def test_generate_config_name_a_b_coverage_publish_amd64(): + assert ( + generate_config_name( + "a", + "b", + "", + "", + Arch.LINUX_AMD64, + BuildType.PUBLISH, + BuildMode.UNITY_OFF, + BuildOption.COVERAGE, + ) + == "a-b-coverage-publish-amd64" + ) + + +def test_generate_config_name_a_asan_debug_unity_arm64(): + assert ( + generate_config_name( + "a", + "", + "", + "", + Arch.LINUX_ARM64, + BuildType.DEBUG, + BuildMode.UNITY_ON, + BuildOption.SANITIZE_ASAN, + ) + == "a-asan-debug-unity-arm64" + ) + + +def test_generate_config_name_a_c_tsan_release_amd64(): + assert ( + generate_config_name( + "a", + "", + "c", + "", + Arch.LINUX_AMD64, + BuildType.RELEASE, + BuildMode.UNITY_OFF, + BuildOption.SANITIZE_TSAN, + ) + == "a-c-tsan-release-amd64" + ) + + +def test_generate_config_name_a_d_voidstar_debug_amd64(): + assert ( + generate_config_name( + "a", + "", + "", + "d", + Arch.LINUX_AMD64, + BuildType.DEBUG, + BuildMode.UNITY_OFF, + BuildOption.VOIDSTAR, + ) + == "a-d-voidstar-debug-amd64" + ) + + +def test_generate_config_name_raises_on_none_os_name(): + with pytest.raises(ValueError): + generate_config_name( + None, + "b", + "c", + "d", + Arch.LINUX_AMD64, + BuildType.DEBUG, + BuildMode.UNITY_OFF, + BuildOption.NONE, + ) + + +def test_generate_config_name_raises_on_empty_os_name(): + with pytest.raises(ValueError): + generate_config_name( + "", + "b", + "c", + "d", + Arch.LINUX_AMD64, + BuildType.DEBUG, + BuildMode.UNITY_OFF, + BuildOption.NONE, + ) + + +def test_generate_cmake_args_a_b_debug(): + assert ( + generate_cmake_args( + "a", + "b", + BuildType.DEBUG, + BuildMode.UNITY_OFF, + BuildOption.NONE, + TestOption.NONE, + ) + == "-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON" + ) + + +def test_generate_cmake_args_gcc_12_no_wextra(): + assert ( + generate_cmake_args( + "gcc", + "12", + BuildType.DEBUG, + BuildMode.UNITY_OFF, + BuildOption.NONE, + TestOption.NONE, + ) + == "-Dtests=ON -Dwerr=ON -Dxrpld=ON" + ) + + +def test_generate_cmake_args_clang_16_no_wextra(): + assert ( + generate_cmake_args( + "clang", + "16", + BuildType.DEBUG, + BuildMode.UNITY_OFF, + BuildOption.NONE, + TestOption.NONE, + ) + == "-Dtests=ON -Dwerr=ON -Dxrpld=ON" + ) + + +def test_generate_cmake_args_a_b_release(): + assert ( + generate_cmake_args( + "a", + "b", + BuildType.RELEASE, + BuildMode.UNITY_OFF, + BuildOption.NONE, + TestOption.NONE, + ) + == "-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -Dassert=ON" + ) + + +def test_generate_cmake_args_a_b_publish(): + assert ( + generate_cmake_args( + "a", + "b", + BuildType.PUBLISH, + BuildMode.UNITY_OFF, + BuildOption.NONE, + TestOption.NONE, + ) + == "-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON" + ) + + +def test_generate_cmake_args_a_b_unity(): + assert ( + generate_cmake_args( + "a", + "b", + BuildType.DEBUG, + BuildMode.UNITY_ON, + BuildOption.NONE, + TestOption.NONE, + ) + == "-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -Dunity=ON" + ) + + +def test_generate_cmake_args_a_b_coverage(): + assert ( + generate_cmake_args( + "a", + "b", + BuildType.DEBUG, + BuildMode.UNITY_OFF, + BuildOption.COVERAGE, + TestOption.NONE, + ) + == "-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0" + ) + + +def test_generate_cmake_args_a_b_voidstar(): + assert ( + generate_cmake_args( + "a", + "b", + BuildType.DEBUG, + BuildMode.UNITY_OFF, + BuildOption.VOIDSTAR, + TestOption.NONE, + ) + == "-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -Dvoidstar=ON" + ) + + +def test_generate_cmake_args_a_b_reference_fee_500(): + assert ( + generate_cmake_args( + "a", + "b", + BuildType.DEBUG, + BuildMode.UNITY_OFF, + BuildOption.NONE, + TestOption.REFERENCE_FEE_500, + ) + == "-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -DUNIT_TEST_REFERENCE_FEE=500" + ) + + +def test_generate_cmake_args_a_b_reference_fee_1000(): + assert ( + generate_cmake_args( + "a", + "b", + BuildType.DEBUG, + BuildMode.UNITY_OFF, + BuildOption.NONE, + TestOption.REFERENCE_FEE_1000, + ) + == "-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -DUNIT_TEST_REFERENCE_FEE=1000" + ) + + +def test_generate_cmake_args_a_b_multiple(): + assert ( + generate_cmake_args( + "a", + "b", + BuildType.RELEASE, + BuildMode.UNITY_ON, + BuildOption.VOIDSTAR, + TestOption.REFERENCE_FEE_500, + ) + == "-Dtests=ON -Dwerr=ON -Dxrpld=ON -Dwextra=ON -Dassert=ON -Dunity=ON -Dvoidstar=ON -DUNIT_TEST_REFERENCE_FEE=500" + ) + + +def test_generate_cmake_target_linux_debug(): + assert generate_cmake_target("linux", BuildType.DEBUG) == "all" + + +def test_generate_cmake_target_linux_release(): + assert generate_cmake_target("linux", BuildType.RELEASE) == "all" + + +def test_generate_cmake_target_linux_publish(): + assert generate_cmake_target("linux", BuildType.PUBLISH) == "install" + + +def test_generate_cmake_target_macos_debug(): + assert generate_cmake_target("macos", BuildType.DEBUG) == "all" + + +def test_generate_cmake_target_macos_release(): + assert generate_cmake_target("macos", BuildType.RELEASE) == "all" + + +def test_generate_cmake_target_macos_publish(): + assert generate_cmake_target("macos", BuildType.PUBLISH) == "install" + + +def test_generate_cmake_target_windows_debug(): + assert generate_cmake_target("windows", BuildType.DEBUG) == "install" + + +def test_generate_cmake_target_windows_release(): + assert generate_cmake_target("windows", BuildType.DEBUG) == "install" + + +def test_generate_cmake_target_windows_publish(): + assert generate_cmake_target("windows", BuildType.DEBUG) == "install" + + +def test_generate_enable_options_linux_debug_no_publish(): + assert generate_enable_options("linux", BuildType.DEBUG, PublishOption.NONE) == ( + True, + False, + False, + ) + + +def test_generate_enable_options_linux_release_package_only(): + assert generate_enable_options( + "linux", BuildType.RELEASE, PublishOption.PACKAGE_ONLY + ) == (True, True, False) + + +def test_generate_enable_options_linux_publish_image_only(): + assert generate_enable_options( + "linux", BuildType.PUBLISH, PublishOption.IMAGE_ONLY + ) == (True, False, True) + + +def test_generate_enable_options_macos_debug_package_only(): + assert generate_enable_options( + "macos", BuildType.DEBUG, PublishOption.PACKAGE_ONLY + ) == (True, True, False) + + +def test_generate_enable_options_macos_release_image_only(): + assert generate_enable_options( + "macos", BuildType.RELEASE, PublishOption.IMAGE_ONLY + ) == (True, False, True) + + +def test_generate_enable_options_macos_publish_package_and_image(): + assert generate_enable_options( + "macos", BuildType.PUBLISH, PublishOption.PACKAGE_AND_IMAGE + ) == (True, True, True) + + +def test_generate_enable_options_windows_debug_package_and_image(): + assert generate_enable_options( + "windows", BuildType.DEBUG, PublishOption.PACKAGE_AND_IMAGE + ) == (False, True, True) + + +def test_generate_enable_options_windows_release_no_publish(): + assert generate_enable_options( + "windows", BuildType.RELEASE, PublishOption.NONE + ) == (True, False, False) + + +def test_generate_enable_options_windows_publish_image_only(): + assert generate_enable_options( + "windows", BuildType.PUBLISH, PublishOption.IMAGE_ONLY + ) == (True, False, True) + + +def test_generate_image_name_linux(): + assert generate_image_name("a", "b", "c", "d", "e") == "ghcr.io/xrplf/ci/a-b:c-d-e" + + +def test_generate_image_name_linux_raises_on_empty_os_name(): + with pytest.raises(ValueError): + generate_image_name("", "b", "c", "d", "e") + + +def test_generate_image_name_linux_raises_on_empty_os_version(): + with pytest.raises(ValueError): + generate_image_name("a", "", "c", "d", "e") + + +def test_generate_image_name_linux_raises_on_empty_compiler_name(): + with pytest.raises(ValueError): + generate_image_name("a", "b", "", "d", "e") + + +def test_generate_image_name_linux_raises_on_empty_compiler_version(): + with pytest.raises(ValueError): + generate_image_name("a", "b", "c", "", "e") + + +def test_generate_image_name_linux_raises_on_empty_image_sha(): + with pytest.raises(ValueError): + generate_image_name("a", "b", "c", "e", "") + + +def test_generate_image_name_macos(): + assert generate_image_name("macos", "", "", "", "") is None + + +def test_generate_image_name_macos_extra(): + assert generate_image_name("macos", "value", "does", "not", "matter") is None + + +def test_generate_image_name_windows(): + assert generate_image_name("windows", "", "", "", "") is None + + +def test_generate_image_name_windows_extra(): + assert generate_image_name("windows", "value", "does", "not", "matter") is None diff --git a/.github/scripts/strategy-matrix/helpers/unique.py b/.github/scripts/strategy-matrix/helpers/unique.py new file mode 100644 index 0000000000..01d68837eb --- /dev/null +++ b/.github/scripts/strategy-matrix/helpers/unique.py @@ -0,0 +1,30 @@ +from dataclasses import asdict, _is_dataclass_instance +import json +from typing import Any + + +def is_unique(items: list[Any]) -> bool: + """Check if a list of dataclass objects contains only unique items. + + As the items may not be hashable, we convert them to json strings first, and + then check if the list of strings is the same size as the set of strings. + + Args: + items: The list of dataclass objects to check. + + Returns: + True if the list contains only unique items, False otherwise. + + Raises: + TypeError: If any of the items is not a dataclass. + """ + + l = list() + s = set() + for item in items: + if not _is_dataclass_instance(item): + raise TypeError("items must be a list of dataclasses") + j = json.dumps(asdict(item)) + l.append(j) + s.add(j) + return len(l) == len(s) diff --git a/.github/scripts/strategy-matrix/helpers/unique_test.py b/.github/scripts/strategy-matrix/helpers/unique_test.py new file mode 100644 index 0000000000..94a629d7a7 --- /dev/null +++ b/.github/scripts/strategy-matrix/helpers/unique_test.py @@ -0,0 +1,39 @@ +import pytest +from dataclasses import dataclass + +from helpers.unique import * + + +@dataclass +class ExampleInt: + value: int + + +@dataclass +class ExampleList: + values: list[int] + + +def test_unique_int(): + assert is_unique([ExampleInt(1), ExampleInt(2), ExampleInt(3)]) + + +def test_not_unique_int(): + assert not is_unique([ExampleInt(1), ExampleInt(2), ExampleInt(1)]) + + +def test_unique_list(): + assert is_unique( + [ExampleList([1, 2, 3]), ExampleList([4, 5, 6]), ExampleList([7, 8, 9])] + ) + + +def test_not_unique_list(): + assert not is_unique( + [ExampleList([1, 2, 3]), ExampleList([4, 5, 6]), ExampleList([1, 2, 3])] + ) + + +def test_unique_raises_on_non_dataclass(): + with pytest.raises(TypeError): + is_unique([1, 2, 3]) diff --git a/.github/scripts/strategy-matrix/linux.json b/.github/scripts/strategy-matrix/linux.json deleted file mode 100644 index 748ee031c9..0000000000 --- a/.github/scripts/strategy-matrix/linux.json +++ /dev/null @@ -1,212 +0,0 @@ -{ - "architecture": [ - { - "platform": "linux/amd64", - "runner": ["self-hosted", "Linux", "X64", "heavy"] - }, - { - "platform": "linux/arm64", - "runner": ["self-hosted", "Linux", "ARM64", "heavy-arm64"] - } - ], - "os": [ - { - "distro_name": "debian", - "distro_version": "bookworm", - "compiler_name": "gcc", - "compiler_version": "12", - "image_sha": "0525eae" - }, - { - "distro_name": "debian", - "distro_version": "bookworm", - "compiler_name": "gcc", - "compiler_version": "13", - "image_sha": "0525eae" - }, - { - "distro_name": "debian", - "distro_version": "bookworm", - "compiler_name": "gcc", - "compiler_version": "14", - "image_sha": "0525eae" - }, - { - "distro_name": "debian", - "distro_version": "bookworm", - "compiler_name": "gcc", - "compiler_version": "15", - "image_sha": "0525eae" - }, - { - "distro_name": "debian", - "distro_version": "bookworm", - "compiler_name": "clang", - "compiler_version": "16", - "image_sha": "0525eae" - }, - { - "distro_name": "debian", - "distro_version": "bookworm", - "compiler_name": "clang", - "compiler_version": "17", - "image_sha": "0525eae" - }, - { - "distro_name": "debian", - "distro_version": "bookworm", - "compiler_name": "clang", - "compiler_version": "18", - "image_sha": "0525eae" - }, - { - "distro_name": "debian", - "distro_version": "bookworm", - "compiler_name": "clang", - "compiler_version": "19", - "image_sha": "0525eae" - }, - { - "distro_name": "debian", - "distro_version": "bookworm", - "compiler_name": "clang", - "compiler_version": "20", - "image_sha": "0525eae" - }, - { - "distro_name": "debian", - "distro_version": "trixie", - "compiler_name": "gcc", - "compiler_version": "14", - "image_sha": "0525eae" - }, - { - "distro_name": "debian", - "distro_version": "trixie", - "compiler_name": "gcc", - "compiler_version": "15", - "image_sha": "0525eae" - }, - { - "distro_name": "debian", - "distro_version": "trixie", - "compiler_name": "clang", - "compiler_version": "20", - "image_sha": "0525eae" - }, - { - "distro_name": "debian", - "distro_version": "trixie", - "compiler_name": "clang", - "compiler_version": "21", - "image_sha": "0525eae" - }, - { - "distro_name": "rhel", - "distro_version": "8", - "compiler_name": "gcc", - "compiler_version": "14", - "image_sha": "e1782cd" - }, - { - "distro_name": "rhel", - "distro_version": "8", - "compiler_name": "clang", - "compiler_version": "any", - "image_sha": "e1782cd" - }, - { - "distro_name": "rhel", - "distro_version": "9", - "compiler_name": "gcc", - "compiler_version": "12", - "image_sha": "e1782cd" - }, - { - "distro_name": "rhel", - "distro_version": "9", - "compiler_name": "gcc", - "compiler_version": "13", - "image_sha": "e1782cd" - }, - { - "distro_name": "rhel", - "distro_version": "9", - "compiler_name": "gcc", - "compiler_version": "14", - "image_sha": "e1782cd" - }, - { - "distro_name": "rhel", - "distro_version": "9", - "compiler_name": "clang", - "compiler_version": "any", - "image_sha": "e1782cd" - }, - { - "distro_name": "rhel", - "distro_version": "10", - "compiler_name": "gcc", - "compiler_version": "14", - "image_sha": "e1782cd" - }, - { - "distro_name": "rhel", - "distro_version": "10", - "compiler_name": "clang", - "compiler_version": "any", - "image_sha": "e1782cd" - }, - { - "distro_name": "ubuntu", - "distro_version": "jammy", - "compiler_name": "gcc", - "compiler_version": "12", - "image_sha": "e1782cd" - }, - { - "distro_name": "ubuntu", - "distro_version": "noble", - "compiler_name": "gcc", - "compiler_version": "13", - "image_sha": "e1782cd" - }, - { - "distro_name": "ubuntu", - "distro_version": "noble", - "compiler_name": "gcc", - "compiler_version": "14", - "image_sha": "e1782cd" - }, - { - "distro_name": "ubuntu", - "distro_version": "noble", - "compiler_name": "clang", - "compiler_version": "16", - "image_sha": "e1782cd" - }, - { - "distro_name": "ubuntu", - "distro_version": "noble", - "compiler_name": "clang", - "compiler_version": "17", - "image_sha": "e1782cd" - }, - { - "distro_name": "ubuntu", - "distro_version": "noble", - "compiler_name": "clang", - "compiler_version": "18", - "image_sha": "e1782cd" - }, - { - "distro_name": "ubuntu", - "distro_version": "noble", - "compiler_name": "clang", - "compiler_version": "19", - "image_sha": "e1782cd" - } - ], - "build_type": ["Debug", "Release"], - "cmake_args": ["-Dunity=OFF", "-Dunity=ON"] -} diff --git a/.github/scripts/strategy-matrix/linux.py b/.github/scripts/strategy-matrix/linux.py new file mode 100755 index 0000000000..c92dc91f58 --- /dev/null +++ b/.github/scripts/strategy-matrix/linux.py @@ -0,0 +1,490 @@ +from helpers.defs import * +from helpers.enums import * + +# The default CI image SHAs to use, which can be specified per distro group and +# can be overridden for individual distros, which is useful when debugging using +# a locally built CI image. See https://github.com/XRPLF/ci for the images. +DEBIAN_SHA = "sha-0525eae" +RHEL_SHA = "sha-0525eae" +UBUNTU_SHA = "sha-84afd81" + +DEBIAN_DISTROS = [ + Distro( + os_name="debian", + os_version="bullseye", + compiler_name="gcc", + compiler_version="12", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="bullseye", + compiler_name="gcc", + compiler_version="13", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="bullseye", + compiler_name="gcc", + compiler_version="14", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + publish_option=PublishOption.PACKAGE_ONLY, + triggers=[Trigger.COMMIT, Trigger.LABEL], + ), + Spec( + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.PUBLISH], + publish_option=PublishOption.PACKAGE_AND_IMAGE, + triggers=[Trigger.MERGE], + ), + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="bullseye", + compiler_name="gcc", + compiler_version="15", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_ARM64], + build_modes=[BuildMode.UNITY_ON], + build_option=BuildOption.COVERAGE, + build_types=[BuildType.DEBUG], + triggers=[Trigger.COMMIT, Trigger.MERGE], + ), + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="bookworm", + compiler_name="gcc", + compiler_version="13", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="bookworm", + compiler_name="gcc", + compiler_version="14", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="bookworm", + compiler_name="gcc", + compiler_version="15", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="bookworm", + compiler_name="clang", + compiler_version="16", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_AMD64], + build_modes=[BuildMode.UNITY_OFF], + build_option=BuildOption.VOIDSTAR, + build_types=[BuildType.DEBUG], + publish_option=PublishOption.IMAGE_ONLY, + triggers=[Trigger.COMMIT], + ), + Spec( + archs=[Arch.LINUX_ARM64], + build_modes=[BuildMode.UNITY_ON], + build_types=[BuildType.RELEASE], + triggers=[Trigger.MERGE], + ), + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="bookworm", + compiler_name="clang", + compiler_version="17", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="bookworm", + compiler_name="clang", + compiler_version="18", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="bookworm", + compiler_name="clang", + compiler_version="19", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="bookworm", + compiler_name="clang", + compiler_version="20", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="trixie", + compiler_name="gcc", + compiler_version="14", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="trixie", + compiler_name="gcc", + compiler_version="15", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="trixie", + compiler_name="clang", + compiler_version="20", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_AMD64], + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="debian", + os_version="trixie", + compiler_name="clang", + compiler_version="21", + image_sha=DEBIAN_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_AMD64], + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + triggers=[Trigger.MERGE], + ), + Spec( + archs=[Arch.LINUX_AMD64], + triggers=[Trigger.SCHEDULE], + ), + ], + ), +] +# The Debian distros to build for. + +# The RHEL distros to build for. +RHEL_DISTROS = [ + Distro( + os_name="rhel", + os_version="8", + compiler_name="gcc", + compiler_version="14", + image_sha=RHEL_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_AMD64], + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="rhel", + os_version="8", + compiler_name="clang", + compiler_version="any", + image_sha=RHEL_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_AMD64], + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="rhel", + os_version="9", + compiler_name="gcc", + compiler_version="12", + image_sha=RHEL_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_AMD64], + build_modes=[BuildMode.UNITY_ON], + build_types=[BuildType.DEBUG], + triggers=[Trigger.COMMIT], + ), + Spec( + archs=[Arch.LINUX_AMD64], + build_modes=[BuildMode.UNITY_ON], + build_types=[BuildType.RELEASE], + triggers=[Trigger.MERGE], + ), + Spec( + archs=[Arch.LINUX_AMD64], + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="rhel", + os_version="9", + compiler_name="gcc", + compiler_version="13", + image_sha=RHEL_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_AMD64], + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="rhel", + os_version="9", + compiler_name="gcc", + compiler_version="14", + image_sha=RHEL_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_AMD64], + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="rhel", + os_version="9", + compiler_name="clang", + compiler_version="any", + image_sha=RHEL_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_AMD64], + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="rhel", + os_version="10", + compiler_name="gcc", + compiler_version="14", + image_sha=RHEL_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_AMD64], + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="rhel", + os_version="10", + compiler_name="clang", + compiler_version="any", + image_sha=RHEL_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_AMD64], + triggers=[Trigger.SCHEDULE], + ), + ], + ), +] + +# The Ubuntu distros to build for. +UBUNTU_DISTROS = [ + Distro( + os_name="ubuntu", + os_version="jammy", + compiler_name="gcc", + compiler_version="12", + image_sha=UBUNTU_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="ubuntu", + os_version="noble", + compiler_name="gcc", + compiler_version="13", + image_sha=UBUNTU_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_ARM64], + build_modes=[BuildMode.UNITY_ON], + build_types=[BuildType.RELEASE], + triggers=[Trigger.MERGE], + ), + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="ubuntu", + os_version="noble", + compiler_name="gcc", + compiler_version="14", + image_sha=UBUNTU_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="ubuntu", + os_version="noble", + compiler_name="clang", + compiler_version="16", + image_sha=UBUNTU_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="ubuntu", + os_version="noble", + compiler_name="clang", + compiler_version="17", + image_sha=UBUNTU_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_ARM64], + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.DEBUG], + triggers=[Trigger.MERGE], + ), + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="ubuntu", + os_version="noble", + compiler_name="clang", + compiler_version="18", + image_sha=UBUNTU_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="ubuntu", + os_version="noble", + compiler_name="clang", + compiler_version="19", + image_sha=UBUNTU_SHA, + specs=[ + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), + Distro( + os_name="ubuntu", + os_version="noble", + compiler_name="clang", + compiler_version="20", + image_sha=UBUNTU_SHA, + specs=[ + Spec( + archs=[Arch.LINUX_ARM64], + build_modes=[BuildMode.UNITY_ON], + build_types=[BuildType.DEBUG], + triggers=[Trigger.COMMIT], + ), + Spec( + archs=[Arch.LINUX_AMD64], + build_modes=[BuildMode.UNITY_OFF], + build_types=[BuildType.RELEASE], + triggers=[Trigger.MERGE], + ), + Spec( + triggers=[Trigger.SCHEDULE], + ), + ], + ), +] diff --git a/.github/scripts/strategy-matrix/macos.json b/.github/scripts/strategy-matrix/macos.json deleted file mode 100644 index 14b6089620..0000000000 --- a/.github/scripts/strategy-matrix/macos.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "architecture": [ - { - "platform": "macos/arm64", - "runner": ["self-hosted", "macOS", "ARM64", "mac-runner-m1"] - } - ], - "os": [ - { - "distro_name": "macos", - "distro_version": "", - "compiler_name": "", - "compiler_version": "", - "image_sha": "" - } - ], - "build_type": ["Debug", "Release"], - "cmake_args": [ - "-Dunity=OFF -DCMAKE_POLICY_VERSION_MINIMUM=3.5", - "-Dunity=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5" - ] -} diff --git a/.github/scripts/strategy-matrix/macos.py b/.github/scripts/strategy-matrix/macos.py new file mode 100755 index 0000000000..b0d689416c --- /dev/null +++ b/.github/scripts/strategy-matrix/macos.py @@ -0,0 +1,9 @@ +from helpers.defs import Distro, Spec +from helpers.enums import Arch + +DISTROS = [ + Distro( + os_name="macos", + specs=[Spec(archs=[Arch.MACOS_ARM64])], + ), +] diff --git a/.github/scripts/strategy-matrix/windows.json b/.github/scripts/strategy-matrix/windows.json deleted file mode 100644 index 8637b31012..0000000000 --- a/.github/scripts/strategy-matrix/windows.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "architecture": [ - { - "platform": "windows/amd64", - "runner": ["self-hosted", "Windows", "devbox"] - } - ], - "os": [ - { - "distro_name": "windows", - "distro_version": "", - "compiler_name": "", - "compiler_version": "", - "image_sha": "" - } - ], - "build_type": ["Debug", "Release"], - "cmake_args": ["-Dunity=OFF", "-Dunity=ON"] -} diff --git a/.github/scripts/strategy-matrix/windows.py b/.github/scripts/strategy-matrix/windows.py new file mode 100755 index 0000000000..92ec551c2d --- /dev/null +++ b/.github/scripts/strategy-matrix/windows.py @@ -0,0 +1,9 @@ +from helpers.defs import Distro, Spec +from helpers.enums import Arch + +DISTROS = [ + Distro( + os_name="windows", + specs=[Spec(archs=[Arch.WINDOWS_AMD64])], + ), +] diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index ff3d25812a..683adbb89a 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -112,9 +112,10 @@ jobs: strategy: fail-fast: false matrix: - os: [linux, macos, windows] + platform: [linux, macos, windows] with: - os: ${{ matrix.os }} + platform: ${{ matrix.platform }} + trigger: commit secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/on-trigger.yml b/.github/workflows/on-trigger.yml index b5a56fb671..2adf5c3e3a 100644 --- a/.github/workflows/on-trigger.yml +++ b/.github/workflows/on-trigger.yml @@ -66,9 +66,10 @@ jobs: strategy: fail-fast: ${{ github.event_name == 'merge_group' }} matrix: - os: [linux, macos, windows] + platform: [linux, macos, windows] with: - os: ${{ matrix.os }} - strategy_matrix: ${{ github.event_name == 'schedule' && 'all' || 'minimal' }} + platform: ${{ matrix.platform }} + # The workflow dispatch event uses the same trigger as the schedule event. + trigger: ${{ github.event_name == 'push' && 'merge' || 'schedule' }} secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/reusable-build-test-config.yml b/.github/workflows/reusable-build-test-config.yml index 98bf107225..38d3432423 100644 --- a/.github/workflows/reusable-build-test-config.yml +++ b/.github/workflows/reusable-build-test-config.yml @@ -3,11 +3,6 @@ name: Build and test configuration on: workflow_call: inputs: - build_only: - description: 'Whether to only build or to build and test the code ("true", "false").' - required: true - type: boolean - build_type: description: 'The build type to use ("Debug", "Release").' type: string @@ -24,6 +19,21 @@ on: type: string required: true + enable_tests: + description: "Whether to run the tests." + required: true + type: boolean + + enable_package: + description: "Whether to publish a package." + required: true + type: boolean + + enable_image: + description: "Whether to publish an image." + required: true + type: boolean + runs_on: description: Runner to run the job on as a JSON string required: true @@ -156,7 +166,7 @@ jobs: ./xrpld --version | grep libvoidstar - name: Run the separate tests - if: ${{ !inputs.build_only }} + if: ${{ inputs.enable_tests }} working-directory: ${{ env.BUILD_DIR }} # Windows locks some of the build files while running tests, and parallel jobs can collide env: @@ -169,7 +179,7 @@ jobs: -j "${PARALLELISM}" - name: Run the embedded tests - if: ${{ !inputs.build_only }} + if: ${{ inputs.enable_tests }} working-directory: ${{ runner.os == 'Windows' && format('{0}/{1}', env.BUILD_DIR, inputs.build_type) || env.BUILD_DIR }} env: BUILD_NPROC: ${{ steps.nproc.outputs.nproc }} @@ -177,7 +187,7 @@ jobs: ./xrpld --unittest --unittest-jobs "${BUILD_NPROC}" - name: Debug failure (Linux) - if: ${{ failure() && runner.os == 'Linux' && !inputs.build_only }} + if: ${{ failure() && runner.os == 'Linux' && inputs.enable_tests }} run: | echo "IPv4 local port range:" cat /proc/sys/net/ipv4/ip_local_port_range @@ -185,7 +195,7 @@ jobs: netstat -an - name: Prepare coverage report - if: ${{ !inputs.build_only && env.ENABLED_COVERAGE == 'true' }} + if: ${{ github.repository_owner == 'XRPLF' && env.ENABLED_COVERAGE == 'true' }} working-directory: ${{ env.BUILD_DIR }} env: BUILD_NPROC: ${{ steps.nproc.outputs.nproc }} @@ -198,7 +208,7 @@ jobs: --target coverage - name: Upload coverage report - if: ${{ github.repository_owner == 'XRPLF' && !inputs.build_only && env.ENABLED_COVERAGE == 'true' }} + if: ${{ github.repository_owner == 'XRPLF' && env.ENABLED_COVERAGE == 'true' }} uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3 with: disable_search: true diff --git a/.github/workflows/reusable-build-test.yml b/.github/workflows/reusable-build-test.yml index 7f14aacb9b..fab5673f1e 100644 --- a/.github/workflows/reusable-build-test.yml +++ b/.github/workflows/reusable-build-test.yml @@ -8,16 +8,23 @@ name: Build and test on: workflow_call: inputs: - os: - description: 'The operating system to use for the build ("linux", "macos", "windows").' - required: true - type: string - strategy_matrix: - # TODO: Support additional strategies, e.g. "ubuntu" for generating all Ubuntu configurations. - description: 'The strategy matrix to use for generating the configurations ("minimal", "all").' + platform: + description: "The platform to generate the strategy matrix for. If not provided all platforms are used." required: false - type: string - default: "minimal" + type: choice + options: + - linux + - macos + - windows + trigger: + description: "The trigger that caused the workflow to run." + required: true + type: choice + options: + - commit + - label + - merge + - schedule secrets: CODECOV_TOKEN: description: "The Codecov token to use for uploading coverage reports." @@ -28,8 +35,8 @@ jobs: generate-matrix: uses: ./.github/workflows/reusable-strategy-matrix.yml with: - os: ${{ inputs.os }} - strategy_matrix: ${{ inputs.strategy_matrix }} + platform: ${{ inputs.platform }} + trigger: ${{ inputs.trigger }} # Build and test the binary for each configuration. build-test-config: @@ -41,12 +48,14 @@ jobs: matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} max-parallel: 10 with: - build_only: ${{ matrix.build_only }} build_type: ${{ matrix.build_type }} cmake_args: ${{ matrix.cmake_args }} cmake_target: ${{ matrix.cmake_target }} - runs_on: ${{ toJSON(matrix.architecture.runner) }} - image: ${{ contains(matrix.architecture.platform, 'linux') && format('ghcr.io/xrplf/ci/{0}-{1}:{2}-{3}-sha-{4}', matrix.os.distro_name, matrix.os.distro_version, matrix.os.compiler_name, matrix.os.compiler_version, matrix.os.image_sha) || '' }} + enable_tests: ${{ matrix.enable_tests }} + enable_package: ${{ matrix.enable_package }} + enable_image: ${{ matrix.enable_image }} + runs_on: ${{ matrix.runs_on }} + image: ${{ matrix.image }} config_name: ${{ matrix.config_name }} secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/reusable-strategy-matrix.yml b/.github/workflows/reusable-strategy-matrix.yml index c975347307..36a1dced2f 100644 --- a/.github/workflows/reusable-strategy-matrix.yml +++ b/.github/workflows/reusable-strategy-matrix.yml @@ -3,16 +3,23 @@ name: Generate strategy matrix on: workflow_call: inputs: - os: - description: 'The operating system to use for the build ("linux", "macos", "windows").' + platform: + description: "The platform to generate the strategy matrix for. If not provided all platforms are used." required: false - type: string - strategy_matrix: - # TODO: Support additional strategies, e.g. "ubuntu" for generating all Ubuntu configurations. - description: 'The strategy matrix to use for generating the configurations ("minimal", "all").' - required: false - type: string - default: "minimal" + type: choice + options: + - linux + - macos + - windows + trigger: + description: "The trigger that caused the workflow to run." + required: true + type: choice + options: + - commit + - label + - merge + - schedule outputs: matrix: description: "The generated strategy matrix." @@ -40,6 +47,6 @@ jobs: working-directory: .github/scripts/strategy-matrix id: generate env: - GENERATE_CONFIG: ${{ inputs.os != '' && format('--config={0}.json', inputs.os) || '' }} - GENERATE_OPTION: ${{ inputs.strategy_matrix == 'all' && '--all' || '' }} - run: ./generate.py ${GENERATE_OPTION} ${GENERATE_CONFIG} >> "${GITHUB_OUTPUT}" + PLATFORM: ${{ inputs.platform != '' && format('--platform={0}', inputs.platform) || '' }} + TRIGGER: ${{ format('--trigger={0}', inputs.trigger) }} + run: ./generate.py ${PLATFORM} ${TRIGGER} >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/upload-conan-deps.yml b/.github/workflows/upload-conan-deps.yml index 5024666394..788e2fcb5a 100644 --- a/.github/workflows/upload-conan-deps.yml +++ b/.github/workflows/upload-conan-deps.yml @@ -19,17 +19,17 @@ on: branches: [develop] paths: # This allows testing changes to the upload workflow in a PR - - .github/workflows/upload-conan-deps.yml + - ".github/workflows/upload-conan-deps.yml" push: branches: [develop] paths: - - .github/workflows/upload-conan-deps.yml - - .github/workflows/reusable-strategy-matrix.yml - - .github/actions/build-deps/action.yml - - .github/actions/setup-conan/action.yml + - ".github/workflows/upload-conan-deps.yml" + - ".github/workflows/reusable-strategy-matrix.yml" + - ".github/actions/build-deps/action.yml" + - ".github/actions/setup-conan/action.yml" - ".github/scripts/strategy-matrix/**" - - conanfile.py - - conan.lock + - "conanfile.py" + - "conan.lock" env: CONAN_REMOTE_NAME: xrplf @@ -49,7 +49,8 @@ jobs: generate-matrix: uses: ./.github/workflows/reusable-strategy-matrix.yml with: - strategy_matrix: ${{ github.event_name == 'pull_request' && 'minimal' || 'all' }} + # The workflow dispatch event uses the same trigger as the schedule event. + trigger: ${{ github.event_name == 'pull_request' && 'commit' || (github.event_name == 'push' && 'merge' || 'schedule') }} # Build and upload the dependencies for each configuration. run-upload-conan-deps: @@ -59,8 +60,8 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} max-parallel: 10 - runs-on: ${{ matrix.architecture.runner }} - container: ${{ contains(matrix.architecture.platform, 'linux') && format('ghcr.io/xrplf/ci/{0}-{1}:{2}-{3}-sha-{4}', matrix.os.distro_name, matrix.os.distro_version, matrix.os.compiler_name, matrix.os.compiler_version, matrix.os.image_sha) || null }} + runs-on: ${{ matrix.runs_on }} + container: ${{ matrix.image }} steps: - name: Cleanup workspace (macOS and Windows) if: ${{ runner.os == 'macOS' || runner.os == 'Windows' }} diff --git a/.gitignore b/.gitignore index 55844462e5..35fd75b9ae 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ Release/ /tmp/ CMakeSettings.json CMakeUserPresets.json +__pycache__ # Coverage files. *.gcno