rippled/.github/scripts/strategy-matrix/generate.py

#!/usr/bin/env python3
import argparse
import dataclasses
import itertools
import json
from pathlib import Path

THIS_DIR = Path(__file__).parent.resolve()

_BASE_CMAKE_ARGS = ["-Dtests=ON", "-Dwerr=ON", "-Dxrpld=ON", "-Dwextra=ON"]

# Maps sanitizer names (as used in cmake) to short config-name suffixes.
_SANITIZER_SUFFIX: dict[str, str] = {
    "address": "asan",
    "undefinedbehavior": "ubsan",
    "thread": "tsan",
}


def get_cmake_args(build_type: str, extra_args: str) -> str:
    """Get the full list of CMake arguments for a config."""
    args = _BASE_CMAKE_ARGS.copy()
    if extra_args:
        args.extend(extra_args.split())
    return " ".join(args)


# ---------------------------------------------------------------------------
# Input types — shapes of the JSON config files
# ---------------------------------------------------------------------------


# Every config must declare 'minimal'. Minimal configs form the reduced matrix
# built for pull requests by default; the full matrix adds the rest. Packaging
# configs declare it too, but packaging is gated in the workflow, not by it.
#
# Configs may also opt into 'benchmark' to smoke-run the benchmarks. Note that
# the flag applies to every entry a config expands into, so only set it on
# configs that expand to a single combination.


@dataclasses.dataclass
class LinuxConfig:
    """One entry in linux.json's 'configs' or 'package_configs' arrays."""

    compiler: list[str]
    build_type: list[str]
    arch: list[str]
    minimal: bool
    benchmark: bool = False  # if true, smoke-run the benchmarks after testing
    sanitizers: list[str] = dataclasses.field(default_factory=list)
    suffix: str = ""
    extra_cmake_args: str = ""
    image: str = ""  # only used by package_configs entries


@dataclasses.dataclass
class LinuxFile:
    """Shape of linux.json."""

    image_tag: str
    configs: dict[str, list[LinuxConfig]]  # distro → configs
    package_configs: dict[str, list[LinuxConfig]]  # distro → packaging configs

    @classmethod
    def load(cls, path: Path) -> "LinuxFile":
        data = json.loads(path.read_text())

        def parse(section: dict) -> dict[str, list[LinuxConfig]]:
            return {
                distro: [LinuxConfig(**c) for c in cfgs]
                for distro, cfgs in section.items()
            }

        return cls(
            image_tag=data["image_tag"],
            configs=parse(data["configs"]),
            package_configs=parse(data.get("package_configs", {})),
        )


@dataclasses.dataclass
class PlatformConfig:
    """One entry in macos.json's or windows.json's 'configs' array."""

    build_type: list[str]
    minimal: bool
    build_only: bool = False  # if true, skip tests (e.g. macos/Windows Debug)
    benchmark: bool = False  # if true, smoke-run the benchmarks after testing
    extra_cmake_args: str = ""

    def __post_init__(self) -> None:
        if isinstance(self.build_type, str):
            self.build_type = [self.build_type]


@dataclasses.dataclass
class PlatformFile:
    """Shape of macos.json and windows.json."""

    platform: str  # e.g. "macos/arm64" or "windows/amd64"
    runner: list[str]  # GitHub Actions runner labels
    configs: list[PlatformConfig]

    @classmethod
    def load(cls, path: Path) -> "PlatformFile":
        data = json.loads(path.read_text())
        return cls(
            platform=data["platform"],
            runner=data["runner"],
            configs=[PlatformConfig(**c) for c in data["configs"]],
        )


# ---------------------------------------------------------------------------
# Output types — shapes of the generated GitHub Actions matrix entries
# ---------------------------------------------------------------------------


@dataclasses.dataclass
class Architecture:
    platform: str
    runner: list[str]


@dataclasses.dataclass
class MatrixEntry:
    """One entry in the generated build/test strategy matrix."""

    config_name: str
    cmake_args: str
    cmake_target: str
    build_only: bool
    benchmark: bool
    build_type: str
    architecture: Architecture
    sanitizers: str
    image: str = ""  # container image; empty for macOS/Windows (runs natively)
    compiler: str = ""  # compiler name ("gcc" or "clang"); empty for macOS/Windows


@dataclasses.dataclass
class PackagingEntry:
    """One entry in the generated packaging strategy matrix."""

    xrpld_artifact_name: str
    validator_keys_artifact_name: str
    image: str
    distro: str  # e.g. "debian" or "rhel"; drives package-format-specific steps


# ---------------------------------------------------------------------------
# Matrix expansion
# ---------------------------------------------------------------------------

_ARCHS: dict[str, Architecture] = {
    "amd64": Architecture(
        platform="linux/amd64", runner=["self-hosted", "Linux", "X64", "heavy"]
    ),
    "arm64": Architecture(
        platform="linux/arm64",
        runner=["self-hosted", "Linux", "ARM64", "heavy-arm64"],
    ),
}


def expand_linux_matrix(linux: LinuxFile, minimal: bool) -> list[MatrixEntry]:
    """Expand a LinuxFile into a flat list of matrix entries.

    Each config entry is expanded over the cross-product of its
    compiler, build_type, sanitizers, and architecture lists. When 'minimal' is
    true, only configs flagged as minimal are included.
    """
    entries: list[MatrixEntry] = []

    for distro, configs in linux.configs.items():
        for cfg in configs:
            if minimal and not cfg.minimal:
                continue
            # An empty sanitizers list means "one entry with no sanitizer".
            effective_sanitizers = cfg.sanitizers or [""]
            effective_archs = {arch: _ARCHS[arch] for arch in cfg.arch}

            for compiler, build_type, sanitizer, (arch, arch_info) in itertools.product(
                cfg.compiler,
                cfg.build_type,
                effective_sanitizers,
                effective_archs.items(),
            ):
                name = f"{distro}-{compiler}-{build_type.lower()}-{arch}"
                suffix_parts = [
                    s for s in [cfg.suffix, _SANITIZER_SUFFIX.get(sanitizer, "")] if s
                ]
                if suffix_parts:
                    name += "-" + "-".join(suffix_parts)

                entries.append(
                    MatrixEntry(
                        config_name=name,
                        image=f"ghcr.io/xrplf/xrpld/nix-{distro}:{linux.image_tag}",
                        cmake_args=get_cmake_args(build_type, cfg.extra_cmake_args),
                        cmake_target="all",
                        build_only=False,
                        benchmark=cfg.benchmark,
                        build_type=build_type,
                        architecture=arch_info,
                        sanitizers=sanitizer,
                        compiler=compiler,
                    )
                )

    return entries


def expand_linux_packaging(linux: LinuxFile) -> list[PackagingEntry]:
    """Generate the packaging matrix from a LinuxFile's package_configs section.

    Packaging uses vanilla distro images (debian:bookworm, ubi9, …) instead of
    the nix-based build images, because deb/rpm tooling (debhelper, rpm-build)
    is taken from the distro's archive rather than from nixpkgs. Each config
    entry carries its own 'image'.

    The artifact names must match what the build job uploads: one artifact per
    binary, each named after the build config.
    """
    entries = []
    for distro, configs in linux.package_configs.items():
        for cfg in configs:
            for compiler, build_type in itertools.product(cfg.compiler, cfg.build_type):
                config_name = f"{distro}-{compiler}-{build_type.lower()}-amd64"
                entries.append(
                    PackagingEntry(
                        xrpld_artifact_name=f"xrpld-{config_name}",
                        validator_keys_artifact_name=f"validator-keys-{config_name}",
                        image=cfg.image,
                        distro=distro,
                    )
                )

    return entries


def expand_platform_matrix(pf: PlatformFile, minimal: bool) -> list[MatrixEntry]:
    """Expand a PlatformFile (macOS or Windows) into matrix entries.

    When 'minimal' is true, only configs flagged as minimal are included.
    """
    platform_name, arch = pf.platform.split("/")
    is_windows = platform_name == "windows"

    entries: list[MatrixEntry] = []
    for cfg in pf.configs:
        if minimal and not cfg.minimal:
            continue
        for build_type in cfg.build_type:
            entries.append(
                MatrixEntry(
                    config_name=f"{platform_name}-{arch}-{build_type.lower()}",
                    cmake_args=get_cmake_args(build_type, cfg.extra_cmake_args),
                    cmake_target="install" if is_windows else "all",
                    build_only=cfg.build_only,
                    benchmark=cfg.benchmark,
                    build_type=build_type,
                    architecture=Architecture(platform=pf.platform, runner=pf.runner),
                    sanitizers="",
                )
            )
    return entries


# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Generate a CI strategy matrix for all platforms or a specific one."
    )
    parser.add_argument(
        "-c",
        "--config",
        help="Platform to generate for ('linux', 'macos', or 'windows'). Defaults to all platforms.",
        choices=["linux", "macos", "windows"],
        default=None,
    )
    parser.add_argument(
        "-p",
        "--packaging",
        help="Emit the Linux packaging matrix instead of the build/test matrix.",
        action="store_true",
    )
    parser.add_argument(
        "-m",
        "--minimal",
        help="Emit only the minimal matrix (the configs flagged 'minimal'), "
        "used for pull requests by default. If omitted, the full matrix is "
        "emitted.",
        action="store_true",
    )
    args = parser.parse_args()

    matrix: list[MatrixEntry] | list[PackagingEntry] = []

    if args.packaging:
        matrix = expand_linux_packaging(LinuxFile.load(THIS_DIR / "linux.json"))
    else:
        if args.config in ("linux", None):
            matrix += expand_linux_matrix(
                LinuxFile.load(THIS_DIR / "linux.json"), args.minimal
            )
        if args.config in ("macos", None):
            matrix += expand_platform_matrix(
                PlatformFile.load(THIS_DIR / "macos.json"), args.minimal
            )
        if args.config in ("windows", None):
            matrix += expand_platform_matrix(
                PlatformFile.load(THIS_DIR / "windows.json"), args.minimal
            )

    print(f"matrix={json.dumps({'include': [dataclasses.asdict(e) for e in matrix]})}")