diff --git a/.github/scripts/strategy-matrix/generate.py b/.github/scripts/strategy-matrix/generate.py index c783f32fb7..3797e5881d 100755 --- a/.github/scripts/strategy-matrix/generate.py +++ b/.github/scripts/strategy-matrix/generate.py @@ -33,6 +33,10 @@ def get_cmake_args(build_type: str, extra_args: str) -> str: # Every config must declare 'minimal'. Minimal configs form the reduced matrix # built for pull requests by default; the full matrix adds the rest. Packaging # configs declare it too, but packaging is gated in the workflow, not by it. +# +# Configs may also opt into 'benchmark' to smoke-run the benchmarks. Note that +# the flag applies to every entry a config expands into, so only set it on +# configs that expand to a single combination. @dataclasses.dataclass @@ -43,6 +47,7 @@ class LinuxConfig: build_type: list[str] arch: list[str] minimal: bool + benchmark: bool = False # if true, smoke-run the benchmarks after testing sanitizers: list[str] = dataclasses.field(default_factory=list) suffix: str = "" extra_cmake_args: str = "" @@ -81,6 +86,7 @@ class PlatformConfig: build_type: list[str] minimal: bool build_only: bool = False # if true, skip tests (e.g. macos/Windows Debug) + benchmark: bool = False # if true, smoke-run the benchmarks after testing extra_cmake_args: str = "" def __post_init__(self) -> None: @@ -125,6 +131,7 @@ class MatrixEntry: cmake_args: str cmake_target: str build_only: bool + benchmark: bool build_type: str architecture: Architecture sanitizers: str @@ -193,6 +200,7 @@ def expand_linux_matrix(linux: LinuxFile, minimal: bool) -> list[MatrixEntry]: cmake_args=get_cmake_args(build_type, cfg.extra_cmake_args), cmake_target="all", build_only=False, + benchmark=cfg.benchmark, build_type=build_type, architecture=arch_info, sanitizers=sanitizer, @@ -245,6 +253,7 @@ def expand_platform_matrix(pf: PlatformFile, minimal: bool) -> list[MatrixEntry] cmake_args=get_cmake_args(build_type, cfg.extra_cmake_args), cmake_target="install" if is_windows else "all", build_only=cfg.build_only, + benchmark=cfg.benchmark, build_type=build_type, architecture=Architecture(platform=pf.platform, runner=pf.runner), sanitizers="", diff --git a/.github/scripts/strategy-matrix/linux.json b/.github/scripts/strategy-matrix/linux.json index 9510212344..159c76b6c2 100644 --- a/.github/scripts/strategy-matrix/linux.json +++ b/.github/scripts/strategy-matrix/linux.json @@ -14,7 +14,8 @@ "compiler": ["clang"], "build_type": ["Release"], "arch": ["amd64"], - "minimal": true + "minimal": true, + "benchmark": true }, { diff --git a/.github/workflows/reusable-build-test-config.yml b/.github/workflows/reusable-build-test-config.yml index 3023f70cdf..e21067cc5f 100644 --- a/.github/workflows/reusable-build-test-config.yml +++ b/.github/workflows/reusable-build-test-config.yml @@ -3,6 +3,12 @@ name: Build and test configuration on: workflow_call: inputs: + benchmark: + description: "Whether to smoke-run the benchmarks after testing." + required: false + type: boolean + default: false + build_only: description: 'Whether to only build or to build and test the code ("true", "false").' required: true @@ -328,11 +334,14 @@ jobs: # Smoke-run every benchmark module with a single repetition to confirm the # benchmarks still build and execute. This is a correctness check, not a - # performance measurement, so it is skipped for instrumented builds - # (sanitizers/coverage/voidstar), where it would be slow and meaningless, - # and on Windows, where the `install` target does not build them. + # performance measurement, so there is nothing to gain from repeating it + # across configurations: it is opted into by a single config in the + # strategy matrix (see the 'benchmark' flag in the JSON files), which + # keeps it off instrumented builds (sanitizers/coverage/voidstar), where + # it would be slow and meaningless, off Debug builds, where it is much + # slower, and off Windows, where the `install` target does not build them. - name: Run the benchmarks - if: ${{ !inputs.build_only && runner.os != 'Windows' && env.SANITIZERS_ENABLED == 'false' && env.COVERAGE_ENABLED != 'true' && env.VOIDSTAR_ENABLED != 'true' }} + if: ${{ inputs.benchmark }} working-directory: ${{ env.BUILD_DIR }} run: | rc=0 diff --git a/.github/workflows/reusable-build-test.yml b/.github/workflows/reusable-build-test.yml index 4b64c53521..5368274a16 100644 --- a/.github/workflows/reusable-build-test.yml +++ b/.github/workflows/reusable-build-test.yml @@ -40,6 +40,7 @@ jobs: fail-fast: ${{ github.event_name == 'merge_group' }} matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} with: + benchmark: ${{ matrix.benchmark }} build_only: ${{ matrix.build_only }} build_type: ${{ matrix.build_type }} ccache_enabled: ${{ inputs.ccache_enabled }}