ci: Use a separate benchmark filter (#7919)

.github/scripts/strategy-matrix/generate.py

@@ -33,6 +33,10 @@ def get_cmake_args(build_type: str, extra_args: str) -> str:
 # Every config must declare 'minimal'. Minimal configs form the reduced matrix
 # built for pull requests by default; the full matrix adds the rest. Packaging
 # configs declare it too, but packaging is gated in the workflow, not by it.
+#
+# Configs may also opt into 'benchmark' to smoke-run the benchmarks. Note that
+# the flag applies to every entry a config expands into, so only set it on
+# configs that expand to a single combination.
 
 
 @dataclasses.dataclass
@@ -43,6 +47,7 @@ class LinuxConfig:
     build_type: list[str]
     arch: list[str]
     minimal: bool
+    benchmark: bool = False  # if true, smoke-run the benchmarks after testing
     sanitizers: list[str] = dataclasses.field(default_factory=list)
     suffix: str = ""
     extra_cmake_args: str = ""
@@ -81,6 +86,7 @@ class PlatformConfig:
     build_type: list[str]
     minimal: bool
     build_only: bool = False  # if true, skip tests (e.g. macos/Windows Debug)
+    benchmark: bool = False  # if true, smoke-run the benchmarks after testing
     extra_cmake_args: str = ""
 
     def __post_init__(self) -> None:
@@ -125,6 +131,7 @@ class MatrixEntry:
     cmake_args: str
     cmake_target: str
     build_only: bool
+    benchmark: bool
     build_type: str
     architecture: Architecture
     sanitizers: str
@@ -193,6 +200,7 @@ def expand_linux_matrix(linux: LinuxFile, minimal: bool) -> list[MatrixEntry]:
                         cmake_args=get_cmake_args(build_type, cfg.extra_cmake_args),
                         cmake_target="all",
                         build_only=False,
+                        benchmark=cfg.benchmark,
                         build_type=build_type,
                         architecture=arch_info,
                         sanitizers=sanitizer,
@@ -245,6 +253,7 @@ def expand_platform_matrix(pf: PlatformFile, minimal: bool) -> list[MatrixEntry]
                     cmake_args=get_cmake_args(build_type, cfg.extra_cmake_args),
                     cmake_target="install" if is_windows else "all",
                     build_only=cfg.build_only,
+                    benchmark=cfg.benchmark,
                     build_type=build_type,
                     architecture=Architecture(platform=pf.platform, runner=pf.runner),
                     sanitizers="",

.github/scripts/strategy-matrix/linux.json

@@ -14,7 +14,8 @@
         "compiler": ["clang"],
         "build_type": ["Release"],
         "arch": ["amd64"],
-        "minimal": true
+        "minimal": true,
+        "benchmark": true
       },
 
       {

.github/workflows/reusable-build-test-config.yml

@@ -3,6 +3,12 @@ name: Build and test configuration
 on:
   workflow_call:
     inputs:
+      benchmark:
+        description: "Whether to smoke-run the benchmarks after testing."
+        required: false
+        type: boolean
+        default: false
+
       build_only:
         description: 'Whether to only build or to build and test the code ("true", "false").'
         required: true
@@ -328,11 +334,14 @@ jobs:
 
       # Smoke-run every benchmark module with a single repetition to confirm the
       # benchmarks still build and execute. This is a correctness check, not a
-      # performance measurement, so it is skipped for instrumented builds
-      # (sanitizers/coverage/voidstar), where it would be slow and meaningless,
-      # and on Windows, where the `install` target does not build them.
+      # performance measurement, so there is nothing to gain from repeating it
+      # across configurations: it is opted into by a single config in the
+      # strategy matrix (see the 'benchmark' flag in the JSON files), which
+      # keeps it off instrumented builds (sanitizers/coverage/voidstar), where
+      # it would be slow and meaningless, off Debug builds, where it is much
+      # slower, and off Windows, where the `install` target does not build them.
       - name: Run the benchmarks
-        if: ${{ !inputs.build_only && runner.os != 'Windows' && env.SANITIZERS_ENABLED == 'false' && env.COVERAGE_ENABLED != 'true' && env.VOIDSTAR_ENABLED != 'true' }}
+        if: ${{ inputs.benchmark }}
         working-directory: ${{ env.BUILD_DIR }}
         run: |
           rc=0

.github/workflows/reusable-build-test.yml

@@ -40,6 +40,7 @@ jobs:
       fail-fast: ${{ github.event_name == 'merge_group' }}
       matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
     with:
+      benchmark: ${{ matrix.benchmark }}
       build_only: ${{ matrix.build_only }}
       build_type: ${{ matrix.build_type }}
       ccache_enabled: ${{ inputs.ccache_enabled }}