From aab3305abee51698e682c47e57df4d39613ead69 Mon Sep 17 00:00:00 2001 From: Michael Legleux Date: Tue, 3 Feb 2026 15:19:59 -0800 Subject: [PATCH] use currenty generate.py --- .github/scripts/strategy-matrix/generate.py | 293 +++++++++++++------- 1 file changed, 200 insertions(+), 93 deletions(-) diff --git a/.github/scripts/strategy-matrix/generate.py b/.github/scripts/strategy-matrix/generate.py index ac21544f14..0e44b1be54 100755 --- a/.github/scripts/strategy-matrix/generate.py +++ b/.github/scripts/strategy-matrix/generate.py @@ -29,20 +29,17 @@ We will further set additional CMake arguments as follows: - All release builds will have the `assert` option. - Certain Debian Bookworm configurations will change the reference fee, enable codecov, and enable voidstar in PRs. -''' -def generate_strategy_matrix(all_configs: bool, config: Config) -> list: +""" + + +def generate_strategy_matrix(all: bool, config: Config) -> list: configurations = [] for architecture, os, build_type, cmake_args in itertools.product( config.architecture, config.os, config.build_type, config.cmake_args ): # The default CMake target is 'all' for Linux and MacOS and 'install' # for Windows, but it can get overridden for certain configurations. - - arch = architecture['platform'].split('/')[-1] - compiler = f'{os['compiler_name']}-{os['compiler_version']}' - distro = f'{os['distro_name']}-{os['distro_version']}' - cmake_target = 'install' if os["distro_name"] == 'windows' else 'all' - build_package = False + cmake_target = "install" if os["distro_name"] == "windows" else "all" # We build and test all configurations by default, except for Windows in # Debug, because it is too slow, as well as when code coverage is @@ -51,15 +48,8 @@ def generate_strategy_matrix(all_configs: bool, config: Config) -> list: if os["distro_name"] == "windows" and build_type == "Debug": build_only = True - if build_type == "Release" \ - and arch == 'amd64' \ - and cmake_args == '-Dunity=OFF' \ - and compiler == "gcc-12" \ - and distro in ('debian-bullseye', 'rhel-9.4'): - build_package = True - # Only generate a subset of configurations in PRs. - if not all_configs and not build_package: + if not all: # Debian: # - Bookworm using GCC 13: Release and Unity on linux/amd64, set # the reference fee to 500. @@ -72,38 +62,70 @@ def generate_strategy_matrix(all_configs: bool, config: Config) -> list: # - Bookworm using Clang 20: Debug and Unity on linux/amd64. if os["distro_name"] == "debian": skip = True - if os['distro_version'] == 'bookworm': - if compiler == 'gcc-13' and build_type == 'Release' and '-Dunity=ON' in cmake_args and arch == 'amd64': - cmake_args = f'-DUNIT_TEST_REFERENCE_FEE=500 {cmake_args}' + if os["distro_version"] == "bookworm": + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-13" + and build_type == "Release" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "linux/amd64" + ): + cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=500 {cmake_args}" skip = False - if compiler == 'gcc-15' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and arch == 'amd64': + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15" + and build_type == "Debug" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "linux/amd64" + ): skip = False - if compiler == 'clang-16' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and arch =='arm64': - cmake_args = f'-Dvoidstar=ON {cmake_args}' + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "clang-16" + and build_type == "Debug" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "linux/arm64" + ): + cmake_args = f"-Dvoidstar=ON {cmake_args}" skip = False - if compiler == 'clang-17' and build_type == 'Release' and '-Dunity=ON' in cmake_args and arch == 'amd64': - cmake_args = f'-DUNIT_TEST_REFERENCE_FEE=1000 {cmake_args}' + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "clang-17" + and build_type == "Release" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "linux/amd64" + ): + cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=1000 {cmake_args}" skip = False - if compiler == 'clang-20' and build_type == 'Debug' and '-Dunity=ON' in cmake_args and arch == 'amd64': + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "clang-20" + and build_type == "Debug" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "linux/amd64" + ): skip = False - if skip and not args.package: + if skip: continue # RHEL: - # - 9.4 using GCC 12: Debug and Unity on linux/amd64. - # - 9.6 using Clang: Release and no Unity on linux/amd64. - - if os['distro_name'] == 'rhel': + # - 9 using GCC 12: Debug and Unity on linux/amd64. + # - 10 using Clang: Release and no Unity on linux/amd64. + if os["distro_name"] == "rhel": skip = True - if os['distro_version'] == '9.4': - if compiler == 'gcc-12' and build_type == 'Release' and '-Dunity=OFF' in cmake_args and arch == 'amd64': + if os["distro_version"] == "9": + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12" + and build_type == "Debug" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "linux/amd64" + ): skip = False - if compiler == 'gcc-12' and build_type == 'Debug' and '-Dunity=ON' in cmake_args and arch == 'amd64': + elif os["distro_version"] == "10": + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "clang-any" + and build_type == "Release" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "linux/amd64" + ): skip = False - elif os['distro_version'] == '9.6': - if compiler == 'clang-any' and build_type == 'Release' and '-Dunity=OFF' in cmake_args and arch == 'amd64': - skip = False - if skip and not build_package: + if skip: continue # Ubuntu: @@ -113,15 +135,35 @@ def generate_strategy_matrix(all_configs: bool, config: Config) -> list: # - Noble using Clang 19: Release and Unity on linux/arm64. if os["distro_name"] == "ubuntu": skip = True - if os['distro_version'] == 'jammy': - if compiler == 'gcc-12' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and arch =='arm64': + if os["distro_version"] == "jammy": + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12" + and build_type == "Debug" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "linux/arm64" + ): skip = False - elif os['distro_version'] == 'noble': - if compiler == 'gcc-14' and build_type == 'Release' and '-Dunity=ON' in cmake_args and arch == 'amd64': + elif os["distro_version"] == "noble": + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-14" + and build_type == "Release" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "linux/amd64" + ): skip = False - if compiler == 'clang-18' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and arch == 'amd64': + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "clang-18" + and build_type == "Debug" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "linux/amd64" + ): skip = False - if compiler == 'clang-19' and build_type == 'Release' and '-Dunity=ON' in cmake_args and arch =='arm64': + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "clang-19" + and build_type == "Release" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "linux/arm64" + ): skip = False if skip: continue @@ -145,86 +187,151 @@ def generate_strategy_matrix(all_configs: bool, config: Config) -> list: continue # Additional CMake arguments. - cmake_args = f'{cmake_args} -Dtests=ON -Dxrpld=ON' - if compiler not in ('gcc-12', 'clang-16'): - cmake_args = f'{cmake_args} -Dwerr -Dwextra=ON' - - if build_type == 'Release': - if build_package: - cmake_args = f'{cmake_args} -Dvalidator-keys=ON' - cmake_target = 'rippled validator-keys' - else: - cmake_args = f'{cmake_args} -Dwerr -Dassert=ON' + cmake_args = f"{cmake_args} -Dtests=ON -Dwerr=ON -Dxrpld=ON" + if not f"{os['compiler_name']}-{os['compiler_version']}" in [ + "gcc-12", + "clang-16", + ]: + cmake_args = f"{cmake_args} -Dwextra=ON" + if build_type == "Release": + cmake_args = f"{cmake_args} -Dassert=ON" # We skip all RHEL on arm64 due to a build failure that needs further # investigation. - if os['distro_name'] == 'rhel' and arch =='arm64': + if os["distro_name"] == "rhel" and architecture["platform"] == "linux/arm64": continue - # We skip all clang-20 on arm64 due to boost 1.86 build error - if compiler == 'clang-20' and arch =='arm64': + # We skip all clang 20+ on arm64 due to Boost build error. + if ( + f"{os['compiler_name']}-{os['compiler_version']}" + in ["clang-20", "clang-21"] + and architecture["platform"] == "linux/arm64" + ): continue # Enable code coverage for Debian Bookworm using GCC 15 in Debug and no # Unity on linux/amd64 - if compiler == 'gcc-15' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and arch == 'amd64': - cmake_args = f'-Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0 {cmake_args}' - cmake_target = 'coverage' - build_only = True + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15" + and build_type == "Debug" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "linux/amd64" + ): + cmake_args = f"-Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0 {cmake_args}" # Generate a unique name for the configuration, e.g. macos-arm64-debug # or debian-bookworm-gcc-12-amd64-release-unity. - config_name = os['distro_name'] - if (n := os['distro_version']) != '': - config_name += f'-{n}' - if (n := os['compiler_name']) != '': - config_name += f'-{n}' - if (n := os['compiler_version']) != '': - config_name += f'-{n}' - config_name += f'-{arch}' - config_name += f'-{build_type.lower()}' - - if '-Dunity=ON' in cmake_args: - config_name += '-unity' + config_name = os["distro_name"] + if (n := os["distro_version"]) != "": + config_name += f"-{n}" + if (n := os["compiler_name"]) != "": + config_name += f"-{n}" + if (n := os["compiler_version"]) != "": + config_name += f"-{n}" + config_name += ( + f"-{architecture['platform'][architecture['platform'].find('/')+1:]}" + ) + config_name += f"-{build_type.lower()}" + if "-Dcoverage=ON" in cmake_args: + config_name += "-coverage" + if "-Dunity=ON" in cmake_args: + config_name += "-unity" # Add the configuration to the list, with the most unique fields first, # so that they are easier to identify in the GitHub Actions UI, as long # names get truncated. - if args.package and not build_package: - continue - configurations.append({ - 'config_name': config_name, - 'cmake_args': cmake_args, - 'cmake_target': cmake_target, - 'build_only': build_only, - 'build_type': build_type, - 'package': build_package, - 'os': os, - 'architecture': architecture, - }) + # Add Address and Thread (both coupled with UB) sanitizers for specific bookworm distros. + # GCC-Asan rippled-embedded tests are failing because of https://github.com/google/sanitizers/issues/856 + if ( + os["distro_version"] == "bookworm" + and f"{os['compiler_name']}-{os['compiler_version']}" == "clang-20" + ): + # Add ASAN + UBSAN configuration. + configurations.append( + { + "config_name": config_name + "-asan-ubsan", + "cmake_args": cmake_args, + "cmake_target": cmake_target, + "build_only": build_only, + "build_type": build_type, + "os": os, + "architecture": architecture, + "sanitizers": "address,undefinedbehavior", + } + ) + # TSAN is deactivated due to seg faults with latest compilers. + activate_tsan = False + if activate_tsan: + configurations.append( + { + "config_name": config_name + "-tsan-ubsan", + "cmake_args": cmake_args, + "cmake_target": cmake_target, + "build_only": build_only, + "build_type": build_type, + "os": os, + "architecture": architecture, + "sanitizers": "thread,undefinedbehavior", + } + ) + else: + configurations.append( + { + "config_name": config_name, + "cmake_args": cmake_args, + "cmake_target": cmake_target, + "build_only": build_only, + "build_type": build_type, + "os": os, + "architecture": architecture, + "sanitizers": "", + } + ) return configurations def read_config(file: Path) -> Config: config = json.loads(file.read_text()) + if ( + config["architecture"] is None + or config["os"] is None + or config["build_type"] is None + or config["cmake_args"] is None + ): + raise Exception("Invalid configuration file.") + return Config(**config) if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-a', '--all', help='Set to generate all configurations (generally used when merging a PR) or leave unset to generate a subset of configurations (generally used when committing to a PR).', action="store_true") - parser.add_argument('-c', '--config', type=Path, help='Path to the JSON file containing the strategy matrix configurations.') - parser.add_argument('-p', '--package', action='store_true', help='Build packages') + parser.add_argument( + "-a", + "--all", + help="Set to generate all configurations (generally used when merging a PR) or leave unset to generate a subset of configurations (generally used when committing to a PR).", + action="store_true", + ) + parser.add_argument( + "-c", + "--config", + help="Path to the JSON file containing the strategy matrix configurations.", + required=False, + type=Path, + ) args = parser.parse_args() matrix = [] - if args.package: - matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "linux.json")) - elif args.config is None or args.config == '': - matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "linux.json")) - # matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "macos.json")) - # matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "windows.json")) + if args.config is None or args.config == "": + matrix += generate_strategy_matrix( + args.all, read_config(THIS_DIR / "linux.json") + ) + matrix += generate_strategy_matrix( + args.all, read_config(THIS_DIR / "macos.json") + ) + matrix += generate_strategy_matrix( + args.all, read_config(THIS_DIR / "windows.json") + ) else: matrix += generate_strategy_matrix(args.all, read_config(args.config))