ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2026-07-27 00:50:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into pratik/Add-sanitizers-to-CI-builds

Browse Source 
Signed-off-by: Pratik Mankawde <pratikmankawde@gmail.com>
This commit is contained in:
Pratik Mankawde
2025-11-27 13:21:51 +00:00
parent d942fd9f9c f399749ee2
commit c182d13ab5
11 changed files with 517 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

388
.github/scripts/strategy-matrix/generate.py vendored
View File

@@ -2,11 +2,12 @@
import argparse
import itertools
import json
from pathlib import Path
from dataclasses import dataclass
from pathlib import Path
THIS_DIR = Path(__file__).parent.resolve()
@dataclass
class Config:
architecture: list[dict]
@@ -14,7 +15,8 @@ class Config:
build_type: list[str]
cmake_args: list[str]
'''
"""
Generate a strategy matrix for GitHub Actions CI.
On each PR commit we will build a selection of Debian, RHEL, Ubuntu, MacOS, and
@@ -27,19 +29,23 @@ We will further set additional CMake arguments as follows:
- All release builds will have the `assert` option.
- Certain Debian Bookworm configurations will change the reference fee, enable
codecov, and enable voidstar in PRs.
'''
"""
def generate_strategy_matrix(all: bool, config: Config) -> list:
configurations = []
for architecture, os, build_type, cmake_args in itertools.product(config.architecture, config.os, config.build_type, config.cmake_args):
for architecture, os, build_type, cmake_args in itertools.product(
config.architecture, config.os, config.build_type, config.cmake_args
):
# The default CMake target is 'all' for Linux and MacOS and 'install'
# for Windows, but it can get overridden for certain configurations.
cmake_target = 'install' if os["distro_name"] == 'windows' else 'all'
cmake_target = "install" if os["distro_name"] == "windows" else "all"
# We build and test all configurations by default, except for Windows in
# Debug, because it is too slow, as well as when code coverage is
# enabled as that mode already runs the tests.
build_only = False
if os['distro_name'] == 'windows' and build_type == 'Debug':
if os["distro_name"] == "windows" and build_type == "Debug":
build_only = True
# Only generate a subset of configurations in PRs.
@@ -54,21 +60,46 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
# - Bookworm using Clang 17: Release and no Unity on linux/amd64,
# set the reference fee to 1000.
# - Bookworm using Clang 20: Debug and Unity on linux/amd64.
if os['distro_name'] == 'debian':
if os["distro_name"] == "debian":
skip = True
if os['distro_version'] == 'bookworm':
if f'{os["compiler_name"]}-{os["compiler_version"]}' == 'gcc-13' and build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64':
cmake_args = f'-DUNIT_TEST_REFERENCE_FEE=500 {cmake_args}'
if os["distro_version"] == "bookworm":
if (
f'{os["compiler_name"]}-{os["compiler_version"]}' == "gcc-13"
and build_type == "Release"
and "-Dunity=ON" in cmake_args
and architecture["platform"] == "linux/amd64"
):
cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=500 {cmake_args}"
skip = False
if f'{os["compiler_name"]}-{os["compiler_version"]}' == 'gcc-15' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/amd64':
if (
f'{os["compiler_name"]}-{os["compiler_version"]}' == "gcc-15"
and build_type == "Debug"
and "-Dunity=OFF" in cmake_args
and architecture["platform"] == "linux/amd64"
):
skip = False
if f'{os["compiler_name"]}-{os["compiler_version"]}' == 'clang-16' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/arm64':
cmake_args = f'-Dvoidstar=ON {cmake_args}'
if (
f'{os["compiler_name"]}-{os["compiler_version"]}' == "clang-16"
and build_type == "Debug"
and "-Dunity=OFF" in cmake_args
and architecture["platform"] == "linux/arm64"
):
cmake_args = f"-Dvoidstar=ON {cmake_args}"
skip = False
if f'{os["compiler_name"]}-{os["compiler_version"]}' == 'clang-17' and build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64':
cmake_args = f'-DUNIT_TEST_REFERENCE_FEE=1000 {cmake_args}'
if (
f'{os["compiler_name"]}-{os["compiler_version"]}' == "clang-17"
and build_type == "Release"
and "-Dunity=ON" in cmake_args
and architecture["platform"] == "linux/amd64"
):
cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=1000 {cmake_args}"
skip = False
if f'{os["compiler_name"]}-{os["compiler_version"]}' == 'clang-20' and build_type == 'Debug' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64':
if (
f'{os["compiler_name"]}-{os["compiler_version"]}' == "clang-20"
and build_type == "Debug"
and "-Dunity=ON" in cmake_args
and architecture["platform"] == "linux/amd64"
):
skip = False
if skip:
continue
@@ -76,13 +107,23 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
# RHEL:
# - 9 using GCC 12: Debug and Unity on linux/amd64.
# - 10 using Clang: Release and no Unity on linux/amd64.
if os['distro_name'] == 'rhel':
if os["distro_name"] == "rhel":
skip = True
if os['distro_version'] == '9':
if f'{os["compiler_name"]}-{os["compiler_version"]}' == 'gcc-12' and build_type == 'Debug' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64':
if os["distro_version"] == "9":
if (
f'{os["compiler_name"]}-{os["compiler_version"]}' == "gcc-12"
and build_type == "Debug"
and "-Dunity=ON" in cmake_args
and architecture["platform"] == "linux/amd64"
):
skip = False
elif os['distro_version'] == '10':
if f'{os["compiler_name"]}-{os["compiler_version"]}' == 'clang-any' and build_type == 'Release' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/amd64':
elif os["distro_version"] == "10":
if (
f'{os["compiler_name"]}-{os["compiler_version"]}' == "clang-any"
and build_type == "Release"
and "-Dunity=OFF" in cmake_args
and architecture["platform"] == "linux/amd64"
):
skip = False
if skip:
continue
@@ -92,126 +133,202 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
# - Noble using GCC 14: Release and Unity on linux/amd64.
# - Noble using Clang 18: Debug and no Unity on linux/amd64.
# - Noble using Clang 19: Release and Unity on linux/arm64.
if os['distro_name'] == 'ubuntu':
if os["distro_name"] == "ubuntu":
skip = True
if os['distro_version'] == 'jammy':
if f'{os["compiler_name"]}-{os["compiler_version"]}' == 'gcc-12' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/arm64':
if os["distro_version"] == "jammy":
if (
f'{os["compiler_name"]}-{os["compiler_version"]}' == "gcc-12"
and build_type == "Debug"
and "-Dunity=OFF" in cmake_args
and architecture["platform"] == "linux/arm64"
):
skip = False
elif os['distro_version'] == 'noble':
if f'{os["compiler_name"]}-{os["compiler_version"]}' == 'gcc-14' and build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64':
elif os["distro_version"] == "noble":
if (
f'{os["compiler_name"]}-{os["compiler_version"]}' == "gcc-14"
and build_type == "Release"
and "-Dunity=ON" in cmake_args
and architecture["platform"] == "linux/amd64"
):
skip = False
if f'{os["compiler_name"]}-{os["compiler_version"]}' == 'clang-18' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/amd64':
if (
f'{os["compiler_name"]}-{os["compiler_version"]}' == "clang-18"
and build_type == "Debug"
and "-Dunity=OFF" in cmake_args
and architecture["platform"] == "linux/amd64"
):
skip = False
if f'{os["compiler_name"]}-{os["compiler_version"]}' == 'clang-19' and build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/arm64':
if (
f'{os["compiler_name"]}-{os["compiler_version"]}' == "clang-19"
and build_type == "Release"
and "-Dunity=ON" in cmake_args
and architecture["platform"] == "linux/arm64"
):
skip = False
if skip:
continue
# MacOS:
# - Debug and no Unity on macos/arm64.
if os['distro_name'] == 'macos' and not (build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'macos/arm64'):
if os["distro_name"] == "macos" and not (
build_type == "Debug"
and "-Dunity=OFF" in cmake_args
and architecture["platform"] == "macos/arm64"
):
continue
# Windows:
# - Release and Unity on windows/amd64.
if os['distro_name'] == 'windows' and not (build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'windows/amd64'):
if os["distro_name"] == "windows" and not (
build_type == "Release"
and "-Dunity=ON" in cmake_args
and architecture["platform"] == "windows/amd64"
):
continue
# Additional CMake arguments.
cmake_args = f'{cmake_args} -Dtests=ON -Dwerr=ON -Dxrpld=ON'
if not f'{os["compiler_name"]}-{os["compiler_version"]}' in ['gcc-12', 'clang-16']:
cmake_args = f'{cmake_args} -Dwextra=ON'
if build_type == 'Release':
cmake_args = f'{cmake_args} -Dassert=ON'
cmake_args = f"{cmake_args} -Dtests=ON -Dwerr=ON -Dxrpld=ON"
if not f'{os["compiler_name"]}-{os["compiler_version"]}' in [
"gcc-12",
"clang-16",
]:
cmake_args = f"{cmake_args} -Dwextra=ON"
if build_type == "Release":
cmake_args = f"{cmake_args} -Dassert=ON"
# We skip all RHEL on arm64 due to a build failure that needs further
# investigation.
if os['distro_name'] == 'rhel' and architecture['platform'] == 'linux/arm64':
if os["distro_name"] == "rhel" and architecture["platform"] == "linux/arm64":
continue
# We skip all clang 20+ on arm64 due to Boost build error.
if f'{os["compiler_name"]}-{os["compiler_version"]}' in ['clang-20', 'clang-21'] and architecture['platform'] == 'linux/arm64':
if (
f'{os["compiler_name"]}-{os["compiler_version"]}'
in ["clang-20", "clang-21"]
and architecture["platform"] == "linux/arm64"
):
continue
cxx_flags = '-g'
cxx_flags = "-g"
# Enable code coverage for Debian Bookworm using GCC 14 in Debug and no
# Unity on linux/amd64
if f'{os["compiler_name"]}-{os["compiler_version"]}' == 'gcc-14' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/amd64':
cmake_args = f'-Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 {cmake_args}'
cxx_flags = f'-O0 {cxx_flags}'
if (
f'{os["compiler_name"]}-{os["compiler_version"]}' == "gcc-14"
and build_type == "Debug"
and "-Dunity=OFF" in cmake_args
and architecture["platform"] == "linux/amd64"
):
cmake_args = f"-Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 {cmake_args}"
cxx_flags = f"-O0 {cxx_flags}"
# Generate a unique name for the configuration, e.g. macos-arm64-debug
# or debian-bookworm-gcc-12-amd64-release-unity.
config_name = os['distro_name']
if (n := os['distro_version']) != '':
config_name += f'-{n}'
if (n := os['compiler_name']) != '':
config_name += f'-{n}'
if (n := os['compiler_version']) != '':
config_name += f'-{n}'
config_name += f'-{architecture["platform"][architecture["platform"].find("/")+1:]}'
config_name += f'-{build_type.lower()}'
if '-Dunity=ON' in cmake_args:
config_name += '-unity'
config_name = os["distro_name"]
if (n := os["distro_version"]) != "":
config_name += f"-{n}"
if (n := os["compiler_name"]) != "":
config_name += f"-{n}"
if (n := os["compiler_version"]) != "":
config_name += f"-{n}"
config_name += (
f'-{architecture["platform"][architecture["platform"].find("/")+1:]}'
)
config_name += f"-{build_type.lower()}"
if "-Dunity=ON" in cmake_args:
config_name += "-unity"
# Add the configuration to the list, with the most unique fields first,
# so that they are easier to identify in the GitHub Actions UI, as long
# names get truncated.
# Add Address and Thread (both coupled with UB) sanitizers when the distro is bookworm.
if os['distro_version'] == 'bookworm' and f'{os["compiler_name"]}-{os["compiler_version"]}' in {'gcc-15', 'clang-20'}:
addSanitizerConfigs(architecture, os, build_type, cmake_args, config_name, build_only, cmake_target, cxx_flags, configurations)
if os[
"distro_version"
] == "bookworm" and f'{os["compiler_name"]}-{os["compiler_version"]}' in {
"gcc-15",
"clang-20",
}:
addSanitizerConfigs(
architecture,
os,
build_type,
cmake_args,
config_name,
build_only,
cmake_target,
cxx_flags,
configurations,
)
else:
if cxx_flags:
cmake_args_flags = f'{cmake_args} -DCMAKE_CXX_FLAGS={cxx_flags}'
cmake_args_flags = f"{cmake_args} -DCMAKE_CXX_FLAGS={cxx_flags}"
else:
cmake_args_flags = f'{cmake_args}'
configurations.append({
'config_name': config_name,
'cmake_args': cmake_args_flags,
'cmake_target': cmake_target,
'build_only': build_only,
'build_type': build_type,
'os': os,
'architecture': architecture
})
cmake_args_flags = f"{cmake_args}"
configurations.append(
{
"config_name": config_name,
"cmake_args": cmake_args_flags,
"cmake_target": cmake_target,
"build_only": build_only,
"build_type": build_type,
"os": os,
"architecture": architecture,
}
)
return configurations
def addSanitizerConfigs(architecture: dict, os: dict, build_type: str, cmake_args: str, config_name: str, build_only: bool, cmake_target: str, cxx_flags: str, configurations: list[dict]):
extra_warning_flags = ''
linker_relocation_flags = ''
linker_flags = ''
def addSanitizerConfigs(
architecture: dict,
os: dict,
build_type: str,
cmake_args: str,
config_name: str,
build_only: bool,
cmake_target: str,
cxx_flags: str,
configurations: list[dict],
):
extra_warning_flags = ""
linker_relocation_flags = ""
linker_flags = ""
# Use large code model to avoid relocation errors with large binaries
# Only for x86-64 (amd64) - ARM64 doesn't support -mcmodel=large
if architecture['platform'] == 'linux/amd64' and os['compiler_name'] == 'gcc':
if architecture["platform"] == "linux/amd64" and os["compiler_name"] == "gcc":
# Add -mcmodel=large to both compiler AND linker flags
# This is needed because sanitizers create very large binaries and
# large model removes the 2GB limitation that medium model has
cxx_flags += ' -mcmodel=large -fno-PIC'
linker_relocation_flags+=' -mcmodel=large -fno-PIC'
cxx_flags += " -mcmodel=large -fno-PIC"
linker_relocation_flags += " -mcmodel=large -fno-PIC"
# Create default sanitizer flags
sanitizers_flags = 'undefined,float-divide-by-zero'
sanitizers_flags = "undefined,float-divide-by-zero"
if os['compiler_name'] == 'gcc':
if os["compiler_name"] == "gcc":
# Suppress false positive warnings in GCC with stringop-overflow
extra_warning_flags += ' -Wno-stringop-overflow'
extra_warning_flags += " -Wno-stringop-overflow"
# Disable mold, gold and lld linkers.
# Use default linker (bfd/ld) which is more lenient with mixed code models
cmake_args += ' -Duse_mold=OFF -Duse_gold=OFF -Duse_lld=OFF'
cmake_args += " -Duse_mold=OFF -Duse_gold=OFF -Duse_lld=OFF"
# Add linker flags for Sanitizers
linker_flags += f' -DCMAKE_EXE_LINKER_FLAGS="{linker_relocation_flags} -fsanitize=address,{sanitizers_flags}"'
linker_flags += f' -DCMAKE_SHARED_LINKER_FLAGS="{linker_relocation_flags} -fsanitize=address,{sanitizers_flags}"'
elif os['compiler_name'] == 'clang':
elif os["compiler_name"] == "clang":
# Note: We use $GITHUB_WORKSPACE environment variable which will be expanded by the shell
# before CMake processes it. This ensures the compiler receives an absolute path.
# CMAKE_SOURCE_DIR won't work here because it's inside CMAKE_CXX_FLAGS string.
# GCC doesn't support ignorelist.
cxx_flags += ' -fsanitize-ignorelist=$GITHUB_WORKSPACE/sanitizers/suppressions/sanitizer-ignorelist.txt'
sanitizers_flags = f'{sanitizers_flags},signed-integer-overflow,unsigned-integer-overflow'
linker_flags += f' -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,{sanitizers_flags}"'
linker_flags += f' -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=address,{sanitizers_flags}"'
cxx_flags += " -fsanitize-ignorelist=$GITHUB_WORKSPACE/sanitizers/suppressions/sanitizer-ignorelist.txt"
sanitizers_flags = (
f"{sanitizers_flags},signed-integer-overflow,unsigned-integer-overflow"
)
linker_flags += (
f' -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,{sanitizers_flags}"'
)
linker_flags += (
f' -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=address,{sanitizers_flags}"'
)
# Sanitizers recommend minimum of -O1 for reasonable performance
cxx_flags += " -O1"
@@ -220,70 +337,103 @@ def addSanitizerConfigs(architecture: dict, os: dict, build_type: str, cmake_arg
cmake_args_flags = f'{cmake_args} -DCMAKE_CXX_FLAGS="-fsanitize=address,{sanitizers_flags} -fno-omit-frame-pointer {cxx_flags} {extra_warning_flags}" {linker_flags}'
# Add config with asan
configurations.append({
'config_name': config_name + "_asan",
'cmake_args': cmake_args_flags,
'cmake_target': cmake_target,
'build_only': build_only,
'build_type': build_type,
'os': os,
'architecture': architecture,
'sanitizers': 'Address'
})
configurations.append(
{
"config_name": config_name + "_asan",
"cmake_args": cmake_args_flags,
"cmake_target": cmake_target,
"build_only": build_only,
"build_type": build_type,
"os": os,
"architecture": architecture,
"sanitizers": "Address",
}
)
# Since TSAN runs are crashing with seg faults(could be compatibility issues with latest compilers)
# We deactivate it for now. But I would keep the code, since it took some effort to find the correct set of config needed to run this.
# This will be useful when we decide to activate it again in future.
activateTSAN = False
if activateTSAN:
linker_flags = ''
linker_flags = ""
# Update configs for tsan
# gcc doesn't supports atomic_thread_fence with tsan. Suppress warnings.
# Also tsan doesn't work well with mcmode=large and bfd linker
if os['compiler_name'] == 'gcc':
extra_warning_flags += ' -Wno-tsan'
cxx_flags = cxx_flags.replace('-mcmodel=large', '-mcmodel=medium')
linker_relocation_flags = linker_relocation_flags.replace('-mcmodel=large', '-mcmodel=medium')
if os["compiler_name"] == "gcc":
extra_warning_flags += " -Wno-tsan"
cxx_flags = cxx_flags.replace("-mcmodel=large", "-mcmodel=medium")
linker_relocation_flags = linker_relocation_flags.replace(
"-mcmodel=large", "-mcmodel=medium"
)
# Add linker flags for Sanitizers
linker_flags += f' -DCMAKE_EXE_LINKER_FLAGS="{linker_relocation_flags} -fsanitize=thread,{sanitizers_flags}"'
linker_flags += f' -DCMAKE_SHARED_LINKER_FLAGS="{linker_relocation_flags} -fsanitize=thread,{sanitizers_flags}"'
elif os['compiler_name'] == 'clang':
linker_flags += f' -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=thread,{sanitizers_flags}"'
linker_flags += f' -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=thread,{sanitizers_flags}"'
elif os["compiler_name"] == "clang":
linker_flags += (
f' -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=thread,{sanitizers_flags}"'
)
linker_flags += (
f' -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=thread,{sanitizers_flags}"'
)
cmake_args_flags = f'{cmake_args} -DCMAKE_CXX_FLAGS="-fsanitize=thread,{sanitizers_flags} -fno-omit-frame-pointer {cxx_flags} {extra_warning_flags}" {linker_flags}'
configurations.append({
'config_name': config_name+ "_tsan",
'cmake_args': cmake_args_flags,
'cmake_target': cmake_target,
'build_only': build_only,
'build_type': build_type,
'os': os,
'architecture': architecture,
'sanitizers': 'Thread'
})
configurations.append(
{
"config_name": config_name + "_tsan",
"cmake_args": cmake_args_flags,
"cmake_target": cmake_target,
"build_only": build_only,
"build_type": build_type,
"os": os,
"architecture": architecture,
"sanitizers": "Thread",
}
)
def read_config(file: Path) -> Config:
config = json.loads(file.read_text())
if config['architecture'] is None or config['os'] is None or config['build_type'] is None or config['cmake_args'] is None:
raise Exception('Invalid configuration file.')
if (
config["architecture"] is None
or config["os"] is None
or config["build_type"] is None
or config["cmake_args"] is None
):
raise Exception("Invalid configuration file.")
return Config(**config)
if __name__ == '__main__':
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('-a', '--all', help='Set to generate all configurations (generally used when merging a PR) or leave unset to generate a subset of configurations (generally used when committing to a PR).', action="store_true")
parser.add_argument('-c', '--config', help='Path to the JSON file containing the strategy matrix configurations.', required=False, type=Path)
parser.add_argument(
"-a",
"--all",
help="Set to generate all configurations (generally used when merging a PR) or leave unset to generate a subset of configurations (generally used when committing to a PR).",
action="store_true",
)
parser.add_argument(
"-c",
"--config",
help="Path to the JSON file containing the strategy matrix configurations.",
required=False,
type=Path,
)
args = parser.parse_args()
matrix = []
if args.config is None or args.config == '':
matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "linux.json"))
matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "macos.json"))
matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "windows.json"))
if args.config is None or args.config == "":
matrix += generate_strategy_matrix(
args.all, read_config(THIS_DIR / "linux.json")
)
matrix += generate_strategy_matrix(
args.all, read_config(THIS_DIR / "macos.json")
)
matrix += generate_strategy_matrix(
args.all, read_config(THIS_DIR / "windows.json")
)
else:
matrix += generate_strategy_matrix(args.all, read_config(args.config))
# Generate the strategy matrix.
print(f'matrix={json.dumps({"include": matrix})}')
print(f"matrix={json.dumps({'include': matrix})}")