From 97956b1718f39113811735031263fea6b123936b Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Wed, 18 Jun 2025 18:20:26 +0100 Subject: [PATCH] feat: Build macos dependencies with sanitizers (#2240) --- .github/actions/setup_conan_macos/action.yml | 36 +++++++-------- .../setup_conan_macos/apple-clang.profile | 8 ++++ .github/scripts/generate_conan_matrix.py | 42 ++++++++++++++++++ .github/workflows/build.yml | 2 +- .github/workflows/build_impl.yml | 3 +- .github/workflows/nightly.yml | 4 +- .github/workflows/release.yml | 2 +- .github/workflows/upload_conan_deps.yml | 44 +++++++------------ 8 files changed, 90 insertions(+), 51 deletions(-) create mode 100644 .github/actions/setup_conan_macos/apple-clang.profile create mode 100755 .github/scripts/generate_conan_matrix.py diff --git a/.github/actions/setup_conan_macos/action.yml b/.github/actions/setup_conan_macos/action.yml index f87359fb..a30f8805 100644 --- a/.github/actions/setup_conan_macos/action.yml +++ b/.github/actions/setup_conan_macos/action.yml @@ -1,12 +1,9 @@ name: Setup conan -description: Setup conan profile and artifactory on macOS runner +description: Setup conan profiles and artifactory on macOS runner inputs: - conan_profile: - description: Conan profile name - required: true - global_conf_file: - description: Path to global.conf file + conan_files_dir: + description: Directory with conan files required: true runs: @@ -17,21 +14,24 @@ runs: shell: bash run: exit 1 - - name: Check profile name is default_apple_clang - if: inputs.conan_profile != 'default_apple_clang' - shell: bash - run: exit 1 - - - name: Create conan profile - shell: bash - run: | - conan profile detect --name "${{ inputs.conan_profile }}" - sed -i '' 's/compiler.cppstd=[^ ]*/compiler.cppstd=20/' "${{ env.CONAN_HOME }}/profiles/${{ inputs.conan_profile }}" - - name: Copy global.conf shell: bash run: | - cp "${{ inputs.global_conf_file }}" "${{ env.CONAN_HOME }}/global.conf" + cp "${{ inputs.conan_files_dir }}/global.conf" "${{ env.CONAN_HOME }}/global.conf" + + - name: Create apple-clang conan profile + shell: bash + run: | + mkdir -p "${{ env.CONAN_HOME }}/profiles" + cp .github/actions/setup_conan_macos/apple-clang.profile "${{ env.CONAN_HOME }}/profiles/apple-clang" + + - name: Create conan profiles for sanitizers + shell: bash + working-directory: ${{ inputs.conan_files_dir }} + run: | + cp ./sanitizer_template.profile "${{ env.CONAN_HOME }}/profiles/apple-clang.asan" + cp ./sanitizer_template.profile "${{ env.CONAN_HOME }}/profiles/apple-clang.tsan" + cp ./sanitizer_template.profile "${{ env.CONAN_HOME }}/profiles/apple-clang.ubsan" - name: Add artifactory remote shell: bash diff --git a/.github/actions/setup_conan_macos/apple-clang.profile b/.github/actions/setup_conan_macos/apple-clang.profile new file mode 100644 index 00000000..b61f5eb4 --- /dev/null +++ b/.github/actions/setup_conan_macos/apple-clang.profile @@ -0,0 +1,8 @@ +[settings] +arch={{detect_api.detect_arch()}} +build_type=Release +compiler=apple-clang +compiler.cppstd=20 +compiler.libcxx=libc++ +compiler.version=16 +os=Macos diff --git a/.github/scripts/generate_conan_matrix.py b/.github/scripts/generate_conan_matrix.py new file mode 100755 index 00000000..c645d487 --- /dev/null +++ b/.github/scripts/generate_conan_matrix.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +import itertools +import json + +LINUX_OS = ["heavy", "heavy-arm64"] +LINUX_CONTAINERS = ['{ "image": "ghcr.io/xrplf/clio-ci:latest" }'] +LINUX_COMPILERS = ["gcc", "clang"] + +MACOS_OS = ["macos15"] +MACOS_CONTAINERS = [""] +MACOS_COMPILERS = ["apple-clang"] + +BUILD_TYPES = ["Release", "Debug"] +SANITIZER_EXT = [".asan", ".tsan", ".ubsan", ""] + + +def generate_matrix(): + configurations = [] + + for os, container, compiler in itertools.chain( + itertools.product(LINUX_OS, LINUX_CONTAINERS, LINUX_COMPILERS), + itertools.product(MACOS_OS, MACOS_CONTAINERS, MACOS_COMPILERS), + ): + for sanitizer_ext, build_type in itertools.product(SANITIZER_EXT, BUILD_TYPES): + # libbacktrace doesn't build on arm64 with gcc.tsan + if os == "heavy-arm64" and compiler == "gcc" and sanitizer_ext == ".tsan": + continue + configurations.append( + { + "os": os, + "container": container, + "compiler": compiler, + "sanitizer_ext": sanitizer_ext, + "build_type": build_type, + } + ) + + return {"include": configurations} + + +if __name__ == "__main__": + print(f"matrix={json.dumps(generate_matrix())}") diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fc8076c3..2e38116d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,7 +47,7 @@ jobs: include: - os: macos15 - conan_profile: default_apple_clang + conan_profile: apple-clang build_type: Release container: "" static: false diff --git a/.github/workflows/build_impl.yml b/.github/workflows/build_impl.yml index cc52591f..c3ca9eb3 100644 --- a/.github/workflows/build_impl.yml +++ b/.github/workflows/build_impl.yml @@ -81,8 +81,7 @@ jobs: if: runner.os == 'macOS' uses: ./.github/actions/setup_conan_macos with: - conan_profile: ${{ inputs.conan_profile }} - global_conf_file: docker/ci/conan/global.conf + conan_files_dir: docker/ci/conan/ - name: Restore cache if: ${{ !inputs.disable_cache }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e472ad0f..f4dec61a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -32,7 +32,7 @@ jobs: matrix: include: - os: macos15 - conan_profile: default_apple_clang + conan_profile: apple-clang build_type: Release static: false - os: heavy @@ -75,7 +75,7 @@ jobs: container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }' static: true - os: macos15 - conan_profile: default_apple_clang + conan_profile: apple-clang container: "" static: false uses: ./.github/workflows/build_impl.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2b351d70..eb221d53 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: matrix: include: - os: macos15 - conan_profile: default_apple_clang + conan_profile: apple-clang build_type: Release static: false - os: heavy diff --git a/.github/workflows/upload_conan_deps.yml b/.github/workflows/upload_conan_deps.yml index 72fe1523..1e43a172 100644 --- a/.github/workflows/upload_conan_deps.yml +++ b/.github/workflows/upload_conan_deps.yml @@ -22,35 +22,26 @@ concurrency: cancel-in-progress: true jobs: + generate-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Calculate recipes matrix 🛠 + id: set-matrix + run: .github/scripts/generate_conan_matrix.py >> "${GITHUB_OUTPUT}" + upload-conan-deps: name: Build and Upload Conan Deps + needs: generate-matrix + strategy: fail-fast: false - matrix: - os: [heavy, heavy-arm64] - container: ['{ "image": "ghcr.io/xrplf/clio-ci:latest" }'] - compiler: ["gcc", "clang"] - sanitizer_ext: [".asan", ".tsan", ".ubsan", ""] - build_type: ["Release", "Debug"] - # libbacktrace doesn't build on arm64 with gcc.tsan - exclude: - - os: heavy-arm64 - container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }' - compiler: gcc - sanitizer_ext: .tsan - build_type: Release - - os: heavy-arm64 - container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }' - compiler: gcc - sanitizer_ext: .tsan - build_type: Debug - include: - - os: macos15 - container: "" - compiler: default_apple_clang - sanitizer_ext: "" - build_type: Release + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} runs-on: ${{ matrix.os }} container: ${{ matrix.container != '' && fromJson(matrix.container) || null }} @@ -70,8 +61,7 @@ jobs: if: runner.os == 'macOS' uses: ./.github/actions/setup_conan_macos with: - conan_profile: ${{ env.CONAN_PROFILE }} - global_conf_file: docker/ci/conan/global.conf + conan_files_dir: docker/ci/conan/ - name: Show conan profile run: conan profile show --profile:all ${{ env.CONAN_PROFILE }} @@ -89,4 +79,4 @@ jobs: - name: Upload Conan packages if: github.event_name != 'pull_request' - run: conan upload "*" -r=ripple --confirm --force + run: conan upload "*" -r=ripple --confirm