From e0a0d7c49d58dfa7647029efe14d6bef35eebd48 Mon Sep 17 00:00:00 2001 From: Bart Thomee <11445373+bthomee@users.noreply.github.com> Date: Sun, 27 Jul 2025 17:59:59 -0400 Subject: [PATCH] Use self-hosted runners --- .../{build-test-nix => build-test}/action.yml | 19 ++- .github/workflows/build-debian.yml | 17 +- .github/workflows/build-macos.yml | 16 +- .github/workflows/build-rhel.yml | 52 +++--- .github/workflows/build-ubuntu.yml | 18 +- .github/workflows/build-windows.yml | 154 ++++++++++++++++++ .github/workflows/main.yml | 62 +++---- 7 files changed, 258 insertions(+), 80 deletions(-) rename .github/actions/{build-test-nix => build-test}/action.yml (86%) create mode 100644 .github/workflows/build-windows.yml diff --git a/.github/actions/build-test-nix/action.yml b/.github/actions/build-test/action.yml similarity index 86% rename from .github/actions/build-test-nix/action.yml rename to .github/actions/build-test/action.yml index 5c07da9c76..621ed06fdf 100644 --- a/.github/actions/build-test-nix/action.yml +++ b/.github/actions/build-test/action.yml @@ -19,13 +19,16 @@ inputs: type: string cmake_target: description: 'The CMake target to build.' - required: false - type: string - default: 'all' - link_check: - description: 'A boolean representing whether to check how the binary is linked.' required: true - type: boolean + type: string + os: + description: 'A string representing which operating system is used.' + required: true + type: choice + options: + - Linux + - MacOS + - Windows # Install the Conan profiles and log into the specified remote. We first remove # the remote if it already exists, which can occur on self-hosted runners where @@ -54,7 +57,7 @@ runs: --parallel $(nproc) \ --target ${{ inputs.cmake_target }} - name: Check linking - if: inputs.link_check + if: inputs.os == 'Linux' shell: bash working-directory: ${{ inputs.build_dir }} run: | @@ -67,7 +70,7 @@ runs: fi - name: Test the binary shell: bash - working-directory: ${{ inputs.build_dir }} + working-directory: ${{ inputs.build_dir }}/${{ inputs.os == 'Windows' && inputs.build_type || '' }} run: | ./rippled --unittest --unittest-jobs $(nproc) ctest -j $(nproc) --output-on-failure diff --git a/.github/workflows/build-debian.yml b/.github/workflows/build-debian.yml index da008f2576..cc54843547 100644 --- a/.github/workflows/build-debian.yml +++ b/.github/workflows/build-debian.yml @@ -51,11 +51,11 @@ env: # [ # { # "platform": "linux/amd64", -# "runner": "ubuntu-24.04" +# "runner": ["self-hosted", "Linux", "X64"] # }, # { # "platform": "linux/arm64", -# "runner": "ubuntu-24.04-arm" +# "runner": ["self-hosted", "Linux", "ARM64"] # } # ] # STRATEGY_MATRIX_OS: >- @@ -113,11 +113,12 @@ env: # "-DUnity=OFF", # "-DUnity=ON" # ] + STRATEGY_MATRIX_ARCHITECTURE: >- [ { "platform": "linux/amd64", - "runner": "ubuntu-24.04" + "runner": ["self-hosted", "Linux", "X64"] } ] STRATEGY_MATRIX_OS: >- @@ -131,7 +132,7 @@ env: ] STRATEGY_MATRIX_BUILD_TYPE: >- [ - "Debug" + "Release" ] STRATEGY_MATRIX_CMAKE_ARGS: >- [ @@ -197,6 +198,9 @@ jobs: echo "Checking Conan version." conan --version + + echo "Checking Ninja version." + ninja --version - name: Configure Conan uses: ./.github/actions/configure-conan with: @@ -212,10 +216,11 @@ jobs: build_type: ${{ matrix.build_type }} conan_remote_name: ${{ inputs.conan_remote_name }} - name: Build and test the binary - uses: ./.github/actions/build-test-nix + uses: ./.github/actions/build-test with: build_dir: ${{ inputs.build_dir }} build_type: ${{ matrix.build_type }} cmake_args: ${{ matrix.cmake_args }} cmake_generator: 'Ninja' - link_check: true + cmake_target: 'all' + os: 'Linux' diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index e5b4313eeb..ae662b9aab 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -49,7 +49,7 @@ env: # STRATEGY_MATRIX_ARCHITECTURE: >- # [ # { -# "runner": ["self-hosted", "macOS", "mac-runner-m1"] +# "runner": ["self-hosted", "macOS", "ARM64", "mac-runner-m1"] # } # ] # STRATEGY_MATRIX_BUILD_TYPE: >- @@ -65,7 +65,7 @@ env: STRATEGY_MATRIX_ARCHITECTURE: >- [ { - "runner": ["self-hosted", "macOS", "mac-runner-m1"] + "runner": ["self-hosted", "macOS", "ARM64", "mac-runner-m1"] } ] STRATEGY_MATRIX_BUILD_TYPE: >- @@ -113,6 +113,10 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - name: Install build tools + run: | + echo "Installing build tools." + brew install cmake conan coreutils ninja - name: Check configuration shell: bash run: | @@ -132,6 +136,9 @@ jobs: echo "Checking Conan version." conan --version + + echo "Checking Ninja version." + ninja --version - name: Configure Conan uses: ./.github/actions/configure-conan with: @@ -147,10 +154,11 @@ jobs: build_type: ${{ matrix.build_type }} conan_remote_name: ${{ inputs.conan_remote_name }} - name: Build and test the binary - uses: ./.github/actions/build-test-nix + uses: ./.github/actions/build-test with: build_dir: ${{ inputs.build_dir }} build_type: ${{ matrix.build_type }} cmake_args: ${{ matrix.cmake_args }} cmake_generator: 'Ninja' - link_check: false + cmake_target: 'all' + os: 'MacOS' diff --git a/.github/workflows/build-rhel.yml b/.github/workflows/build-rhel.yml index 14dc3e93f1..77848943a9 100644 --- a/.github/workflows/build-rhel.yml +++ b/.github/workflows/build-rhel.yml @@ -52,11 +52,11 @@ env: # [ # { # "platform": "linux/amd64", -# "runner": "ubuntu-24.04" +# "runner": ["self-hosted", "Linux", "X64"] # }, # { # "platform": "linux/arm64", -# "runner": "ubuntu-24.04-arm" +# "runner": ["self-hosted", "Linux", "ARM64"] # } # ] # STRATEGY_MATRIX_OS: >- @@ -91,29 +91,29 @@ env: # "-DUnity=ON" # ] STRATEGY_MATRIX_ARCHITECTURE: >- - [ - { - "platform": "linux/amd64", - "runner": "ubuntu-24.04" - } - ] + [ + { + "platform": "linux/amd64", + "runner": ["self-hosted", "Linux", "X64"] + } + ] STRATEGY_MATRIX_OS: >- - [ - { - "distro": "rhel", - "release": "9.6", - "compiler_name": "gcc", - "compiler_version": "13" - } - ] + [ + { + "distro": "rhel", + "release": "9.6", + "compiler_name": "gcc", + "compiler_version": "13" + } + ] STRATEGY_MATRIX_BUILD_TYPE: >- - [ - "Debug" - ] + [ + "Release" + ] STRATEGY_MATRIX_CMAKE_ARGS: >- - [ - "-DUnity=ON" - ] + [ + "-DUnity=ON" + ] jobs: # Generate the strategy matrix and expose environment variables to be used by @@ -174,6 +174,9 @@ jobs: echo "Checking Conan version." conan --version + + echo "Checking Ninja version." + ninja --version - name: Configure Conan uses: ./.github/actions/configure-conan with: @@ -189,10 +192,11 @@ jobs: build_type: ${{ matrix.build_type }} conan_remote_name: ${{ inputs.conan_remote_name }} - name: Build and test the binary - uses: ./.github/actions/build-test-nix + uses: ./.github/actions/build-test with: build_dir: ${{ inputs.build_dir }} build_type: ${{ matrix.build_type }} cmake_args: ${{ matrix.cmake_args }} cmake_generator: 'Ninja' - link_check: true + cmake_target: 'all' + os: 'Linux' diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 1bb1925c6b..ecb9565791 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -51,11 +51,11 @@ env: # [ # { # "platform": "linux/amd64", -# "runner": "ubuntu-24.04" +# "runner": ["self-hosted", "Linux", "X64"] # }, # { # "platform": "linux/arm64", -# "runner": "ubuntu-24.04-arm" +# "runner": ["self-hosted", "Linux", "ARM64"] # } # ] # STRATEGY_MATRIX_OS: >- @@ -101,7 +101,7 @@ env: # "release": "noble", # "compiler_name": "clang", # "compiler_version": "19" -# }, +# } # ] # STRATEGY_MATRIX_BUILD_TYPE: >- # [ @@ -117,7 +117,7 @@ env: [ { "platform": "linux/amd64", - "runner": "ubuntu-24.04" + "runner": ["self-hosted", "Linux", "X64"] } ] STRATEGY_MATRIX_OS: >- @@ -131,7 +131,7 @@ env: ] STRATEGY_MATRIX_BUILD_TYPE: >- [ - "Debug" + "Release" ] STRATEGY_MATRIX_CMAKE_ARGS: >- [ @@ -197,6 +197,9 @@ jobs: echo "Checking Conan version." conan --version + + echo "Checking Ninja version." + ninja --version - name: Configure Conan uses: ./.github/actions/configure-conan with: @@ -212,10 +215,11 @@ jobs: build_type: ${{ matrix.build_type }} conan_remote_name: ${{ inputs.conan_remote_name }} - name: Build and test the binary - uses: ./.github/actions/build-test-nix + uses: ./.github/actions/build-test with: build_dir: ${{ inputs.build_dir }} build_type: ${{ matrix.build_type }} cmake_args: ${{ matrix.cmake_args }} cmake_generator: 'Ninja' - link_check: true + cmake_target: 'all' + os: 'Linux' diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml new file mode 100644 index 0000000000..600364650b --- /dev/null +++ b/.github/workflows/build-windows.yml @@ -0,0 +1,154 @@ +# This workflow builds and tests the binary on various Windows configurations. +name: Windows + +on: + workflow_call: + inputs: + build_dir: + description: 'The directory where to build.' + required: false + type: string + default: '.build' + conan_remote_name: + description: 'The name of the Conan remote to use.' + required: true + type: string + conan_remote_url: + description: 'The URL of the Conan remote to use.' + required: true + type: string + secrets: + conan_remote_username: + description: 'The username for logging into the Conan remote.' + required: true + conan_remote_password: + description: 'The password for logging into the Conan remote.' + required: true + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-windows + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + # Global configuration for Conan. This is used to set the number of parallel + # downloads, uploads, and build jobs. The verbosity is set to verbose to + # provide more information during the build process. + CONAN_GLOBAL_CONF: | + core.download:parallel={{ os.cpu_count() }} + core.upload:parallel={{ os.cpu_count() }} + tools.build:jobs={{ (os.cpu_count() * 4/5) | int }} + tools.build:verbosity=verbose + tools.compilation:verbosity=verbose + # GitHub does not allow us to specify a reusable matrix strategy, so to avoid + # duplication, we define it here using environment variables and create the + # matrix in the first job. +# STRATEGY_MATRIX_ARCHITECTURE: >- +# [ +# { +# "runner": ["self-hosted", "Windows", "X64", "devbox"] +# } +# ] +# STRATEGY_MATRIX_BUILD_TYPE: >- +# [ +# "Debug", +# "Release" +# ] +# STRATEGY_MATRIX_CMAKE_ARGS: >- +# [ +# "-DUnity=OFF", +# "-DUnity=ON" +# ] + STRATEGY_MATRIX_ARCHITECTURE: >- + [ + { + "runner": ["self-hosted", "Windows", "X64", "devbox"] + } + ] + STRATEGY_MATRIX_BUILD_TYPE: >- + [ + "Debug" + ] + STRATEGY_MATRIX_CMAKE_ARGS: >- + [ + "-DUnity=ON" + ] + +jobs: + # Generate the strategy matrix and expose environment variables to be used by + # following jobs. Exposing env vars this way is needed as they cannot be + # directly passed as inputs to reusable workflows (although they can be passed + # as inputs to actions). + generate-outputs: + runs-on: ubuntu-latest + steps: + - name: Generate outputs + id: generate + run: | + echo "strategy_matrix_architecture=$(jq -c <<< '${{ env.STRATEGY_MATRIX_ARCHITECTURE }}')" >> "$GITHUB_OUTPUT" + echo "strategy_matrix_build_type=$(jq -c <<< '${{ env.STRATEGY_MATRIX_BUILD_TYPE }}')" >> "$GITHUB_OUTPUT" + echo "strategy_matrix_cmake_args=$(jq -c <<< '${{ env.STRATEGY_MATRIX_CMAKE_ARGS }}')" >> "$GITHUB_OUTPUT" + outputs: + conan_global_conf: ${{ env.CONAN_GLOBAL_CONF }} + strategy_matrix_architecture: ${{ steps.generate.outputs.strategy_matrix_architecture }} + strategy_matrix_build_type: ${{ steps.generate.outputs.strategy_matrix_build_type }} + strategy_matrix_cmake_args: ${{ steps.generate.outputs.strategy_matrix_cmake_args }} + + # Install and cache the dependencies, and then build and test the binary using + # various configurations. + build-test: + needs: + - generate-outputs + runs-on: ${{ matrix.architecture.runner }} + strategy: + fail-fast: false + max-parallel: 4 + matrix: + architecture: ${{ fromJson(needs.generate-outputs.outputs.strategy_matrix_architecture) }} + build_type: ${{ fromJson(needs.generate-outputs.outputs.strategy_matrix_build_type) }} + cmake_args: ${{ fromJson(needs.generate-outputs.outputs.strategy_matrix_cmake_args) }} + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - name: Check configuration + shell: bash + run: | + echo "Checking path" + $env:PATH -split ';' | Sort-Object + + echo "Checking environment variables." + ls env: + - name: Check versions + shell: bash + run: | + echo "Checking CMake version." + cmake --version + + echo "Checking Conan version." + conan --version + - name: Configure Conan + uses: ./.github/actions/configure-conan + with: + conan_global_conf: ${{ inputs.conan_global_conf }} + conan_remote_name: ${{ inputs.conan_remote_name }} + conan_remote_url: ${{ inputs.conan_remote_url }} + conan_remote_username: ${{ secrets.conan_remote_username }} + conan_remote_password: ${{ secrets.conan_remote_password }} + - name: Install dependencies + uses: ./.github/actions/install-dependencies + with: + build_dir: ${{ inputs.build_dir }} + build_type: ${{ matrix.build_type }} + conan_remote_name: ${{ inputs.conan_remote_name }} + - name: Build and test the binary + uses: ./.github/actions/build-test + with: + build_dir: ${{ inputs.build_dir }} + build_type: ${{ matrix.build_type }} + cmake_args: ${{ matrix.cmake_args }} + cmake_generator: 'Visual Studio 17 2022' + cmake_target: 'install' + os: 'Windows' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 553c54fccf..325f4b208c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,29 +28,29 @@ jobs: # if: github.event.pull_request.draft == false || contains(github.event.pull_request.labels.*.name, 'DraftRunCI') # uses: ./.github/workflows/check-levelization.yml -# debian: -## needs: -## - check-clang-format -## - check-levelization -# uses: ./.github/workflows/build-debian.yml -# with: -# conan_remote_name: ${{ vars.CONAN_REMOTE_NAME }} -# conan_remote_url: ${{ vars.CONAN_REMOTE_URL }} -# secrets: -# conan_remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }} -# conan_remote_password: ${{ secrets.CONAN_REMOTE_PASSWORD }} -# -# rhel: -## needs: -## - check-clang-format -## - check-levelization -# uses: ./.github/workflows/build-rhel.yml -# with: -# conan_remote_name: ${{ vars.CONAN_REMOTE_NAME }} -# conan_remote_url: ${{ vars.CONAN_REMOTE_URL }} -# secrets: -# conan_remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }} -# conan_remote_password: ${{ secrets.CONAN_REMOTE_PASSWORD }} + debian: +# needs: +# - check-clang-format +# - check-levelization + uses: ./.github/workflows/build-debian.yml + with: + conan_remote_name: ${{ vars.CONAN_REMOTE_NAME }} + conan_remote_url: ${{ vars.CONAN_REMOTE_URL }} + secrets: + conan_remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }} + conan_remote_password: ${{ secrets.CONAN_REMOTE_PASSWORD }} + + rhel: +# needs: +# - check-clang-format +# - check-levelization + uses: ./.github/workflows/build-rhel.yml + with: + conan_remote_name: ${{ vars.CONAN_REMOTE_NAME }} + conan_remote_url: ${{ vars.CONAN_REMOTE_URL }} + secrets: + conan_remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }} + conan_remote_password: ${{ secrets.CONAN_REMOTE_PASSWORD }} ubuntu: # needs: @@ -76,15 +76,15 @@ jobs: conan_remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }} conan_remote_password: ${{ secrets.CONAN_REMOTE_PASSWORD }} -# windows: + windows: # needs: # - check-clang-format # - check-levelization -# uses: ./.github/workflows/build-windows.yml -# with: -# conan_remote_name: ${{ vars.CONAN_REMOTE_NAME }} -# conan_remote_url: ${{ vars.CONAN_REMOTE_URL }} -# secrets: -# conan_remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }} -# conan_remote_password: ${{ secrets.CONAN_REMOTE_PASSWORD }} + uses: ./.github/workflows/build-windows.yml + with: + conan_remote_name: ${{ vars.CONAN_REMOTE_NAME }} + conan_remote_url: ${{ vars.CONAN_REMOTE_URL }} + secrets: + conan_remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }} + conan_remote_password: ${{ secrets.CONAN_REMOTE_PASSWORD }}