diff --git a/.github/scripts/strategy-matrix/generate.py b/.github/scripts/strategy-matrix/generate.py index 9743d5a4e3..b6f6601291 100755 --- a/.github/scripts/strategy-matrix/generate.py +++ b/.github/scripts/strategy-matrix/generate.py @@ -2,7 +2,17 @@ import argparse import itertools import json -import re +from pathlib import Path +from dataclasses import dataclass + +THIS_DIR = Path(__file__).parent.resolve() + +@dataclass +class Config: + architecture: list[dict] + os: list[dict] + build_type: list[str] + cmake_args: list[str] ''' Generate a strategy matrix for GitHub Actions CI. @@ -18,9 +28,9 @@ We will further set additional CMake arguments as follows: - Certain Debian Bookworm configurations will change the reference fee, enable codecov, and enable voidstar in PRs. ''' -def generate_strategy_matrix(all: bool, architecture: list[dict], os: list[dict], build_type: list[str], cmake_args: list[str]) -> dict: +def generate_strategy_matrix(all: bool, config: Config) -> list: configurations = [] - for architecture, os, build_type, cmake_args in itertools.product(architecture, os, build_type, cmake_args): + for architecture, os, build_type, cmake_args in itertools.product(config.architecture, config.os, config.build_type, config.cmake_args): # The default CMake target is 'all' for Linux and MacOS and 'install' # for Windows, but it can get overridden for certain configurations. cmake_target = 'install' if os["distro_name"] == 'windows' else 'all' @@ -158,21 +168,30 @@ def generate_strategy_matrix(all: bool, architecture: list[dict], os: list[dict] 'architecture': architecture, }) - return {'include': configurations} + return configurations + + +def read_config(file: Path) -> Config: + config = json.loads(file.read_text()) + if config['architecture'] is None or config['os'] is None or config['build_type'] is None or config['cmake_args'] is None: + raise Exception('Invalid configuration file.') + + return Config(**config) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('-a', '--all', help='Set to generate all configurations (generally used when merging a PR) or leave unset to generate a subset of configurations (generally used when committing to a PR).', action="store_true") - parser.add_argument('-c', '--config', help='Path to the JSON file containing the strategy matrix configurations.', required=True, type=str) + parser.add_argument('-c', '--config', help='Path to the JSON file containing the strategy matrix configurations.', required=False, type=Path) args = parser.parse_args() - # Load the JSON configuration file. - config = None - with open(args.config, 'r') as f: - config = json.load(f) - if config['architecture'] is None or config['os'] is None or config['build_type'] is None or config['cmake_args'] is None: - raise Exception('Invalid configuration file.') + matrix = [] + if args.config is None or args.config == '': + matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "linux.json")) + matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "macos.json")) + matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "windows.json")) + else: + matrix += generate_strategy_matrix(args.all, read_config(args.config)) # Generate the strategy matrix. - print(f'matrix={json.dumps(generate_strategy_matrix(args.all, config['architecture'], config['os'], config['build_type'], config['cmake_args']))}') + print(f'matrix={json.dumps({"include": matrix})}') diff --git a/.github/workflows/reusable-strategy-matrix.yml b/.github/workflows/reusable-strategy-matrix.yml index 5c84b51de1..20a90fc2e3 100644 --- a/.github/workflows/reusable-strategy-matrix.yml +++ b/.github/workflows/reusable-strategy-matrix.yml @@ -5,7 +5,7 @@ on: inputs: os: description: 'The operating system to use for the build ("linux", "macos", "windows").' - required: true + required: false type: string strategy_matrix: # TODO: Support additional strategies, e.g. "ubuntu" for generating all Ubuntu configurations. @@ -35,4 +35,4 @@ jobs: - name: Generate strategy matrix working-directory: .github/scripts/strategy-matrix id: generate - run: ./generate.py ${{ inputs.strategy_matrix == 'all' && '--all' || '' }} --config=${{ inputs.os }}.json >> "${GITHUB_OUTPUT}" + run: ./generate.py ${{ inputs.strategy_matrix == 'all' && '--all' || '' }} ${{ inputs.os != '' && format('--config={0}.json', inputs.os) || '' }} >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/reusable-upload-conan-deps-os.yml b/.github/workflows/reusable-upload-conan-deps-os.yml deleted file mode 100644 index 787cf6a39e..0000000000 --- a/.github/workflows/reusable-upload-conan-deps-os.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Upload Conan Dependencies - -on: - workflow_call: - inputs: - build_dir: - description: "The directory where to build." - required: false - type: string - default: ".build" - conan_remote_name: - description: "The name of the Conan remote to use." - required: false - type: string - default: xrplf - - os: - description: 'The operating system to use for the build ("linux", "macos", "windows").' - required: true - type: string - force_source_build: - description: "Force source build of all dependencies" - required: true - type: boolean - force_upload: - description: "Force upload of all dependencies" - required: true - type: boolean - secrets: - CONAN_USERNAME: - required: true - CONAN_PASSWORD: - required: true - -jobs: - generate-matrix: - uses: ./.github/workflows/reusable-strategy-matrix.yml - with: - os: ${{ inputs.os }} - strategy_matrix: all - - upload-conan-deps: - needs: - - generate-matrix - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} - runs-on: ${{ matrix.architecture.runner }} - container: ${{ inputs.os == 'linux' && format('ghcr.io/xrplf/ci/{0}-{1}:{2}-{3}', matrix.os.distro_name, matrix.os.distro_version, matrix.os.compiler_name, matrix.os.compiler_version) || null }} - - steps: - - name: Cleanup workspace - if: ${{ runner.os == 'macOS' }} - uses: XRPLF/actions/.github/actions/cleanup-workspace@3f044c7478548e3c32ff68980eeb36ece02b364e - - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - - name: Prepare runner - uses: XRPLF/actions/.github/actions/prepare-runner@638e0dc11ea230f91bd26622fb542116bb5254d5 - with: - disable_ccache: false - - - name: Setup Conan - uses: ./.github/actions/setup-conan - - - name: Build dependencies - uses: ./.github/actions/build-deps - with: - build_dir: ${{ inputs.build_dir }} - build_type: ${{ matrix.build_type }} - force_build: ${{ inputs.force_source_build }} - - - name: Login to Conan - if: github.repository_owner == 'XRPLF' && github.event_name != 'pull_request' - run: conan remote login -p ${{ secrets.CONAN_PASSWORD }} ${{ inputs.conan_remote_name }} ${{ secrets.CONAN_USERNAME }} - - - name: Upload Conan packages - if: github.repository_owner == 'XRPLF' && github.event_name != 'pull_request' && github.event_name != 'schedule' - run: conan upload "*" -r=${{ inputs.conan_remote_name }} --confirm ${{ inputs.force_upload == 'true' && '--force' || '' }} diff --git a/.github/workflows/upload-conan-deps.yml b/.github/workflows/upload-conan-deps.yml index 5a6adc99be..f63482761d 100644 --- a/.github/workflows/upload-conan-deps.yml +++ b/.github/workflows/upload-conan-deps.yml @@ -21,7 +21,6 @@ on: - .github/workflows/upload-conan-deps.yml - .github/workflows/reusable-strategy-matrix.yml - - .github/workflows/reusable-upload-conan-deps-os.yml - .github/actions/build-deps/action.yml - ".github/scripts/strategy-matrix/**" @@ -34,7 +33,6 @@ on: - .github/workflows/upload-conan-deps.yml - .github/workflows/reusable-strategy-matrix.yml - - .github/workflows/reusable-upload-conan-deps-os.yml - .github/actions/build-deps/action.yml - ".github/scripts/strategy-matrix/**" @@ -47,16 +45,46 @@ concurrency: cancel-in-progress: true jobs: - run-upload-conan-deps: - strategy: - fail-fast: true - matrix: - os: ["linux", "macos", "windows"] - uses: ./.github/workflows/reusable-upload-conan-deps-os.yml + generate-matrix: + uses: ./.github/workflows/reusable-strategy-matrix.yml with: - force_source_build: ${{ github.event_name == 'schedule' || github.event.inputs.force_source_build == 'true' }} - force_upload: ${{ github.event.inputs.force_upload == 'true' }} - os: ${{ matrix.os }} - secrets: - CONAN_USERNAME: ${{ secrets.CONAN_REMOTE_USERNAME }} - CONAN_PASSWORD: ${{ secrets.CONAN_REMOTE_PASSWORD }} + strategy_matrix: ${{ github.event_name == 'pull_request' && 'minimal' || 'all' }} + + run-upload-conan-deps: + needs: + - generate-matrix + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} + max-parallel: 10 + runs-on: ${{ matrix.architecture.runner }} + container: ${{ contains(matrix.architecture.platform, 'linux') && format('ghcr.io/xrplf/ci/{0}-{1}:{2}-{3}', matrix.os.distro_name, matrix.os.distro_version, matrix.os.compiler_name, matrix.os.compiler_version) || null }} + + steps: + - name: Cleanup workspace + if: ${{ runner.os == 'macOS' }} + uses: XRPLF/actions/.github/actions/cleanup-workspace@3f044c7478548e3c32ff68980eeb36ece02b364e + + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - name: Prepare runner + uses: XRPLF/actions/.github/actions/prepare-runner@638e0dc11ea230f91bd26622fb542116bb5254d5 + with: + disable_ccache: false + + - name: Setup Conan + uses: ./.github/actions/setup-conan + + - name: Build dependencies + uses: ./.github/actions/build-deps + with: + build_dir: .build + build_type: ${{ matrix.build_type }} + force_build: ${{ github.event_name == 'schedule' || github.event.inputs.force_source_build == 'true' }} + + - name: Login to Conan + if: github.repository_owner == 'XRPLF' && github.event_name != 'pull_request' + run: conan remote login -p ${{ secrets.CONAN_PASSWORD }} ${{ inputs.conan_remote_name }} ${{ secrets.CONAN_USERNAME }} + + - name: Upload Conan packages + if: github.repository_owner == 'XRPLF' && github.event_name != 'pull_request' && github.event_name != 'schedule' + run: conan upload "*" -r=${{ inputs.conan_remote_name }} --confirm ${{ github.event.inputs.force_upload == 'true' && '--force' || '' }}