ci: Limit number of parallel jobs in "upload-conan-deps" (#5781)

- This should prevent Artifactory from being overloaded by too many requests at a time.
- Uses "max-parallel" to limit the build job to 10 simultaneous instances.
- Only run the minimal matrix on PRs.

.github/scripts/strategy-matrix/generate.py

@@ -2,7 +2,17 @@
 import argparse
 import itertools
 import json
-import re
+from pathlib import Path
+from dataclasses import dataclass
+
+THIS_DIR = Path(__file__).parent.resolve()
+
+@dataclass
+class Config:
+    architecture: list[dict]
+    os: list[dict]
+    build_type: list[str]
+    cmake_args: list[str]
 
 '''
 Generate a strategy matrix for GitHub Actions CI.
@@ -18,9 +28,9 @@ We will further set additional CMake arguments as follows:
 - Certain Debian Bookworm configurations will change the reference fee, enable
   codecov, and enable voidstar in PRs.
 '''
-def generate_strategy_matrix(all: bool, architecture: list[dict], os: list[dict], build_type: list[str], cmake_args: list[str]) -> dict:
+def generate_strategy_matrix(all: bool, config: Config) -> list:
     configurations = []
-    for architecture, os, build_type, cmake_args in itertools.product(architecture, os, build_type, cmake_args):
+    for architecture, os, build_type, cmake_args in itertools.product(config.architecture, config.os, config.build_type, config.cmake_args):
         # The default CMake target is 'all' for Linux and MacOS and 'install'
         # for Windows, but it can get overridden for certain configurations.
         cmake_target = 'install' if os["distro_name"] == 'windows' else 'all'
@@ -158,21 +168,30 @@ def generate_strategy_matrix(all: bool, architecture: list[dict], os: list[dict]
             'architecture': architecture,
         })
 
-    return {'include': configurations}
+    return configurations
+
+
+def read_config(file: Path) -> Config:
+    config = json.loads(file.read_text())
+    if config['architecture'] is None or config['os'] is None or config['build_type'] is None or config['cmake_args'] is None:
+        raise Exception('Invalid configuration file.')
+
+    return Config(**config)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('-a', '--all', help='Set to generate all configurations (generally used when merging a PR) or leave unset to generate a subset of configurations (generally used when committing to a PR).', action="store_true")
-    parser.add_argument('-c', '--config', help='Path to the JSON file containing the strategy matrix configurations.', required=True, type=str)
+    parser.add_argument('-c', '--config', help='Path to the JSON file containing the strategy matrix configurations.', required=False, type=Path)
     args = parser.parse_args()
 
-    # Load the JSON configuration file.
-    config = None
-    with open(args.config, 'r') as f:
-        config = json.load(f)
-    if config['architecture'] is None or config['os'] is None or config['build_type'] is None or config['cmake_args'] is None:
-        raise Exception('Invalid configuration file.')
+    matrix = []
+    if args.config is None or args.config == '':
+        matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "linux.json"))
+        matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "macos.json"))
+        matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "windows.json"))
+    else:
+        matrix += generate_strategy_matrix(args.all, read_config(args.config))
 
     # Generate the strategy matrix.
-    print(f'matrix={json.dumps(generate_strategy_matrix(args.all, config['architecture'], config['os'], config['build_type'], config['cmake_args']))}')
+    print(f'matrix={json.dumps({"include": matrix})}')

.github/workflows/reusable-strategy-matrix.yml

@@ -5,7 +5,7 @@ on:
     inputs:
       os:
         description: 'The operating system to use for the build ("linux", "macos", "windows").'
-        required: true
+        required: false
         type: string
       strategy_matrix:
         # TODO: Support additional strategies, e.g. "ubuntu" for generating all Ubuntu configurations.
@@ -35,4 +35,4 @@ jobs:
       - name: Generate strategy matrix
         working-directory: .github/scripts/strategy-matrix
         id: generate
-        run: ./generate.py ${{ inputs.strategy_matrix == 'all' && '--all' || '' }} --config=${{ inputs.os }}.json >> "${GITHUB_OUTPUT}"
+        run: ./generate.py ${{ inputs.strategy_matrix == 'all' && '--all' || '' }} ${{ inputs.os != '' && format('--config={0}.json', inputs.os) || '' }} >> "${GITHUB_OUTPUT}"

.github/workflows/reusable-upload-conan-deps-os.yml

@@ -1,78 +0,0 @@
-name: Upload Conan Dependencies
-
-on:
-  workflow_call:
-    inputs:
-      build_dir:
-        description: "The directory where to build."
-        required: false
-        type: string
-        default: ".build"
-      conan_remote_name:
-        description: "The name of the Conan remote to use."
-        required: false
-        type: string
-        default: xrplf
-
-      os:
-        description: 'The operating system to use for the build ("linux", "macos", "windows").'
-        required: true
-        type: string
-      force_source_build:
-        description: "Force source build of all dependencies"
-        required: true
-        type: boolean
-      force_upload:
-        description: "Force upload of all dependencies"
-        required: true
-        type: boolean
-    secrets:
-      CONAN_USERNAME:
-        required: true
-      CONAN_PASSWORD:
-        required: true
-
-jobs:
-  generate-matrix:
-    uses: ./.github/workflows/reusable-strategy-matrix.yml
-    with:
-      os: ${{ inputs.os }}
-      strategy_matrix: all
-
-  upload-conan-deps:
-    needs:
-      - generate-matrix
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
-    runs-on: ${{ matrix.architecture.runner }}
-    container: ${{ inputs.os == 'linux' && format('ghcr.io/xrplf/ci/{0}-{1}:{2}-{3}', matrix.os.distro_name, matrix.os.distro_version, matrix.os.compiler_name, matrix.os.compiler_version) || null }}
-
-    steps:
-      - name: Cleanup workspace
-        if: ${{ runner.os == 'macOS' }}
-        uses: XRPLF/actions/.github/actions/cleanup-workspace@3f044c7478548e3c32ff68980eeb36ece02b364e
-
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-      - name: Prepare runner
-        uses: XRPLF/actions/.github/actions/prepare-runner@638e0dc11ea230f91bd26622fb542116bb5254d5
-        with:
-          disable_ccache: false
-
-      - name: Setup Conan
-        uses: ./.github/actions/setup-conan
-
-      - name: Build dependencies
-        uses: ./.github/actions/build-deps
-        with:
-          build_dir: ${{ inputs.build_dir }}
-          build_type: ${{ matrix.build_type }}
-          force_build: ${{ inputs.force_source_build }}
-
-      - name: Login to Conan
-        if: github.repository_owner == 'XRPLF' && github.event_name != 'pull_request'
-        run: conan remote login -p ${{ secrets.CONAN_PASSWORD }} ${{ inputs.conan_remote_name }} ${{ secrets.CONAN_USERNAME }}
-
-      - name: Upload Conan packages
-        if: github.repository_owner == 'XRPLF' && github.event_name != 'pull_request' && github.event_name != 'schedule'
-        run: conan upload "*" -r=${{ inputs.conan_remote_name }} --confirm ${{ inputs.force_upload == 'true' && '--force' || '' }}

.github/workflows/upload-conan-deps.yml

@@ -21,7 +21,6 @@ on:
       - .github/workflows/upload-conan-deps.yml
 
       - .github/workflows/reusable-strategy-matrix.yml
-      - .github/workflows/reusable-upload-conan-deps-os.yml
 
       - .github/actions/build-deps/action.yml
       - ".github/scripts/strategy-matrix/**"
@@ -34,7 +33,6 @@ on:
       - .github/workflows/upload-conan-deps.yml
 
       - .github/workflows/reusable-strategy-matrix.yml
-      - .github/workflows/reusable-upload-conan-deps-os.yml
 
       - .github/actions/build-deps/action.yml
       - ".github/scripts/strategy-matrix/**"
@@ -47,16 +45,46 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run-upload-conan-deps:
-    strategy:
-      fail-fast: true
-      matrix:
-        os: ["linux", "macos", "windows"]
-    uses: ./.github/workflows/reusable-upload-conan-deps-os.yml
+  generate-matrix:
+    uses: ./.github/workflows/reusable-strategy-matrix.yml
     with:
-      force_source_build: ${{ github.event_name == 'schedule' || github.event.inputs.force_source_build == 'true' }}
-      force_upload: ${{ github.event.inputs.force_upload == 'true' }}
-      os: ${{ matrix.os }}
-    secrets:
-      CONAN_USERNAME: ${{ secrets.CONAN_REMOTE_USERNAME }}
-      CONAN_PASSWORD: ${{ secrets.CONAN_REMOTE_PASSWORD }}
+      strategy_matrix: ${{ github.event_name == 'pull_request' && 'minimal' || 'all' }}
+
+  run-upload-conan-deps:
+    needs:
+      - generate-matrix
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
+      max-parallel: 10
+    runs-on: ${{ matrix.architecture.runner }}
+    container: ${{ contains(matrix.architecture.platform, 'linux') && format('ghcr.io/xrplf/ci/{0}-{1}:{2}-{3}', matrix.os.distro_name, matrix.os.distro_version, matrix.os.compiler_name, matrix.os.compiler_version) || null }}
+
+    steps:
+      - name: Cleanup workspace
+        if: ${{ runner.os == 'macOS' }}
+        uses: XRPLF/actions/.github/actions/cleanup-workspace@3f044c7478548e3c32ff68980eeb36ece02b364e
+
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+      - name: Prepare runner
+        uses: XRPLF/actions/.github/actions/prepare-runner@638e0dc11ea230f91bd26622fb542116bb5254d5
+        with:
+          disable_ccache: false
+
+      - name: Setup Conan
+        uses: ./.github/actions/setup-conan
+
+      - name: Build dependencies
+        uses: ./.github/actions/build-deps
+        with:
+          build_dir: .build
+          build_type: ${{ matrix.build_type }}
+          force_build: ${{ github.event_name == 'schedule' || github.event.inputs.force_source_build == 'true' }}
+
+      - name: Login to Conan
+        if: github.repository_owner == 'XRPLF' && github.event_name != 'pull_request'
+        run: conan remote login -p ${{ secrets.CONAN_PASSWORD }} ${{ inputs.conan_remote_name }} ${{ secrets.CONAN_USERNAME }}
+
+      - name: Upload Conan packages
+        if: github.repository_owner == 'XRPLF' && github.event_name != 'pull_request' && github.event_name != 'schedule'
+        run: conan upload "*" -r=${{ inputs.conan_remote_name }} --confirm ${{ github.event.inputs.force_upload == 'true' && '--force' || '' }}