feat: Implement separate upload workflow (#5762)

* feat: Implement separate upload workflow
* Use cleanup-workspace
* Name some workflows reusable
* Add dependencies

.github/actions/build-deps/action.yml

@@ -1,7 +1,5 @@
-# This action installs and optionally uploads Conan dependencies to a remote
-# repository. The dependencies will only be uploaded if the credentials are
-# provided.
 name: Build Conan dependencies
+description: "Install Conan dependencies, optionally forcing a rebuild of all dependencies."
 
 # Note that actions do not support 'type' and all inputs are strings, see
 # https://docs.github.com/en/actions/reference/workflows-and-actions/metadata-syntax#inputs.
@@ -12,28 +10,10 @@ inputs:
   build_type:
     description: 'The build type to use ("Debug", "Release").'
     required: true
-  conan_remote_name:
-    description: "The name of the Conan remote to use."
-    required: true
-  conan_remote_url:
-    description: "The URL of the Conan endpoint to use."
-    required: true
-  conan_remote_username:
-    description: "The username for logging into the Conan remote. If not provided, the dependencies will not be uploaded."
-    required: false
-    default: ""
-  conan_remote_password:
-    description: "The password for logging into the Conan remote. If not provided, the dependencies will not be uploaded."
-    required: false
-    default: ""
   force_build:
     description: 'Force building of all dependencies ("true", "false").'
     required: false
     default: "false"
-  force_upload:
-    description: 'Force uploading of all dependencies ("true", "false").'
-    required: false
-    default: "false"
 
 runs:
   using: composite
@@ -51,12 +31,3 @@ runs:
           --options:host '&:xrpld=True' \
           --settings:all build_type=${{ inputs.build_type }} \
           --format=json ..
-    - name: Upload Conan dependencies
-      if: ${{ inputs.conan_remote_username != '' && inputs.conan_remote_password != '' }}
-      shell: bash
-      working-directory: ${{ inputs.build_dir }}
-      run: |
-        echo "Logging into Conan remote '${{ inputs.conan_remote_name }}' at ${{ inputs.conan_remote_url }}."
-        conan remote login ${{ inputs.conan_remote_name }} "${{ inputs.conan_remote_username }}" --password "${{ inputs.conan_remote_password }}"
-        echo 'Uploading dependencies.'
-        conan upload '*' --confirm --check ${{ inputs.force_upload == 'true' && '--force' || '' }} --remote=${{ inputs.conan_remote_name }}

.github/actions/build-test/action.yml

@@ -1,6 +1,7 @@
 # This action build and tests the binary. The Conan dependencies must have
 # already been installed (see the build-deps action).
 name: Build and Test
+description: "Build and test the binary."
 
 # Note that actions do not support 'type' and all inputs are strings, see
 # https://docs.github.com/en/actions/reference/workflows-and-actions/metadata-syntax#inputs.

.github/actions/setup-conan/action.yml

@@ -0,0 +1,43 @@
+name: Setup Conan
+description: "Set up Conan configuration, profile, and remote."
+
+inputs:
+  conan_remote_name:
+    description: "The name of the Conan remote to use."
+    required: false
+    default: xrplf
+  conan_remote_url:
+    description: "The URL of the Conan endpoint to use."
+    required: false
+    default: https://conan.ripplex.io
+
+runs:
+  using: composite
+
+  steps:
+    - name: Set up Conan configuration
+      shell: bash
+      run: |
+        echo 'Installing configuration.'
+        cat conan/global.conf ${{ runner.os == 'Linux' && '>>' || '>' }} $(conan config home)/global.conf
+
+        echo 'Conan configuration:'
+        conan config show '*'
+
+    - name: Set up Conan profile
+      shell: bash
+      run: |
+        echo 'Installing profile.'
+        conan config install conan/profiles/default -tf $(conan config home)/profiles/
+
+        echo 'Conan profile:'
+        conan profile show
+
+    - name: Set up Conan remote
+      shell: bash
+      run: |
+        echo "Adding Conan remote '${{ inputs.conan_remote_name }}' at ${{ inputs.conan_remote_url }}."
+        conan remote add --index 0 --force ${{ inputs.conan_remote_name }} ${{ inputs.conan_remote_url }}
+
+        echo 'Listing Conan remotes.'
+        conan remote list

.github/workflows/build-test.yml

@@ -13,14 +13,6 @@ on:
         required: false
         type: string
         default: ".build"
-      conan_remote_name:
-        description: "The name of the Conan remote to use."
-        required: true
-        type: string
-      conan_remote_url:
-        description: "The URL of the Conan endpoint to use."
-        required: true
-        type: string
       dependencies_force_build:
         description: "Force building of all dependencies."
         required: false
@@ -45,12 +37,6 @@ on:
       codecov_token:
         description: "The Codecov token to use for uploading coverage reports."
         required: false
-      conan_remote_username:
-        description: "The username for logging into the Conan remote. If not provided, the dependencies will not be uploaded."
-        required: false
-      conan_remote_password:
-        description: "The password for logging into the Conan remote. If not provided, the dependencies will not be uploaded."
-        required: false
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.os }}
@@ -63,20 +49,10 @@ defaults:
 jobs:
   # Generate the strategy matrix to be used by the following job.
   generate-matrix:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: 3.13
-      - name: Generate strategy matrix
-        working-directory: .github/scripts/strategy-matrix
-        id: generate
-        run: python generate.py ${{ inputs.strategy_matrix == 'all' && '--all' || '' }} --config=${{ inputs.os }}.json >> "${GITHUB_OUTPUT}"
-    outputs:
-      matrix: ${{ steps.generate.outputs.matrix }}
+    uses: ./.github/workflows/reusable-strategy-matrix.yml
+    with:
+      os: ${{ inputs.os }}
+      strategy_matrix: ${{ inputs.strategy_matrix }}
 
   # Build and test the binary.
   build-test:
@@ -148,40 +124,16 @@ jobs:
           echo 'Checking nproc version.'
           nproc --version
 
-      - name: Set up Conan configuration
-        run: |
-          echo 'Installing configuration.'
-          cat conan/global.conf ${{ inputs.os == 'linux' && '>>' || '>' }} $(conan config home)/global.conf
-
-          echo 'Conan configuration:'
-          conan config show '*'
-      - name: Set up Conan profile
-        run: |
-          echo 'Installing profile.'
-          conan config install conan/profiles/default -tf $(conan config home)/profiles/
-
-          echo 'Conan profile:'
-          conan profile show
-      - name: Set up Conan remote
-        shell: bash
-        run: |
-          echo "Adding Conan remote '${{ inputs.conan_remote_name }}' at ${{ inputs.conan_remote_url }}."
-          conan remote add --index 0 --force ${{ inputs.conan_remote_name }} ${{ inputs.conan_remote_url }}
-
-          echo 'Listing Conan remotes.'
-          conan remote list
+      - name: Setup Conan
+        uses: ./.github/actions/setup-conan
 
       - name: Build dependencies
         uses: ./.github/actions/build-deps
         with:
           build_dir: ${{ inputs.build_dir }}
           build_type: ${{ matrix.build_type }}
-          conan_remote_name: ${{ inputs.conan_remote_name }}
-          conan_remote_url: ${{ inputs.conan_remote_url }}
-          conan_remote_username: ${{ secrets.conan_remote_username }}
-          conan_remote_password: ${{ secrets.conan_remote_password }}
           force_build: ${{ inputs.dependencies_force_build }}
-          force_upload: ${{ inputs.dependencies_force_upload }}
+
       - name: Build and test binary
         uses: ./.github/actions/build-test
         with:

.github/workflows/notify-clio.yml

@@ -9,12 +9,14 @@ on:
     inputs:
       conan_remote_name:
         description: "The name of the Conan remote to use."
-        required: true
+        required: false
         type: string
+        default: xrplf
       conan_remote_url:
         description: "The URL of the Conan endpoint to use."
-        required: true
+        required: false
         type: string
+        default: https://conan.ripplex.io
     secrets:
       clio_notify_token:
         description: "The GitHub token to notify Clio about new versions."
@@ -54,12 +56,13 @@ jobs:
         id: conan_ref
         run: |
           echo "conan_ref=${{ steps.generate.outputs.version }}@${{ steps.generate.outputs.user }}/${{ steps.generate.outputs.channel }}" >> "${GITHUB_OUTPUT}"
-      - name: Add Conan remote
-        run: |
-          echo "Adding Conan remote '${{ inputs.conan_remote_name }}' at ${{ inputs.conan_remote_url }}."
-          conan remote add --index 0 --force ${{ inputs.conan_remote_name }} ${{ inputs.conan_remote_url }}
-          echo 'Listing Conan remotes.'
-          conan remote list
+
+      - name: Set up Conan
+        uses: ./.github/actions/setup-conan
+        with:
+          conan_remote_name: ${{ inputs.conan_remote_name }}
+          conan_remote_url: ${{ inputs.conan_remote_url }}
+
       - name: Log into Conan remote
         run: conan remote login ${{ inputs.conan_remote_name }} "${{ secrets.conan_remote_username }}" --password "${{ secrets.conan_remote_password }}"
       - name: Upload package

.github/workflows/on-pr.yml

@@ -23,10 +23,6 @@ defaults:
   run:
     shell: bash
 
-env:
-  CONAN_REMOTE_NAME: xrplf
-  CONAN_REMOTE_URL: https://conan.ripplex.io
-
 jobs:
   # This job determines whether the rest of the workflow should run. It runs
   # when the PR is not a draft (which should also cover merge-group) or
@@ -105,40 +101,22 @@ jobs:
     if: needs.should-run.outputs.go == 'true'
     uses: ./.github/workflows/check-levelization.yml
 
-  # This job works around the limitation that GitHub Actions does not support
-  # using environment variables as inputs for reusable workflows.
-  generate-outputs:
-    needs: should-run
-    if: needs.should-run.outputs.go == 'true'
-    runs-on: ubuntu-latest
-    steps:
-      - name: No-op
-        run: true
-    outputs:
-      conan_remote_name: ${{ env.CONAN_REMOTE_NAME }}
-      conan_remote_url: ${{ env.CONAN_REMOTE_URL }}
-
   build-test:
-    needs: generate-outputs
+    needs: should-run
     uses: ./.github/workflows/build-test.yml
     strategy:
       matrix:
         os: [linux, macos, windows]
     with:
-      conan_remote_name: ${{ needs.generate-outputs.outputs.conan_remote_name }}
-      conan_remote_url: ${{ needs.generate-outputs.outputs.conan_remote_url }}
       os: ${{ matrix.os }}
     secrets:
       codecov_token: ${{ secrets.CODECOV_TOKEN }}
 
   notify-clio:
     needs:
-      - generate-outputs
+      - should-run
       - build-test
     uses: ./.github/workflows/notify-clio.yml
-    with:
-      conan_remote_name: ${{ needs.generate-outputs.outputs.conan_remote_name }}
-      conan_remote_url: ${{ needs.generate-outputs.outputs.conan_remote_url }}
     secrets:
       clio_notify_token: ${{ secrets.CLIO_NOTIFY_TOKEN }}
       conan_remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }}

.github/workflows/on-trigger.yml

@@ -66,54 +66,18 @@ defaults:
   run:
     shell: bash
 
-env:
-  CONAN_REMOTE_NAME: xrplf
-  CONAN_REMOTE_URL: https://conan.ripplex.io
-
 jobs:
   check-missing-commits:
     if: ${{ github.event_name == 'push' && github.ref_type == 'branch' && contains(fromJSON('["develop", "release"]'), github.ref_name) }}
     uses: ./.github/workflows/check-missing-commits.yml
 
-  # This job works around the limitation that GitHub Actions does not support
-  # using environment variables as inputs for reusable workflows. It also sets
-  # outputs that depend on the event that triggered the workflow.
-  generate-outputs:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check inputs and set outputs
-        id: generate
-        run: |
-          if [[ '${{ github.event_name }}' == 'push' ]]; then
-            echo 'dependencies_force_build=false' >> "${GITHUB_OUTPUT}"
-            echo 'dependencies_force_upload=false' >> "${GITHUB_OUTPUT}"
-          elif [[ '${{ github.event_name }}' == 'schedule' ]]; then
-            echo 'dependencies_force_build=true' >> "${GITHUB_OUTPUT}"
-            echo 'dependencies_force_upload=false' >> "${GITHUB_OUTPUT}"
-          else
-            echo 'dependencies_force_build=${{ inputs.dependencies_force_build }}' >> "${GITHUB_OUTPUT}"
-            echo 'dependencies_force_upload=${{ inputs.dependencies_force_upload }}' >> "${GITHUB_OUTPUT}"
-          fi
-    outputs:
-      conan_remote_name: ${{ env.CONAN_REMOTE_NAME }}
-      conan_remote_url: ${{ env.CONAN_REMOTE_URL }}
-      dependencies_force_build: ${{ steps.generate.outputs.dependencies_force_build }}
-      dependencies_force_upload: ${{ steps.generate.outputs.dependencies_force_upload }}
-
   build-test:
-    needs: generate-outputs
     uses: ./.github/workflows/build-test.yml
     strategy:
       matrix:
         os: [linux, macos, windows]
     with:
-      conan_remote_name: ${{ needs.generate-outputs.outputs.conan_remote_name }}
-      conan_remote_url: ${{ needs.generate-outputs.outputs.conan_remote_url }}
-      dependencies_force_build: ${{ needs.generate-outputs.outputs.dependencies_force_build == 'true' }}
-      dependencies_force_upload: ${{ needs.generate-outputs.outputs.dependencies_force_upload == 'true' }}
       os: ${{ matrix.os }}
       strategy_matrix: "minimal"
     secrets:
       codecov_token: ${{ secrets.CODECOV_TOKEN }}
-      conan_remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }}
-      conan_remote_password: ${{ secrets.CONAN_REMOTE_PASSWORD }}

.github/workflows/reusable-strategy-matrix.yml

@@ -0,0 +1,38 @@
+name: Generate strategy matrix
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        description: 'The operating system to use for the build ("linux", "macos", "windows").'
+        required: true
+        type: string
+      strategy_matrix:
+        # TODO: Support additional strategies, e.g. "ubuntu" for generating all Ubuntu configurations.
+        description: 'The strategy matrix to use for generating the configurations ("minimal", "all").'
+        required: false
+        type: string
+        default: "minimal"
+    outputs:
+      matrix:
+        description: "The generated strategy matrix."
+        value: ${{ jobs.generate-matrix.outputs.matrix }}
+
+jobs:
+  generate-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate.outputs.matrix }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: 3.13
+
+      - name: Generate strategy matrix
+        working-directory: .github/scripts/strategy-matrix
+        id: generate
+        run: ./generate.py ${{ inputs.strategy_matrix == 'all' && '--all' || '' }} --config=${{ inputs.os }}.json >> "${GITHUB_OUTPUT}"

.github/workflows/reusable-upload-conan-deps-os.yml

@@ -0,0 +1,78 @@
+name: Upload Conan Dependencies
+
+on:
+  workflow_call:
+    inputs:
+      build_dir:
+        description: "The directory where to build."
+        required: false
+        type: string
+        default: ".build"
+      conan_remote_name:
+        description: "The name of the Conan remote to use."
+        required: false
+        type: string
+        default: xrplf
+
+      os:
+        description: 'The operating system to use for the build ("linux", "macos", "windows").'
+        required: true
+        type: string
+      force_source_build:
+        description: "Force source build of all dependencies"
+        required: true
+        type: boolean
+      force_upload:
+        description: "Force upload of all dependencies"
+        required: true
+        type: boolean
+    secrets:
+      CONAN_USERNAME:
+        required: true
+      CONAN_PASSWORD:
+        required: true
+
+jobs:
+  generate-matrix:
+    uses: ./.github/workflows/reusable-strategy-matrix.yml
+    with:
+      os: ${{ inputs.os }}
+      strategy_matrix: all
+
+  upload-conan-deps:
+    needs:
+      - generate-matrix
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.architecture.runner }}
+    container: ${{ inputs.os == 'linux' && format('ghcr.io/xrplf/ci/{0}-{1}:{2}-{3}', matrix.os.distro_name, matrix.os.distro_version, matrix.os.compiler_name, matrix.os.compiler_version) || null }}
+
+    steps:
+      - name: Cleanup workspace
+        if: ${{ runner.os == 'macOS' }}
+        uses: XRPLF/actions/.github/actions/cleanup-workspace@3f044c7478548e3c32ff68980eeb36ece02b364e
+
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+      - name: Prepare runner
+        uses: XRPLF/actions/.github/actions/prepare-runner@638e0dc11ea230f91bd26622fb542116bb5254d5
+        with:
+          disable_ccache: false
+
+      - name: Setup Conan
+        uses: ./.github/actions/setup-conan
+
+      - name: Build dependencies
+        uses: ./.github/actions/build-deps
+        with:
+          build_dir: ${{ inputs.build_dir }}
+          build_type: ${{ matrix.build_type }}
+          force_build: ${{ inputs.force_source_build }}
+
+      - name: Login to Conan
+        if: github.repository_owner == 'XRPLF' && github.event_name != 'pull_request'
+        run: conan remote login -p ${{ secrets.CONAN_PASSWORD }} ${{ inputs.conan_remote_name }} ${{ secrets.CONAN_USERNAME }}
+
+      - name: Upload Conan packages
+        if: github.repository_owner == 'XRPLF' && github.event_name != 'pull_request' && github.event_name != 'schedule'
+        run: conan upload "*" -r=${{ inputs.conan_remote_name }} --confirm ${{ inputs.force_upload == 'true' && '--force' || '' }}

.github/workflows/upload-conan-deps.yml

@@ -0,0 +1,62 @@
+name: Upload Conan Dependencies
+
+on:
+  schedule:
+    - cron: "0 9 * * 1-5"
+  workflow_dispatch:
+    inputs:
+      force_source_build:
+        description: "Force source build of all dependencies"
+        required: false
+        default: false
+        type: boolean
+      force_upload:
+        description: "Force upload of all dependencies"
+        required: false
+        default: false
+        type: boolean
+  pull_request:
+    branches: [develop]
+    paths:
+      - .github/workflows/upload-conan-deps.yml
+
+      - .github/workflows/reusable-strategy-matrix.yml
+      - .github/workflows/reusable-upload-conan-deps-os.yml
+
+      - .github/actions/build-deps/action.yml
+      - ".github/scripts/strategy-matrix/**"
+
+      - conanfile.py
+      - conan.lock
+  push:
+    branches: [develop]
+    paths:
+      - .github/workflows/upload-conan-deps.yml
+
+      - .github/workflows/reusable-strategy-matrix.yml
+      - .github/workflows/reusable-upload-conan-deps-os.yml
+
+      - .github/actions/build-deps/action.yml
+      - ".github/scripts/strategy-matrix/**"
+
+      - conanfile.py
+      - conan.lock
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-upload-conan-deps:
+    strategy:
+      fail-fast: true
+      matrix:
+        os: ["linux", "macos", "windows"]
+    uses: ./.github/workflows/reusable-upload-conan-deps-os.yml
+    with:
+      force_source_build: ${{ github.event_name == 'schedule' || github.event.inputs.force_source_build == 'true' }}
+      force_upload: ${{ github.event.inputs.force_upload == 'true' }}
+      os: ${{ matrix.os }}
+    secrets:
+      CONAN_USERNAME: ${{ secrets.CONAN_REMOTE_USERNAME }}
+      CONAN_PASSWORD: ${{ secrets.CONAN_REMOTE_PASSWORD }}