ci: Upload artifacts during build and test in a separate job (#5817)

.github/actions/build-test/action.yml

@@ -1,96 +0,0 @@
-# This action build and tests the binary. The Conan dependencies must have
-# already been installed (see the build-deps action).
-name: Build and Test
-description: "Build and test the binary."
-
-# Note that actions do not support 'type' and all inputs are strings, see
-# https://docs.github.com/en/actions/reference/workflows-and-actions/metadata-syntax#inputs.
-inputs:
-  build_dir:
-    description: "The directory where to build."
-    required: true
-  build_only:
-    description: 'Whether to only build or to build and test the code ("true", "false").'
-    required: false
-    default: "false"
-  build_type:
-    description: 'The build type to use ("Debug", "Release").'
-    required: true
-  cmake_args:
-    description: "Additional arguments to pass to CMake."
-    required: false
-    default: ""
-  cmake_target:
-    description: "The CMake target to build."
-    required: true
-  codecov_token:
-    description: "The Codecov token to use for uploading coverage reports."
-    required: false
-    default: ""
-  os:
-    description: 'The operating system to use for the build ("linux", "macos", "windows").'
-    required: true
-
-runs:
-  using: composite
-  steps:
-    - name: Configure CMake
-      shell: bash
-      working-directory: ${{ inputs.build_dir }}
-      run: |
-        echo 'Configuring CMake.'
-        cmake \
-          -G '${{ inputs.os == 'windows' && 'Visual Studio 17 2022' || 'Ninja' }}' \
-          -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
-          -DCMAKE_BUILD_TYPE=${{ inputs.build_type }} \
-          ${{ inputs.cmake_args }} \
-          ..
-    - name: Build the binary
-      shell: bash
-      working-directory: ${{ inputs.build_dir }}
-      run: |
-        echo 'Building binary.'
-        cmake \
-          --build . \
-          --config ${{ inputs.build_type }} \
-          --parallel $(nproc) \
-          --target ${{ inputs.cmake_target }}
-    - name: Check linking
-      if: ${{ inputs.os == 'linux' }}
-      shell: bash
-      working-directory: ${{ inputs.build_dir }}
-      run: |
-        echo 'Checking linking.'
-        ldd ./rippled
-        if [ "$(ldd ./rippled | grep -E '(libstdc\+\+|libgcc)' | wc -l)" -eq 0 ]; then
-          echo 'The binary is statically linked.'
-        else
-          echo 'The binary is dynamically linked.'
-          exit 1
-        fi
-    - name: Verify voidstar
-      if: ${{ contains(inputs.cmake_args, '-Dvoidstar=ON') }}
-      shell: bash
-      working-directory: ${{ inputs.build_dir }}
-      run: |
-        echo 'Verifying presence of instrumentation.'
-        ./rippled --version | grep libvoidstar
-    - name: Test the binary
-      if: ${{ inputs.build_only == 'false' }}
-      shell: bash
-      working-directory: ${{ inputs.build_dir }}/${{ inputs.os == 'windows' && inputs.build_type || '' }}
-      run: |
-        echo 'Testing binary.'
-        ./rippled --unittest --unittest-jobs $(nproc)
-        ctest -j $(nproc) --output-on-failure
-    - name: Upload coverage report
-      if: ${{ inputs.cmake_target == 'coverage' }}
-      uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
-      with:
-        disable_search: true
-        disable_telem: true
-        fail_ci_if_error: true
-        files: ${{ inputs.build_dir }}/coverage.xml
-        plugins: noop
-        token: ${{ inputs.codecov_token }}
-        verbose: true

.github/actions/print-env/action.yml

@@ -0,0 +1,43 @@
+name: Print build environment
+description: "Print environment and some tooling versions"
+
+runs:
+  using: composite
+  steps:
+    - name: Check configuration (Windows)
+      if: ${{ runner.os == 'Windows' }}
+      shell: bash
+      run: |
+        echo 'Checking environment variables.'
+        set
+
+        echo 'Checking CMake version.'
+        cmake --version
+
+        echo 'Checking Conan version.'
+        conan --version
+
+    - name: Check configuration (Linux and macOS)
+      if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }}
+      shell: bash
+      run: |
+        echo 'Checking path.'
+        echo ${PATH} | tr ':' '\n'
+
+        echo 'Checking environment variables.'
+        env | sort
+
+        echo 'Checking CMake version.'
+        cmake --version
+
+        echo 'Checking compiler version.'
+        ${{ runner.os == 'Linux' && '${CC}' || 'clang' }} --version
+
+        echo 'Checking Conan version.'
+        conan --version
+
+        echo 'Checking Ninja version.'
+        ninja --version
+
+        echo 'Checking nproc version.'
+        nproc --version

.github/scripts/strategy-matrix/generate.py

@@ -162,7 +162,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
             'config_name': config_name,
             'cmake_args': cmake_args,
             'cmake_target': cmake_target,
-            'build_only': 'true' if build_only else 'false',
+            'build_only': build_only,
             'build_type': build_type,
             'os': os,
             'architecture': architecture,

.github/workflows/on-pr.yml

@@ -59,8 +59,11 @@ jobs:
             .github/actions/build-test/**
             .github/actions/setup-conan/**
             .github/scripts/strategy-matrix/**
+            .github/workflows/reusable-build.yml
+            .github/workflows/reusable-build-test-config.yml
             .github/workflows/reusable-build-test.yml
             .github/workflows/reusable-strategy-matrix.yml
+            .github/workflows/reusable-test.yml
             .codecov.yml
             cmake/**
             conan/**
@@ -105,7 +108,7 @@ jobs:
     with:
       os: ${{ matrix.os }}
     secrets:
-      codecov_token: ${{ secrets.CODECOV_TOKEN }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   notify-clio:
     needs:

.github/workflows/on-trigger.yml

@@ -23,8 +23,11 @@ on:
       - ".github/actions/build-test/**"
       - ".github/actions/setup-conan/**"
       - ".github/scripts/strategy-matrix/**"
+      - ".github/workflows/reusable-build.yml"
+      - ".github/workflows/reusable-build-test-config.yml"
       - ".github/workflows/reusable-build-test.yml"
       - ".github/workflows/reusable-strategy-matrix.yml"
+      - ".github/workflows/reusable-test.yml"
       - ".codecov.yml"
       - "cmake/**"
       - "conan/**"
@@ -43,22 +46,8 @@ on:
   schedule:
     - cron: "32 6 * * 1-5"
 
-  # Run when manually triggered via the GitHub UI or API. If `force_upload` is
+  # Run when manually triggered via the GitHub UI or API.
-  # true, then the dependencies that were missing (`force_rebuild` is false) or
-  # rebuilt (`force_rebuild` is true) will be uploaded, overwriting existing
-  # dependencies if needed.
   workflow_dispatch:
-    inputs:
-      dependencies_force_build:
-        description: "Force building of all dependencies."
-        required: false
-        type: boolean
-        default: false
-      dependencies_force_upload:
-        description: "Force uploading of all dependencies."
-        required: false
-        type: boolean
-        default: false
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -82,4 +71,4 @@ jobs:
       os: ${{ matrix.os }}
       strategy_matrix: ${{ github.event_name == 'schedule' && 'all' || 'minimal' }}
     secrets:
-      codecov_token: ${{ secrets.CODECOV_TOKEN }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/reusable-build-test-config.yml

@@ -0,0 +1,69 @@
+name: Build and test configuration
+
+on:
+  workflow_call:
+    inputs:
+      build_dir:
+        description: "The directory where to build."
+        required: true
+        type: string
+      build_only:
+        description: 'Whether to only build or to build and test the code ("true", "false").'
+        required: true
+        type: boolean
+      build_type:
+        description: 'The build type to use ("Debug", "Release").'
+        type: string
+        required: true
+      cmake_args:
+        description: "Additional arguments to pass to CMake."
+        required: false
+        type: string
+        default: ""
+      cmake_target:
+        description: "The CMake target to build."
+        type: string
+        required: true
+
+      runs_on:
+        description: Runner to run the job on as a JSON string
+        required: true
+        type: string
+      image:
+        description: "The image to run in (leave empty to run natively)"
+        required: true
+        type: string
+
+      config_name:
+        description: "The configuration string (used for naming artifacts and such)."
+        required: true
+        type: string
+
+    secrets:
+      CODECOV_TOKEN:
+        description: "The Codecov token to use for uploading coverage reports."
+        required: true
+
+jobs:
+  build:
+    uses: ./.github/workflows/reusable-build.yml
+    with:
+      build_dir: ${{ inputs.build_dir }}
+      build_type: ${{ inputs.build_type }}
+      cmake_args: ${{ inputs.cmake_args }}
+      cmake_target: ${{ inputs.cmake_target }}
+      runs_on: ${{ inputs.runs_on }}
+      image: ${{ inputs.image }}
+      config_name: ${{ inputs.config_name }}
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  test:
+    needs: build
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      run_tests: ${{ !inputs.build_only }}
+      verify_voidstar: ${{ contains(inputs.cmake_args, '-Dvoidstar=ON') }}
+      runs_on: ${{ inputs.runs_on }}
+      image: ${{ inputs.image }}
+      config_name: ${{ inputs.config_name }}

.github/workflows/reusable-build-test.yml

@@ -13,16 +13,6 @@ on:
         required: false
         type: string
         default: ".build"
-      dependencies_force_build:
-        description: "Force building of all dependencies."
-        required: false
-        type: boolean
-        default: false
-      dependencies_force_upload:
-        description: "Force uploading of all dependencies."
-        required: false
-        type: boolean
-        default: false
       os:
         description: 'The operating system to use for the build ("linux", "macos", "windows").'
         required: true
@@ -34,17 +24,9 @@ on:
         type: string
         default: "minimal"
     secrets:
-      codecov_token:
+      CODECOV_TOKEN:
         description: "The Codecov token to use for uploading coverage reports."
-        required: false
+        required: true
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.os }}
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
 
 jobs:
   # Generate the strategy matrix to be used by the following job.
@@ -54,94 +36,23 @@ jobs:
       os: ${{ inputs.os }}
       strategy_matrix: ${{ inputs.strategy_matrix }}
 
-  # Build and test the binary.
+  # Build and test the binary for each configuration.
-  build-test:
+  build-test-config:
     needs:
       - generate-matrix
+    uses: ./.github/workflows/reusable-build-test-config.yml
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
       max-parallel: 10
-    runs-on: ${{ matrix.architecture.runner }}
+    with:
-    container: ${{ inputs.os == 'linux' && format('ghcr.io/xrplf/ci/{0}-{1}:{2}-{3}-sha-5dd7158', matrix.os.distro_name, matrix.os.distro_version, matrix.os.compiler_name, matrix.os.compiler_version) || null }}
+      build_dir: ${{ inputs.build_dir }}
-    steps:
+      build_only: ${{ matrix.build_only }}
-      - name: Check strategy matrix
+      build_type: ${{ matrix.build_type }}
-        run: |
+      cmake_args: ${{ matrix.cmake_args }}
-          echo 'Operating system distro name: ${{ matrix.os.distro_name }}'
+      cmake_target: ${{ matrix.cmake_target }}
-          echo 'Operating system distro version: ${{ matrix.os.distro_version }}'
+      runs_on: ${{ toJSON(matrix.architecture.runner) }}
-          echo 'Operating system compiler name: ${{ matrix.os.compiler_name }}'
+      image: ${{ contains(matrix.architecture.platform, 'linux') && format('ghcr.io/xrplf/ci/{0}-{1}:{2}-{3}-sha-5dd7158', matrix.os.distro_name, matrix.os.distro_version, matrix.os.compiler_name, matrix.os.compiler_version) || '' }}
-          echo 'Operating system compiler version: ${{ matrix.os.compiler_version }}'
+      config_name: ${{ matrix.config_name }}
-          echo 'Architecture platform: ${{ matrix.architecture.platform }}'
+    secrets:
-          echo 'Architecture runner: ${{ toJson(matrix.architecture.runner) }}'
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-          echo 'Build type: ${{ matrix.build_type }}'
-          echo 'Build only: ${{ matrix.build_only }}'
-          echo 'CMake arguments: ${{ matrix.cmake_args }}'
-          echo 'CMake target: ${{ matrix.cmake_target }}'
-          echo 'Config name: ${{ matrix.config_name }}'
-
-      - name: Cleanup workspace
-        if: ${{ runner.os == 'macOS' }}
-        uses: XRPLF/actions/.github/actions/cleanup-workspace@3f044c7478548e3c32ff68980eeb36ece02b364e
-
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-      - name: Prepare runner
-        uses: XRPLF/actions/.github/actions/prepare-runner@638e0dc11ea230f91bd26622fb542116bb5254d5
-        with:
-          disable_ccache: false
-
-      - name: Check configuration (Windows)
-        if: ${{ inputs.os == 'windows' }}
-        run: |
-          echo 'Checking environment variables.'
-          set
-
-          echo 'Checking CMake version.'
-          cmake --version
-
-          echo 'Checking Conan version.'
-          conan --version
-      - name: Check configuration (Linux and MacOS)
-        if: ${{ inputs.os == 'linux' || inputs.os == 'macos' }}
-        run: |
-          echo 'Checking path.'
-          echo ${PATH} | tr ':' '\n'
-
-          echo 'Checking environment variables.'
-          env | sort
-
-          echo 'Checking CMake version.'
-          cmake --version
-
-          echo 'Checking compiler version.'
-          ${{ inputs.os == 'linux' && '${CC}' || 'clang' }} --version
-
-          echo 'Checking Conan version.'
-          conan --version
-
-          echo 'Checking Ninja version.'
-          ninja --version
-
-          echo 'Checking nproc version.'
-          nproc --version
-
-      - name: Setup Conan
-        uses: ./.github/actions/setup-conan
-
-      - name: Build dependencies
-        uses: ./.github/actions/build-deps
-        with:
-          build_dir: ${{ inputs.build_dir }}
-          build_type: ${{ matrix.build_type }}
-          force_build: ${{ inputs.dependencies_force_build }}
-
-      - name: Build and test binary
-        uses: ./.github/actions/build-test
-        with:
-          build_dir: ${{ inputs.build_dir }}
-          build_only: ${{ matrix.build_only }}
-          build_type: ${{ matrix.build_type }}
-          cmake_args: ${{ matrix.cmake_args }}
-          cmake_target: ${{ matrix.cmake_target }}
-          codecov_token: ${{ secrets.codecov_token }}
-          os: ${{ inputs.os }}

.github/workflows/reusable-build.yml

@@ -0,0 +1,115 @@
+name: Build rippled
+
+on:
+  workflow_call:
+    inputs:
+      build_dir:
+        description: "The directory where to build."
+        required: true
+        type: string
+      build_type:
+        description: 'The build type to use ("Debug", "Release").'
+        required: true
+        type: string
+      cmake_args:
+        description: "Additional arguments to pass to CMake."
+        required: true
+        type: string
+      cmake_target:
+        description: "The CMake target to build."
+        required: true
+        type: string
+
+      runs_on:
+        description: Runner to run the job on as a JSON string
+        required: true
+        type: string
+      image:
+        description: "The image to run in (leave empty to run natively)"
+        required: true
+        type: string
+
+      config_name:
+        description: "The name of the configuration."
+        required: true
+        type: string
+
+    secrets:
+      CODECOV_TOKEN:
+        description: "The Codecov token to use for uploading coverage reports."
+        required: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build:
+    name: Build ${{ inputs.config_name }}
+    runs-on: ${{ fromJSON(inputs.runs_on) }}
+    container: ${{ inputs.image != '' && inputs.image || null }}
+    steps:
+      - name: Cleanup workspace
+        if: ${{ runner.os == 'macOS' }}
+        uses: XRPLF/actions/.github/actions/cleanup-workspace@3f044c7478548e3c32ff68980eeb36ece02b364e
+
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+
+      - name: Prepare runner
+        uses: XRPLF/actions/.github/actions/prepare-runner@638e0dc11ea230f91bd26622fb542116bb5254d5
+        with:
+          disable_ccache: false
+
+      - name: Print build environment
+        uses: ./.github/actions/print-env
+
+      - name: Setup Conan
+        uses: ./.github/actions/setup-conan
+
+      - name: Build dependencies
+        uses: ./.github/actions/build-deps
+        with:
+          build_dir: ${{ inputs.build_dir }}
+          build_type: ${{ inputs.build_type }}
+
+      - name: Configure CMake
+        shell: bash
+        working-directory: ${{ inputs.build_dir }}
+        run: |
+          cmake \
+            -G '${{ runner.os == 'Windows' && 'Visual Studio 17 2022' || 'Ninja' }}' \
+            -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
+            -DCMAKE_BUILD_TYPE=${{ inputs.build_type }} \
+            ${{ inputs.cmake_args }} \
+            ..
+
+      - name: Build the binary
+        shell: bash
+        working-directory: ${{ inputs.build_dir }}
+        run: |
+          cmake \
+            --build . \
+            --config ${{ inputs.build_type }} \
+            --parallel $(nproc) \
+            --target ${{ inputs.cmake_target }}
+
+      - name: Upload rippled artifact
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: rippled-${{ inputs.config_name }}
+          path: ${{ inputs.build_dir }}/${{ runner.os == 'Windows' && inputs.build_type || '' }}/rippled${{ runner.os == 'Windows' && '.exe' || '' }}
+          retention-days: 3
+          if-no-files-found: error
+
+      - name: Upload coverage report
+        if: ${{ inputs.cmake_target == 'coverage' }}
+        uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
+        with:
+          disable_search: true
+          disable_telem: true
+          fail_ci_if_error: true
+          files: ${{ inputs.build_dir }}/coverage.xml
+          plugins: noop
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: true

.github/workflows/reusable-test.yml

@@ -0,0 +1,69 @@
+name: Test rippled
+
+on:
+  workflow_call:
+    inputs:
+      verify_voidstar:
+        description: "Whether to verify the presence of voidstar instrumentation."
+        required: true
+        type: boolean
+      run_tests:
+        description: "Whether to run unit tests"
+        required: true
+        type: boolean
+
+      runs_on:
+        description: Runner to run the job on as a JSON string
+        required: true
+        type: string
+      image:
+        description: "The image to run in (leave empty to run natively)"
+        required: true
+        type: string
+
+      config_name:
+        description: "The name of the configuration."
+        required: true
+        type: string
+
+jobs:
+  test:
+    name: Test ${{ inputs.config_name }}
+    runs-on: ${{ fromJSON(inputs.runs_on) }}
+    container: ${{ inputs.image != '' && inputs.image || null }}
+    steps:
+      - name: Download rippled artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: rippled-${{ inputs.config_name }}
+
+      - name: Make binary executable (Linux and macOS)
+        shell: bash
+        if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }}
+        run: |
+          chmod +x ./rippled
+
+      - name: Check linking (Linux)
+        if: ${{ runner.os == 'Linux' }}
+        shell: bash
+        run: |
+          ldd ./rippled
+          if [ "$(ldd ./rippled | grep -E '(libstdc\+\+|libgcc)' | wc -l)" -eq 0 ]; then
+            echo 'The binary is statically linked.'
+          else
+            echo 'The binary is dynamically linked.'
+            exit 1
+          fi
+
+      - name: Verifying presence of instrumentation
+        if: ${{ inputs.verify_voidstar }}
+        shell: bash
+        run: |
+          ./rippled --version | grep libvoidstar
+
+      - name: Test the binary
+        if: ${{ inputs.run_tests }}
+        shell: bash
+        run: |
+          ./rippled --unittest --unittest-jobs $(nproc)
+          ctest -j $(nproc) --output-on-failure