Add debug cache to ci (#1078)

Fixes #1066

.github/actions/code_coverage/action.yml

@@ -0,0 +1,29 @@
+name: Generate code coverage report
+description: Run tests, generate code coverage report and upload it to codecov.io
+runs:
+  using: composite
+  steps:
+    - name: Run tests
+      shell: bash
+      run: |
+        build/clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
+
+    - name: Run gcovr
+      shell: bash
+      run: |
+        gcovr -e unittests --xml build/coverage_report.xml -j8 --exclude-throw-branches
+
+    - name: Archive coverage report
+      uses: actions/upload-artifact@v3
+      with:
+        name: coverage-report.xml
+        path: build/coverage_report.xml
+        retention-days: 30
+
+    - name: Upload coverage report
+      uses: codecov/codecov-action@v3
+      with:
+        files: build/coverage_report.xml
+        fail_ci_if_error: true
+        verbose: true
+        token: ${{ env.CODECOV_TOKEN }}

.github/actions/generate/action.yml

@@ -12,9 +12,10 @@ inputs:
     description: Build type for third-party libraries and clio. Could be 'Release', 'Debug'
     required: true
     default: 'Release'
-  extra_cmake_args:
-    description: Additional cmake options
-    default: null
+  code_coverage:
+    description: Whether conan's coverage option should be on or not
+    required: true
+    default: 'false'
 runs:
   using: composite
   steps:
@@ -26,9 +27,10 @@ runs:
       shell: bash
       env:
         BUILD_OPTION: "${{ inputs.conan_cache_hit == 'true' && 'missing' || '' }}"
+        CODE_COVERAGE: "${{ inputs.code_coverage == 'true' && 'True' || 'False' }}"
       run: |
         cd build
-        conan install .. -of . -b $BUILD_OPTION -s build_type=${{ inputs.build_type }} -o clio:tests=True -o clio:lint=False --profile ${{ inputs.conan_profile }}
+        conan install .. -of . -b $BUILD_OPTION -s build_type=${{ inputs.build_type }} -o clio:tests=True -o clio:lint=False -o clio:coverage="${CODE_COVERAGE}" --profile ${{ inputs.conan_profile }}
 
     - name: Run cmake
       shell: bash

.github/actions/install_packages/action.yml

@@ -1,32 +0,0 @@
-name: Install packages
-description: Install packages and do all other paltform specific preparations
-runs:
-  using: composite
-  steps:
-    - name: Install packages on mac
-      if: ${{ runner.os == 'macOS' }}
-      shell: bash
-      run: |
-          brew install llvm@14 pkg-config ninja bison cmake ccache jq
-
-    - name: Install packages on linux
-      if: ${{ runner.os == 'Linux' }}
-      shell: bash
-      run: |
-          echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main' >> /etc/apt/sources.list
-          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
-          apt update -qq
-          apt install -y jq clang-tidy-17
-
-    - name: Install ccache on Linux
-      if: ${{ runner.os == 'Linux' }}
-      shell: bash
-      run: |
-          wget https://github.com/ccache/ccache/releases/download/v4.8.3/ccache-4.8.3-linux-x86_64.tar.xz
-          tar xf ./ccache-4.8.3-linux-x86_64.tar.xz
-          mv ./ccache-4.8.3-linux-x86_64/ccache /usr/bin/ccache
-
-    - name: Fix git permissions on Linux
-      if: ${{ runner.os == 'Linux' }}
-      shell: bash
-      run: git config --global --add safe.directory $PWD

.github/actions/restore_cache/action.yml

@@ -7,6 +7,14 @@ inputs:
   ccache_dir:
     description: Path to .ccache directory
     required: true
+  build_type:
+    description: Current build type (e.g. Release, Debug)
+    required: true
+    default: Release
+  code_coverage:
+    description: Whether code coverage is on
+    required: true
+    default: 'false'
 outputs:
   conan_hash:
     description: Hash to use as a part of conan cache key
@@ -40,7 +48,7 @@ runs:
       id: conan_cache
       with:
         path: ${{ inputs.conan_dir }}/data
-        key: clio-conan_data-${{ runner.os }}-develop-${{ steps.conan_hash.outputs.hash }}
+        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-develop-${{ steps.conan_hash.outputs.hash }}
 
     - name: Restore ccache cache
       uses: actions/cache/restore@v3
@@ -48,4 +56,4 @@ runs:
       if: ${{ env.CCACHE_DISABLE != '1' }}
       with:
         path: ${{ inputs.ccache_dir }}
-        key: clio-ccache-${{ runner.os }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
+        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-develop-${{ steps.git_common_ancestor.outputs.commit }}

.github/actions/save_cache/action.yml

@@ -18,6 +18,14 @@ inputs:
     required: true
   ccache_cache_miss_rate:
     description: How many cache misses happened
+  build_type:
+    description: Current build type (e.g. Release, Debug)
+    required: true
+    default: Release
+  code_coverage:
+    description: Whether code coverage is on
+    required: true
+    default: 'false'
 runs:
   using: composite
   steps:
@@ -36,13 +44,13 @@ runs:
       uses: actions/cache/save@v3
       with:
         path: ${{ inputs.conan_dir }}/data
-        key: clio-conan_data-${{ runner.os }}-develop-${{ inputs.conan_hash }}
+        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-develop-${{ inputs.conan_hash }}
 
     - name: Save ccache cache
       if: ${{ inputs.ccache_cache_hit != 'true' || inputs.ccache_cache_miss_rate == '100.0' }}
       uses: actions/cache/save@v3
       with:
         path: ${{ inputs.ccache_dir }}
-        key: clio-ccache-${{ runner.os }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
+        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
 
 

.github/workflows/build.yml

@@ -23,10 +23,19 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: Linux
+          - os: heavy
             container:
               image: rippleci/clio_ci:latest
+            build_type: Release
+            code_coverage: false
+          - os: heavy
+            container:
+              image: rippleci/clio_ci:latest
+            build_type: Debug
+            code_coverage: true
           - os: macOS
+            build_type: Release
+            code_coverage: false
     runs-on: [self-hosted, "${{ matrix.os }}"]
     container: ${{ matrix.container }}
 
@@ -50,13 +59,16 @@ jobs:
         with:
           conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
           ccache_dir: ${{ env.CCACHE_DIR }}
+          build_type: ${{ matrix.build_type }}
+          code_coverage: ${{ matrix.code_coverage }}
 
       - name: Run conan and cmake
         uses: ./.github/actions/generate
         with:
           conan_profile: ${{ steps.conan.outputs.conan_profile }}
           conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
-          build_type: 'Release'
+          build_type: ${{ matrix.build_type }}
+          code_coverage: ${{ matrix.code_coverage }}
 
       - name: Build Clio
         uses: ./.github/actions/build_clio
@@ -71,21 +83,16 @@ jobs:
           cat /tmp/ccache.stats
 
       - name: Strip tests
+        if: ${{ !matrix.code_coverage }}
         run: strip build/clio_tests
 
       - name: Upload clio_tests
+        if: ${{ !matrix.code_coverage }}
         uses: actions/upload-artifact@v3
         with:
           name: clio_tests_${{ runner.os }}
           path: build/clio_tests
 
-      - name: Upload compile_commans
-        if: ${{ runner.os == 'macOS' }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: compile_commans_${{ runner.os }}
-          path: build/compile_commands.json
-
       - name: Save cache
         uses: ./.github/actions/save_cache
         with:
@@ -95,6 +102,17 @@ jobs:
           ccache_dir: ${{ env.CCACHE_DIR }}
           ccache_cache_hit: ${{ steps.restore_cache.outputs.ccache_cache_hit }}
           ccache_cache_miss_rate: ${{ steps.ccache_stats.outputs.miss_rate }}
+          build_type: ${{ matrix.build_type }}
+          code_coverage: ${{ matrix.code_coverage }}
+
+      # TODO: This is not a part of build process but it is the easiest way to do it here.
+      # It will be refactored in https://github.com/XRPLF/clio/issues/1075
+      - name: Run code coverage
+        if: ${{ matrix.code_coverage }}
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        uses: ./.github/actions/code_coverage
+
 
   test:
     name: Run Tests
@@ -103,7 +121,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: Linux
+          - os: heavy
             container:
               image: rippleci/clio_ci:latest
           - os: macOS
@@ -118,65 +136,3 @@ jobs:
         run: |
           chmod +x ./clio_tests
           ./clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
-
-  coverage:
-    name: Run Coverage Report
-    needs: lint
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: Linux
-            container:
-              image: rippleci/clio_ci:latest
-    runs-on: [self-hosted, "${{ matrix.os }}"]
-    container: ${{ matrix.container }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-            fetch-depth: 0
-
-      - name: Prepare runner
-        uses: ./.github/actions/prepare_runner
-        with:
-          disable_ccache: true
-
-      - name: Install gcovr
-        run: |
-          pip install gcovr
-
-      - name: Setup conan
-        uses: ./.github/actions/setup_conan
-        id: conan
-
-      - name: Run conan and cmake
-        uses: ./.github/actions/generate
-        with:
-          conan_profile: ${{ steps.conan.outputs.conan_profile }}
-          conan_cache_hit: false
-          build_type: 'Debug'
-          extra_cmake_args: >-
-            -Dcoverage=ON
-            -DCODE_COVERAGE_TESTS_ARGS="--gtest_filter=-BackendCassandra*"
-            -DCODE_COVERAGE_REPORT_FORMAT=xml
-
-      - name: Run tests and coverage report
-        uses: ./.github/actions/build_clio
-        with:
-          target: coverage_report
-
-      - name: Archive coverage report
-        uses: actions/upload-artifact@v3
-        with:
-          name: coverage-report.xml
-          path: build/coverage_report.xml
-          retention-days: 30
-
-      - name: Upload coverage report
-        uses: codecov/codecov-action@v3
-        with:
-          files: build/coverage_report.xml
-          fail_ci_if_error: true
-          verbose: true
-          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/nightly.yml

@@ -13,11 +13,11 @@ jobs:
         include:
           - os: macOS
             build_type: Release
-          - os: Linux
+          - os: heavy
             build_type: Release
             container:
               image: rippleci/clio_ci:latest
-          - os: Linux
+          - os: heavy
             build_type: Debug
             container:
               image: rippleci/clio_ci:latest
@@ -78,9 +78,9 @@ jobs:
         include:
           - os: macOS
             build_type: Release
-          - os: Linux
+          - os: heavy
             build_type: Release
-          - os: Linux
+          - os: heavy
             build_type: Debug
     runs-on: [self-hosted, "${{ matrix.os }}"]
 

README.md

@@ -1,7 +1,8 @@
 # Clio
-![Build status](https://github.com/XRPLF/clio/actions/workflows/build.yml/badge.svg?branch=develop)
-![Nightly release status](https://github.com/XRPLF/clio/actions/workflows/nightly.yml/badge.svg?branch=develop)
-![Clang-tidy checks status](https://github.com/XRPLF/clio/actions/workflows/clang-tidy.yml/badge.svg?branch=develop)
+[![Build status](https://github.com/XRPLF/clio/actions/workflows/build.yml/badge.svg?branch=develop)](https://github.com/XRPLF/clio/actions/workflows/build.yml?query=branch%3Adevelop)
+[![Nightly release status](https://github.com/XRPLF/clio/actions/workflows/nightly.yml/badge.svg?branch=develop)](https://github.com/XRPLF/clio/actions/workflows/nightly.yml?query=branch%3Adevelop)
+[![Clang-tidy checks status](https://github.com/XRPLF/clio/actions/workflows/clang-tidy.yml/badge.svg?branch=develop)](https://github.com/XRPLF/clio/actions/workflows/clang-tidy.yml?query=branch%3Adevelop)
+[![Code coverage develop branch](https://codecov.io/gh/XRPLF/clio/branch/develop/graph/badge.svg?)](https://app.codecov.io/gh/XRPLF/clio)
 
 Clio is an XRP Ledger API server. Clio is optimized for RPC calls, over WebSocket or JSON-RPC. 
 Validated historical ledger and transaction data are stored in a more space-efficient format,