ci: Use ccache to cache build objects for speeding up building (#6104)

Right now, each pipeline invocation builds the source code from scratch. Although compiled Conan dependencies are cached in a remote server, the source build objects are not. We are able to further speed up our builds by leveraging `ccache`. This change enables caching of build objects using `ccache` on Linux, macOS, and Windows.
2026-06-02 08:17:13 +00:00 · 2026-01-06 20:34:21 -05:00
parent 44d21b8f6d
commit d734c8dddd
10 changed files with 159 additions and 49 deletions
--- a/.github/actions/print-env/action.yml
+++ b/.github/actions/print-env/action.yml
@@ -11,12 +11,6 @@ runs:
        echo 'Checking environment variables.'
        set

-        echo 'Checking CMake version.'
-        cmake --version
-
-        echo 'Checking Conan version.'
-        conan --version
-
    - name: Check configuration (Linux and macOS)
      if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }}
      shell: bash
@@ -27,17 +21,23 @@ runs:
        echo 'Checking environment variables.'
        env | sort

-        echo 'Checking CMake version.'
-        cmake --version
-
        echo 'Checking compiler version.'
        ${{ runner.os == 'Linux' && '${CC}' || 'clang' }} --version

-        echo 'Checking Conan version.'
-        conan --version
-
        echo 'Checking Ninja version.'
        ninja --version

        echo 'Checking nproc version.'
        nproc --version
+
+    - name: Check configuration (all)
+      shell: bash
+      run: |
+        echo 'Checking Ccache version.'
+        ccache --version
+
+        echo 'Checking CMake version.'
+        cmake --version
+
+        echo 'Checking Conan version.'
+        conan --version
--- a/.github/scripts/strategy-matrix/generate.py
+++ b/.github/scripts/strategy-matrix/generate.py
@@ -232,6 +232,8 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
            f"-{architecture['platform'][architecture['platform'].find('/') + 1 :]}"
        )
        config_name += f"-{build_type.lower()}"
+        if "-Dcoverage=ON" in cmake_args:
+            config_name += "-coverage"
        if "-Dunity=ON" in cmake_args:
            config_name += "-unity"

--- a/.github/scripts/strategy-matrix/linux.json
+++ b/.github/scripts/strategy-matrix/linux.json
@@ -15,196 +15,196 @@
      "distro_version": "bookworm",
      "compiler_name": "gcc",
      "compiler_version": "12",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "debian",
      "distro_version": "bookworm",
      "compiler_name": "gcc",
      "compiler_version": "13",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "debian",
      "distro_version": "bookworm",
      "compiler_name": "gcc",
      "compiler_version": "14",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "debian",
      "distro_version": "bookworm",
      "compiler_name": "gcc",
      "compiler_version": "15",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "debian",
      "distro_version": "bookworm",
      "compiler_name": "clang",
      "compiler_version": "16",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "debian",
      "distro_version": "bookworm",
      "compiler_name": "clang",
      "compiler_version": "17",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "debian",
      "distro_version": "bookworm",
      "compiler_name": "clang",
      "compiler_version": "18",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "debian",
      "distro_version": "bookworm",
      "compiler_name": "clang",
      "compiler_version": "19",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "debian",
      "distro_version": "bookworm",
      "compiler_name": "clang",
      "compiler_version": "20",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "debian",
      "distro_version": "trixie",
      "compiler_name": "gcc",
      "compiler_version": "14",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "debian",
      "distro_version": "trixie",
      "compiler_name": "gcc",
      "compiler_version": "15",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "debian",
      "distro_version": "trixie",
      "compiler_name": "clang",
      "compiler_version": "20",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "debian",
      "distro_version": "trixie",
      "compiler_name": "clang",
      "compiler_version": "21",
-      "image_sha": "0525eae"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "rhel",
      "distro_version": "8",
      "compiler_name": "gcc",
      "compiler_version": "14",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "rhel",
      "distro_version": "8",
      "compiler_name": "clang",
      "compiler_version": "any",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "rhel",
      "distro_version": "9",
      "compiler_name": "gcc",
      "compiler_version": "12",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "rhel",
      "distro_version": "9",
      "compiler_name": "gcc",
      "compiler_version": "13",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "rhel",
      "distro_version": "9",
      "compiler_name": "gcc",
      "compiler_version": "14",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "rhel",
      "distro_version": "9",
      "compiler_name": "clang",
      "compiler_version": "any",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "rhel",
      "distro_version": "10",
      "compiler_name": "gcc",
      "compiler_version": "14",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "rhel",
      "distro_version": "10",
      "compiler_name": "clang",
      "compiler_version": "any",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "ubuntu",
      "distro_version": "jammy",
      "compiler_name": "gcc",
      "compiler_version": "12",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "ubuntu",
      "distro_version": "noble",
      "compiler_name": "gcc",
      "compiler_version": "13",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "ubuntu",
      "distro_version": "noble",
      "compiler_name": "gcc",
      "compiler_version": "14",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "ubuntu",
      "distro_version": "noble",
      "compiler_name": "clang",
      "compiler_version": "16",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "ubuntu",
      "distro_version": "noble",
      "compiler_name": "clang",
      "compiler_version": "17",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "ubuntu",
      "distro_version": "noble",
      "compiler_name": "clang",
      "compiler_version": "18",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    },
    {
      "distro_name": "ubuntu",
      "distro_version": "noble",
      "compiler_name": "clang",
      "compiler_version": "19",
-      "image_sha": "e1782cd"
+      "image_sha": "cc09fd3"
    }
  ],
  "build_type": ["Debug", "Release"],
--- a/.github/workflows/on-pr.yml
+++ b/.github/workflows/on-pr.yml
@@ -114,6 +114,9 @@ jobs:
      matrix:
        os: [linux, macos, windows]
    with:
+      # Enable ccache only for events targeting the XRPLF repository, since
+      # other accounts will not have access to our remote cache storage.
+      ccache_enabled: ${{ github.repository_owner == 'XRPLF' }}
      os: ${{ matrix.os }}
    secrets:
      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
--- a/.github/workflows/on-trigger.yml
+++ b/.github/workflows/on-trigger.yml
@@ -66,6 +66,12 @@ jobs:
    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
+        # Enable ccache only for events targeting the XRPLF repository, since
+        # other accounts will not have access to our remote cache storage.
+        # However, we do not enable ccache for events targeting the master or a
+        # release branch, to protect against the rare case that the output
+        # produced by ccache is not identical to a regular compilation.
+        ccache_enabled: ${{ github.repository_owner == 'XRPLF' && !(github.base_ref == 'master' || startsWith(github.base_ref, 'release')) }}
        os: [linux, macos, windows]
    with:
      os: ${{ matrix.os }}
--- a/.github/workflows/reusable-build-test-config.yml
+++ b/.github/workflows/reusable-build-test-config.yml
@@ -10,8 +10,14 @@ on:

      build_type:
        description: 'The build type to use ("Debug", "Release").'
-        type: string
        required: true
+        type: string
+
+      ccache_enabled:
+        description: "Whether to enable ccache."
+        required: false
+        type: boolean
+        default: false

      cmake_args:
        description: "Additional arguments to pass to CMake."
@@ -21,8 +27,8 @@ on:

      cmake_target:
        description: "The CMake target to build."
-        type: string
        required: true
+        type: string

      runs_on:
        description: Runner to run the job on as a JSON string
@@ -66,8 +72,25 @@ jobs:
    container: ${{ inputs.image != '' && inputs.image || null }}
    timeout-minutes: 60
    env:
-      ENABLED_VOIDSTAR: ${{ contains(inputs.cmake_args, '-Dvoidstar=ON') }}
-      ENABLED_COVERAGE: ${{ contains(inputs.cmake_args, '-Dcoverage=ON') }}
+      # Use a namespace to keep the objects separate for each configuration.
+      CCACHE_NAMESPACE: ${{ inputs.config_name }}
+      # Ccache supports both Redis and HTTP endpoints.
+      # * For Redis, use the following format: redis://ip:port, see
+      #   https://github.com/ccache/ccache/wiki/Redis-storage. Note that TLS is
+      #   not directly supported by ccache, and requires use of a proxy.
+      # * For HTTP use the following format: http://ip:port/cache when using
+      #   nginx as backend or http://ip:port|layout=bazel when using Bazel
+      #   Remote Cache, see https://github.com/ccache/ccache/wiki/HTTP-storage.
+      #   Note that HTTPS is not directly supported by ccache.
+      CCACHE_REMOTE_ONLY: true
+      CCACHE_REMOTE_STORAGE: http://cache.dev.ripplex.io:8080|layout=bazel
+      # Ignore the creation and modification timestamps on files, since the
+      # header files are copied into separate directories by CMake, which will
+      # otherwise result in cache misses.
+      CCACHE_SLOPPINESS: include_file_ctime,include_file_mtime
+      # Determine if coverage and voidstar should be enabled.
+      COVERAGE_ENABLED: ${{ contains(inputs.cmake_args, '-Dcoverage=ON') }}
+      VOIDSTAR_ENABLED: ${{ contains(inputs.cmake_args, '-Dvoidstar=ON') }}
    steps:
      - name: Cleanup workspace (macOS and Windows)
        if: ${{ runner.os == 'macOS' || runner.os == 'Windows' }}
@@ -79,7 +102,11 @@ jobs:
      - name: Prepare runner
        uses: XRPLF/actions/prepare-runner@2ece4ec6ab7de266859a6f053571425b2bd684b6
        with:
-          disable_ccache: false
+          disable_ccache: ${{ !inputs.ccache_enabled }}
+
+      - name: Set ccache log file
+        if: ${{ inputs.ccache_enabled && runner.debug == '1' }}
+        run: echo "CCACHE_LOGFILE=${{ runner.temp }}/ccache.log" >> "${GITHUB_ENV}"

      - name: Print build environment
        uses: ./.github/actions/print-env
@@ -128,6 +155,15 @@ jobs:
            --parallel "${BUILD_NPROC}" \
            --target "${CMAKE_TARGET}"

+      - name: Show ccache statistics
+        if: ${{ inputs.ccache_enabled }}
+        run: |
+          ccache --show-stats -vv
+          if [ '${{ runner.debug }}' = '1' ]; then
+            cat "${CCACHE_LOGFILE}"
+            curl ${CCACHE_REMOTE_STORAGE%|*}/status || true
+          fi
+
      - name: Upload the binary (Linux)
        if: ${{ github.repository_owner == 'XRPLF' && runner.os == 'Linux' }}
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
@@ -150,7 +186,7 @@ jobs:
          fi

      - name: Verify presence of instrumentation (Linux)
-        if: ${{ runner.os == 'Linux' && env.ENABLED_VOIDSTAR == 'true' }}
+        if: ${{ runner.os == 'Linux' && env.VOIDSTAR_ENABLED == 'true' }}
        working-directory: ${{ env.BUILD_DIR }}
        run: |
          ./xrpld --version | grep libvoidstar
@@ -185,7 +221,7 @@ jobs:
          netstat -an

      - name: Prepare coverage report
-        if: ${{ !inputs.build_only && env.ENABLED_COVERAGE == 'true' }}
+        if: ${{ !inputs.build_only && env.COVERAGE_ENABLED == 'true' }}
        working-directory: ${{ env.BUILD_DIR }}
        env:
          BUILD_NPROC: ${{ steps.nproc.outputs.nproc }}
@@ -198,7 +234,7 @@ jobs:
            --target coverage

      - name: Upload coverage report
-        if: ${{ github.repository_owner == 'XRPLF' && !inputs.build_only && env.ENABLED_COVERAGE == 'true' }}
+        if: ${{ github.repository_owner == 'XRPLF' && !inputs.build_only && env.COVERAGE_ENABLED == 'true' }}
        uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
        with:
          disable_search: true
--- a/.github/workflows/reusable-build-test.yml
+++ b/.github/workflows/reusable-build-test.yml
@@ -8,16 +8,24 @@ name: Build and test
 on:
  workflow_call:
    inputs:
+      ccache_enabled:
+        description: "Whether to enable ccache."
+        required: false
+        type: boolean
+        default: false
+
      os:
        description: 'The operating system to use for the build ("linux", "macos", "windows").'
        required: true
        type: string
+
      strategy_matrix:
        # TODO: Support additional strategies, e.g. "ubuntu" for generating all Ubuntu configurations.
        description: 'The strategy matrix to use for generating the configurations ("minimal", "all").'
        required: false
        type: string
        default: "minimal"
+
    secrets:
      CODECOV_TOKEN:
        description: "The Codecov token to use for uploading coverage reports."
@@ -43,6 +51,7 @@ jobs:
    with:
      build_only: ${{ matrix.build_only }}
      build_type: ${{ matrix.build_type }}
+      ccache_enabled: ${{ inputs.ccache_enabled }}
      cmake_args: ${{ matrix.cmake_args }}
      cmake_target: ${{ matrix.cmake_target }}
      runs_on: ${{ toJSON(matrix.architecture.runner) }}
--- a/.github/workflows/upload-conan-deps.yml
+++ b/.github/workflows/upload-conan-deps.yml
@@ -72,7 +72,7 @@ jobs:
      - name: Prepare runner
        uses: XRPLF/actions/prepare-runner@2ece4ec6ab7de266859a6f053571425b2bd684b6
        with:
-          disable_ccache: false
+          disable_ccache: true

      - name: Print build environment
        uses: ./.github/actions/print-env
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -28,6 +28,9 @@ elseif(MSVC)
    add_compile_options(/wd4068) # Ignore unknown pragmas
 endif()

+# Enable ccache to speed up builds.
+include(Ccache)
+
 # make GIT_COMMIT_HASH define available to all sources
 find_package(Git)
 if(Git_FOUND)
--- a/cmake/Ccache.cmake
+++ b/cmake/Ccache.cmake
@@ -0,0 +1,51 @@
+find_program(CCACHE_PATH "ccache")
+if (NOT CCACHE_PATH)
+    return()
+endif ()
+
+# For Linux and macOS we can use the ccache binary directly.
+if (NOT MSVC)
+    set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_PATH}")
+    set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PATH}")
+    message(STATUS "Found ccache: ${CCACHE_PATH}")
+    return()
+endif ()
+
+# For Windows more effort is required. The code below is a modified version of
+# https://github.com/ccache/ccache/wiki/MS-Visual-Studio#usage-with-cmake.
+if ("${CCACHE_PATH}" MATCHES "chocolatey")
+    message(DEBUG "Ccache path: ${CCACHE_PATH}")
+    # Chocolatey uses a shim executable that we cannot use directly, in which
+    # case we have to find the executable it points to. If we cannot find the
+    # target executable then we cannot use ccache.
+    find_program(BASH_PATH "bash")
+    if (NOT BASH_PATH)
+        message(WARNING "Could not find bash.")
+        return()
+    endif ()
+
+    execute_process(
+            COMMAND bash -c "export LC_ALL='en_US.UTF-8'; ${CCACHE_PATH} --shimgen-noop | grep -oP 'path to executable: \\K.+' | head -c -1"
+            OUTPUT_VARIABLE CCACHE_PATH)
+
+    if (NOT CCACHE_PATH)
+        message(WARNING "Could not find ccache target.")
+        return()
+    endif ()
+    file(TO_CMAKE_PATH "${CCACHE_PATH}" CCACHE_PATH)
+endif ()
+message(STATUS "Found ccache: ${CCACHE_PATH}")
+
+# Tell cmake to use ccache for compiling with Visual Studio.
+file(COPY_FILE
+        ${CCACHE_PATH} ${CMAKE_BINARY_DIR}/cl.exe
+        ONLY_IF_DIFFERENT)
+set(CMAKE_VS_GLOBALS
+        "CLToolExe=cl.exe"
+        "CLToolPath=${CMAKE_BINARY_DIR}"
+        "TrackFileAccess=false"
+        "UseMultiToolTask=true")
+
+# By default Visual Studio generators will use /Zi, which is not compatible with
+# ccache, so tell it to use /Z7 instead.
+set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT "$<$<CONFIG:Debug,RelWithDebInfo>:Embedded>")