ci: add -Dcoverage_tool=gcov|llvm option, wire native llvm-cov in matrix

introduces native llvm source-based coverage as an alternative to the existing gcov + gcovr pipeline. driven by a new -Dcoverage_tool cache variable (default 'gcov' for backwards compat; 'llvm' enables -fprofile-instr-generate / -fcoverage-mapping + llvm-profdata + llvm-cov). cmake: - RippledSettings.cmake: add coverage_tool with validation - RippledInterface.cmake: split coverage compile/link flags by tool - RippledCov.cmake: dispatch to the new helper when tool=llvm - CodeCoverageLLVM.cmake (new): setup_target_for_coverage_llvm() driving profraw -> profdata -> export with format-aware output (lcov/json/txt/html) ci: - new clang-20 llvm-cov coverage matrix row (apt.llvm.org bootstrap for clang >= 19 since 24.04 default repos cap at clang-18) - conditional install of llvm-N vs gcovr based on coverage_tool - artifact + codecov upload generalised to coverage.lcov | coverage.xml - coverage cmake-args switched to *_FLAGS_DEBUG so the build action's stdlib flag isn't clobbered temp (revert before merging to dev): - matrix narrowed to just the llvm-cov row on non-main refs so we can iterate without burning runners - 'coverage-llm' added to push trigger for direct-push CI runs
2026-06-03 08:46:36 +00:00 · 2026-04-30 15:07:52 +07:00
parent a6186d7855
commit 7b8d671f52
5 changed files with 252 additions and 19 deletions
--- a/.github/workflows/xahau-ga-nix.yml
+++ b/.github/workflows/xahau-ga-nix.yml
@@ -2,7 +2,9 @@ name: Nix - GA Runner

 on:
  push:
-    branches: ["dev", "candidate", "release"]
+    # TEMP: coverage-llm added so direct pushes to the branch fire CI while
+    # we iterate on the llvm-cov pipeline. Revert before merging to dev.
+    branches: ["dev", "candidate", "release", "coverage-llm"]
  pull_request:
    branches: ["**"]
    types: [opened, synchronize, reopened, labeled, unlabeled]
@@ -80,7 +82,25 @@ jobs:
                  "compiler_version": 13,
                  "stdlib": "default",
                  "configuration": "Debug",
-                  "job_type": "coverage"
+                  "job_type": "coverage",
+                  "coverage_tool": "gcov",
+                  "coverage_format": "xml"
+              },
+              {
+                  # Latest stable Clang for the most accurate source-based
+                  # coverage mapping (newer language features, fewer bugs in
+                  # llvm-cov region inference). Pulled from apt.llvm.org since
+                  # Ubuntu 24.04 default repos cap at clang-18.
+                  "compiler_id": "clang-20-libcxx",
+                  "compiler": "clang",
+                  "cc": "clang-20",
+                  "cxx": "clang++-20",
+                  "compiler_version": 20,
+                  "stdlib": "libcxx",
+                  "configuration": "Debug",
+                  "job_type": "coverage",
+                  "coverage_tool": "llvm",
+                  "coverage_format": "lcov"
              },
              {
                  "compiler_id": "clang-14-libstdcxx-gcc11",
@@ -132,7 +152,8 @@ jobs:
          minimal_matrix = [
              full_matrix[1],  # gcc-13 (middle-ground gcc)
              full_matrix[2],  # gcc-13 coverage
-              full_matrix[3]   # clang-14 (mature, stable clang)
+              full_matrix[3],  # clang-20 llvm-cov coverage
+              full_matrix[4]   # clang-14 (mature, stable clang)
          ]
          
          # Determine which matrix to use based on the target branch
@@ -215,6 +236,14 @@ jobs:
              print(f"Using MINIMAL matrix (3 configs) - feature branch/PR")
              matrix = minimal_matrix

+          # TEMP (coverage-llm branch): narrow the matrix to just the new
+          # clang-20 llvm-cov coverage row so we can iterate on it without
+          # burning runners on the rest. Guarded so it can't accidentally
+          # apply to dev/candidate/release - revert before merging anyway.
+          if ref not in main_branches and base_ref not in ["dev", "candidate", "release"]:
+              matrix = [e for e in matrix if e.get("coverage_tool") == "llvm"]
+              print(f"TEMP override: matrix narrowed to {len(matrix)} llvm-cov row(s)")
+
          # Add runs_on based on job_type
          for entry in matrix:
              if entry.get("job_type") == "coverage":
@@ -260,6 +289,19 @@ jobs:
          apt-get update
          apt-get install -y software-properties-common
          add-apt-repository ppa:ubuntu-toolchain-r/test -y
+
+          # apt.llvm.org for Clang versions newer than what Ubuntu 24.04 ships
+          # (24.04 default repos cap at clang-18). The bootstrap script adds
+          # the LLVM apt source for the requested version and runs apt-get update.
+          if [ "${{ matrix.compiler }}" = "clang" ] && [ "${{ matrix.compiler_version }}" -ge 19 ]; then
+            apt-get install -y wget gnupg lsb-release
+            wget -qO /tmp/llvm.sh https://apt.llvm.org/llvm.sh
+            chmod +x /tmp/llvm.sh
+            # `all` installs clang + libllvm + lldb + lld + the llvm-N package
+            # (which provides llvm-profdata-N / llvm-cov-N for coverage runs).
+            /tmp/llvm.sh ${{ matrix.compiler_version }} all
+          fi
+
          apt-get update
          apt-get install -y git python3 python-is-python3 pipx
          pipx ensurepath
@@ -332,10 +374,16 @@ jobs:
          pipx install "conan>=2.0,<3"
          echo "$HOME/.local/bin" >> $GITHUB_PATH

-          # Install gcovr for coverage jobs
+          # Install coverage tooling
          if [ "${{ matrix.job_type }}" = "coverage" ]; then
-            pipx install "gcovr>=7,<9"
            apt-get install -y curl lcov
+            if [ "${{ matrix.coverage_tool }}" = "llvm" ]; then
+              # Native LLVM source-based coverage: llvm-profdata + llvm-cov.
+              # The clang-N package doesn't pull these in; the llvm-N package does.
+              apt-get install -y "llvm-${{ matrix.compiler_version }}"
+            else
+              pipx install "gcovr>=7,<9"
+            fi
          fi

      - name: Check environment
@@ -348,10 +396,15 @@ jobs:
          which ${{ matrix.cxx }} && ${{ matrix.cxx }} --version || echo "${{ matrix.cxx }} not found"
          which ccache && ccache --version || echo "ccache not found"

-          # Check gcovr for coverage jobs
+          # Check coverage tooling
          if [ "${{ matrix.job_type }}" = "coverage" ]; then
-            which gcov && gcov --version || echo "gcov not found"
-            which gcovr && gcovr --version || echo "gcovr not found"
+            if [ "${{ matrix.coverage_tool }}" = "llvm" ]; then
+              which "llvm-profdata-${{ matrix.compiler_version }}" && "llvm-profdata-${{ matrix.compiler_version }}" --version || echo "llvm-profdata not found"
+              which "llvm-cov-${{ matrix.compiler_version }}" && "llvm-cov-${{ matrix.compiler_version }}" --version || echo "llvm-cov not found"
+            else
+              which gcov && gcov --version || echo "gcov not found"
+              which gcovr && gcovr --version || echo "gcovr not found"
+            fi
          fi

          echo "---- Full Environment ----"
@@ -410,8 +463,9 @@ jobs:
          cache_version: ${{ env.CACHE_VERSION }}
          main_branch: ${{ env.MAIN_BRANCH_NAME }}
          stdlib: ${{ matrix.stdlib }}
-          # Coverage builds are slower due to instrumentation; use fewer parallel jobs to avoid flakiness
-          cmake-args: '-Dcoverage=ON -Dcoverage_format=xml -Dcoverage_test_parallelism=$(($(nproc)/2)) -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_CXX_FLAGS="-O0" -DCMAKE_C_FLAGS="-O0"'
+          # Coverage builds are slower due to instrumentation; use fewer parallel jobs to avoid flakiness.
+          # Use *_FLAGS_DEBUG so the build action's stdlib flag (e.g. -stdlib=libc++) in CMAKE_CXX_FLAGS isn't clobbered.
+          cmake-args: '-Dcoverage=ON -Dcoverage_tool=${{ matrix.coverage_tool }} -Dcoverage_format=${{ matrix.coverage_format }} -Dcoverage_test_parallelism=$(($(nproc)/2)) -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_CXX_FLAGS_DEBUG="-g -O0" -DCMAKE_C_FLAGS_DEBUG="-g -O0"'
          cmake-target: 'coverage'
          ccache_max_size: '100G'

@@ -443,22 +497,26 @@ jobs:
      - name: Move coverage report
        if: matrix.job_type == 'coverage'
        shell: bash
+        env:
+          COVERAGE_FILE: ${{ matrix.coverage_tool == 'llvm' && 'coverage.lcov' || 'coverage.xml' }}
        run: |
-          mv "${{ env.build_dir }}/coverage.xml" ./
+          mv "${{ env.build_dir }}/${COVERAGE_FILE}" ./
+          echo "COVERAGE_FILE=${COVERAGE_FILE}" >> "$GITHUB_ENV"

      - name: Archive coverage report
        if: matrix.job_type == 'coverage'
        uses: actions/upload-artifact@v4
        with:
-          name: coverage.xml
-          path: coverage.xml
+          name: ${{ env.COVERAGE_FILE }}-${{ matrix.compiler_id }}
+          path: ${{ env.COVERAGE_FILE }}
          retention-days: 30

      - name: Upload coverage report
        if: matrix.job_type == 'coverage'
        uses: codecov/codecov-action@v5
        with:
-          files: coverage.xml
+          files: ${{ env.COVERAGE_FILE }}
+          flags: ${{ matrix.coverage_tool }}
          fail_ci_if_error: true
          disable_search: true
          verbose: true
--- a/cmake/CodeCoverageLLVM.cmake
+++ b/cmake/CodeCoverageLLVM.cmake
@@ -0,0 +1,149 @@
+#[===================================================================[
+  Native LLVM source-based code coverage helper.
+
+  Drives the -fprofile-instr-generate / -fcoverage-mapping pipeline:
+    1. Run instrumented binary with LLVM_PROFILE_FILE=...%m-%p.profraw
+    2. llvm-profdata merge -sparse  -> coverage.profdata
+    3. llvm-cov export/show/report  -> final report
+
+  Output filename per coverage_format:
+    lcov              -> coverage.lcov
+    json              -> coverage.json
+    txt | text        -> coverage.txt
+    html | html-details -> <NAME>/index.html
+#]===================================================================]
+
+include(CMakeParseArguments)
+
+# Locate llvm-profdata / llvm-cov, preferring versioned variants matching the
+# Clang we're building with so we don't accidentally pair clang-20 with
+# llvm-cov-14 (profile format mismatch -> hard failure).
+function(_find_llvm_cov_tools)
+  if(LLVM_PROFDATA_PATH AND LLVM_COV_PATH)
+    return()
+  endif()
+
+  string(REGEX MATCH "^[0-9]+" _major "${CMAKE_CXX_COMPILER_VERSION}")
+
+  set(_pd_names llvm-profdata)
+  set(_cov_names llvm-cov)
+  if(_major)
+    list(PREPEND _pd_names "llvm-profdata-${_major}")
+    list(PREPEND _cov_names "llvm-cov-${_major}")
+  endif()
+
+  if(APPLE)
+    execute_process(COMMAND xcrun -f llvm-profdata
+      OUTPUT_VARIABLE _pd_xcrun OUTPUT_STRIP_TRAILING_WHITESPACE
+      ERROR_QUIET RESULT_VARIABLE _pd_rc)
+    if(_pd_rc EQUAL 0 AND _pd_xcrun)
+      set(LLVM_PROFDATA_PATH "${_pd_xcrun}" CACHE FILEPATH "llvm-profdata" FORCE)
+    endif()
+    execute_process(COMMAND xcrun -f llvm-cov
+      OUTPUT_VARIABLE _cov_xcrun OUTPUT_STRIP_TRAILING_WHITESPACE
+      ERROR_QUIET RESULT_VARIABLE _cov_rc)
+    if(_cov_rc EQUAL 0 AND _cov_xcrun)
+      set(LLVM_COV_PATH "${_cov_xcrun}" CACHE FILEPATH "llvm-cov" FORCE)
+    endif()
+  endif()
+
+  if(NOT LLVM_PROFDATA_PATH)
+    find_program(LLVM_PROFDATA_PATH NAMES ${_pd_names})
+  endif()
+  if(NOT LLVM_COV_PATH)
+    find_program(LLVM_COV_PATH NAMES ${_cov_names})
+  endif()
+endfunction()
+
+function(setup_target_for_coverage_llvm)
+  set(oneValueArgs NAME FORMAT BASE_DIRECTORY)
+  set(multiValueArgs EXCLUDE EXECUTABLE EXECUTABLE_ARGS DEPENDENCIES)
+  cmake_parse_arguments(Cov "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+  _find_llvm_cov_tools()
+  if(NOT LLVM_PROFDATA_PATH)
+    message(FATAL_ERROR "llvm-profdata not found (needed for coverage_tool=llvm)")
+  endif()
+  if(NOT LLVM_COV_PATH)
+    message(FATAL_ERROR "llvm-cov not found (needed for coverage_tool=llvm)")
+  endif()
+
+  if(NOT Cov_FORMAT)
+    set(Cov_FORMAT lcov)
+  endif()
+
+  set(_profraw_dir "${PROJECT_BINARY_DIR}/${Cov_NAME}-profraw")
+  set(_profdata    "${PROJECT_BINARY_DIR}/${Cov_NAME}.profdata")
+
+  # Resolve binary path: accept either an absolute path or a bare target name
+  # (resolved against PROJECT_BINARY_DIR).
+  list(GET Cov_EXECUTABLE 0 _exec_name)
+  if(IS_ABSOLUTE "${_exec_name}")
+    set(_binary "${_exec_name}")
+  else()
+    set(_binary "${PROJECT_BINARY_DIR}/${_exec_name}")
+  endif()
+
+  # llvm-cov takes a single -ignore-filename-regex; OR our excludes together.
+  set(_ignore_regex "")
+  foreach(EXC IN LISTS Cov_EXCLUDE)
+    if(_ignore_regex)
+      string(APPEND _ignore_regex "|")
+    endif()
+    string(APPEND _ignore_regex "${EXC}")
+  endforeach()
+  set(_filter "")
+  if(_ignore_regex)
+    set(_filter "-ignore-filename-regex='${_ignore_regex}'")
+  endif()
+
+  # Pick llvm-cov subcommand + output file for the requested format. Each
+  # branch builds a single shell command string that we'll hand to bash -c.
+  if(Cov_FORMAT STREQUAL "lcov")
+    set(_output "${PROJECT_BINARY_DIR}/coverage.lcov")
+    set(_report_sh "${LLVM_COV_PATH} export -instr-profile='${_profdata}' -format=lcov ${_filter} '${_binary}' > '${_output}'")
+  elseif(Cov_FORMAT STREQUAL "json")
+    set(_output "${PROJECT_BINARY_DIR}/coverage.json")
+    set(_report_sh "${LLVM_COV_PATH} export -instr-profile='${_profdata}' -format=text ${_filter} '${_binary}' > '${_output}'")
+  elseif(Cov_FORMAT STREQUAL "txt" OR Cov_FORMAT STREQUAL "text")
+    set(_output "${PROJECT_BINARY_DIR}/coverage.txt")
+    set(_report_sh "${LLVM_COV_PATH} report -instr-profile='${_profdata}' ${_filter} '${_binary}' > '${_output}'")
+  elseif(Cov_FORMAT STREQUAL "html" OR Cov_FORMAT STREQUAL "html-details")
+    set(_output "${PROJECT_BINARY_DIR}/${Cov_NAME}/index.html")
+    set(_report_sh "${LLVM_COV_PATH} show -instr-profile='${_profdata}' -format=html -output-dir='${PROJECT_BINARY_DIR}/${Cov_NAME}' ${_filter} '${_binary}'")
+  else()
+    message(FATAL_ERROR "coverage_tool=llvm: unsupported coverage_format '${Cov_FORMAT}' (use lcov|json|txt|html)")
+  endif()
+
+  set(_merge_sh "${LLVM_PROFDATA_PATH} merge -sparse -o '${_profdata}' '${_profraw_dir}'/*.profraw")
+
+  if(CODE_COVERAGE_VERBOSE)
+    message(STATUS "[coverage:llvm] binary:   ${_binary}")
+    message(STATUS "[coverage:llvm] profraw:  ${_profraw_dir}")
+    message(STATUS "[coverage:llvm] profdata: ${_profdata}")
+    message(STATUS "[coverage:llvm] format:   ${Cov_FORMAT}")
+    message(STATUS "[coverage:llvm] output:   ${_output}")
+    if(_ignore_regex)
+      message(STATUS "[coverage:llvm] ignore:   ${_ignore_regex}")
+    endif()
+    message(STATUS "[coverage:llvm] merge:    ${_merge_sh}")
+    message(STATUS "[coverage:llvm] report:   ${_report_sh}")
+  endif()
+
+  # %m: hash of the binary, %p: pid. Wipe the dir up front so stale profraw
+  # files can't leak into a fresh merge.
+  add_custom_target(${Cov_NAME}
+    COMMAND ${CMAKE_COMMAND} -E rm -rf "${_profraw_dir}"
+    COMMAND ${CMAKE_COMMAND} -E make_directory "${_profraw_dir}"
+    COMMAND ${CMAKE_COMMAND} -E env
+        "LLVM_PROFILE_FILE=${_profraw_dir}/rippled-%m-%p.profraw"
+        ${Cov_EXECUTABLE} ${Cov_EXECUTABLE_ARGS}
+    COMMAND bash -c "${_merge_sh}"
+    COMMAND bash -c "${_report_sh}"
+    BYPRODUCTS ${_output}
+    WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+    DEPENDS ${Cov_DEPENDENCIES}
+    VERBATIM
+    COMMENT "Running llvm-cov (${Cov_FORMAT}) -> ${_output}"
+  )
+endfunction()
--- a/cmake/RippledCov.cmake
+++ b/cmake/RippledCov.cmake
@@ -11,6 +11,21 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
  return()
 endif()

+if(coverage_tool STREQUAL "llvm")
+  include(CodeCoverageLLVM)
+
+  setup_target_for_coverage_llvm(
+    NAME coverage
+    FORMAT ${coverage_format}
+    EXECUTABLE rippled
+    EXECUTABLE_ARGS --unittest$<$<BOOL:${coverage_test}>:=${coverage_test}> --unittest-jobs ${coverage_test_parallelism} --quiet --unittest-log
+    EXCLUDE "src/test" "include/xrpl/beast/test" "include/xrpl/beast/unit_test" "${CMAKE_BINARY_DIR}/pb-xrpl.libpb"
+    DEPENDENCIES rippled
+  )
+  return()
+endif()
+
+# coverage_tool == "gcov" (default): existing gcovr-driven pipeline.
 include(CodeCoverage)

 # The instructions for these commands come from the `CodeCoverage` module,
--- a/cmake/RippledInterface.cmake
+++ b/cmake/RippledInterface.cmake
@@ -28,15 +28,17 @@ target_compile_options (opts
    $<$<AND:$<BOOL:${is_gcc}>,$<COMPILE_LANGUAGE:CXX>>:-Wsuggest-override>
    $<$<BOOL:${is_gcc}>:-Wno-maybe-uninitialized>
    $<$<BOOL:${perf}>:-fno-omit-frame-pointer>
-    $<$<AND:$<BOOL:${is_gcc}>,$<BOOL:${coverage}>>:-g --coverage -fprofile-abs-path>
-    $<$<AND:$<BOOL:${is_clang}>,$<BOOL:${coverage}>>:-g --coverage>
+    $<$<AND:$<BOOL:${is_gcc}>,$<BOOL:${coverage}>,$<STREQUAL:${coverage_tool},gcov>>:-g --coverage -fprofile-abs-path>
+    $<$<AND:$<BOOL:${is_clang}>,$<BOOL:${coverage}>,$<STREQUAL:${coverage_tool},gcov>>:-g --coverage>
+    $<$<AND:$<BOOL:${is_clang}>,$<BOOL:${coverage}>,$<STREQUAL:${coverage_tool},llvm>>:-g -fprofile-instr-generate -fcoverage-mapping>
    $<$<BOOL:${profile}>:-pg>
    $<$<AND:$<BOOL:${is_gcc}>,$<BOOL:${profile}>>:-p>)

 target_link_libraries (opts
  INTERFACE
-    $<$<AND:$<BOOL:${is_gcc}>,$<BOOL:${coverage}>>:-g --coverage -fprofile-abs-path>
-    $<$<AND:$<BOOL:${is_clang}>,$<BOOL:${coverage}>>:-g --coverage>
+    $<$<AND:$<BOOL:${is_gcc}>,$<BOOL:${coverage}>,$<STREQUAL:${coverage_tool},gcov>>:-g --coverage -fprofile-abs-path>
+    $<$<AND:$<BOOL:${is_clang}>,$<BOOL:${coverage}>,$<STREQUAL:${coverage_tool},gcov>>:-g --coverage>
+    $<$<AND:$<BOOL:${is_clang}>,$<BOOL:${coverage}>,$<STREQUAL:${coverage_tool},llvm>>:-g -fprofile-instr-generate -fcoverage-mapping>
    $<$<BOOL:${profile}>:-pg>
    $<$<AND:$<BOOL:${is_gcc}>,$<BOOL:${profile}>>:-p>)

--- a/cmake/RippledSettings.cmake
+++ b/cmake/RippledSettings.cmake
@@ -29,8 +29,17 @@ if(is_gcc OR is_clang)
    "Unit tests parallelism for the purpose of coverage report.")
  set(coverage_format "html-details" CACHE STRING
    "Output format of the coverage report.")
+  set(coverage_tool "gcov" CACHE STRING
+    "Coverage instrumentation tool: 'gcov' (default, gcc/clang via --coverage + gcovr) or 'llvm' (clang only, native source-based coverage via -fprofile-instr-generate).")
+  set_property(CACHE coverage_tool PROPERTY STRINGS "gcov" "llvm")
+  if(NOT coverage_tool MATCHES "^(gcov|llvm)$")
+    message(FATAL_ERROR "coverage_tool must be 'gcov' or 'llvm', got '${coverage_tool}'")
+  endif()
+  if(coverage AND coverage_tool STREQUAL "llvm" AND NOT is_clang)
+    message(FATAL_ERROR "coverage_tool=llvm requires Clang (got ${CMAKE_CXX_COMPILER_ID})")
+  endif()
  set(coverage_extra_args "" CACHE STRING
-    "Additional arguments to pass to gcovr.")
+    "Additional arguments to pass to gcovr (gcov tool only).")
  set(coverage_test "" CACHE STRING
    "On gcc & clang, the specific unit test(s) to run for coverage. Default is all tests.")
  if(coverage_test AND NOT coverage)