From 7b8d671f5271d56ea00b102ced8239759d8b190c Mon Sep 17 00:00:00 2001 From: Nicholas Dudfield Date: Thu, 30 Apr 2026 15:07:52 +0700 Subject: [PATCH] ci: add -Dcoverage_tool=gcov|llvm option, wire native llvm-cov in matrix introduces native llvm source-based coverage as an alternative to the existing gcov + gcovr pipeline. driven by a new -Dcoverage_tool cache variable (default 'gcov' for backwards compat; 'llvm' enables -fprofile-instr-generate / -fcoverage-mapping + llvm-profdata + llvm-cov). cmake: - RippledSettings.cmake: add coverage_tool with validation - RippledInterface.cmake: split coverage compile/link flags by tool - RippledCov.cmake: dispatch to the new helper when tool=llvm - CodeCoverageLLVM.cmake (new): setup_target_for_coverage_llvm() driving profraw -> profdata -> export with format-aware output (lcov/json/txt/html) ci: - new clang-20 llvm-cov coverage matrix row (apt.llvm.org bootstrap for clang >= 19 since 24.04 default repos cap at clang-18) - conditional install of llvm-N vs gcovr based on coverage_tool - artifact + codecov upload generalised to coverage.lcov | coverage.xml - coverage cmake-args switched to *_FLAGS_DEBUG so the build action's stdlib flag isn't clobbered temp (revert before merging to dev): - matrix narrowed to just the llvm-cov row on non-main refs so we can iterate without burning runners - 'coverage-llm' added to push trigger for direct-push CI runs --- .github/workflows/xahau-ga-nix.yml | 86 ++++++++++++++--- cmake/CodeCoverageLLVM.cmake | 149 +++++++++++++++++++++++++++++ cmake/RippledCov.cmake | 15 +++ cmake/RippledInterface.cmake | 10 +- cmake/RippledSettings.cmake | 11 ++- 5 files changed, 252 insertions(+), 19 deletions(-) create mode 100644 cmake/CodeCoverageLLVM.cmake diff --git a/.github/workflows/xahau-ga-nix.yml b/.github/workflows/xahau-ga-nix.yml index a2636bbb8..c614a3b34 100644 --- a/.github/workflows/xahau-ga-nix.yml +++ b/.github/workflows/xahau-ga-nix.yml @@ -2,7 +2,9 @@ name: Nix - GA Runner on: push: - branches: ["dev", "candidate", "release"] + # TEMP: coverage-llm added so direct pushes to the branch fire CI while + # we iterate on the llvm-cov pipeline. Revert before merging to dev. + branches: ["dev", "candidate", "release", "coverage-llm"] pull_request: branches: ["**"] types: [opened, synchronize, reopened, labeled, unlabeled] @@ -80,7 +82,25 @@ jobs: "compiler_version": 13, "stdlib": "default", "configuration": "Debug", - "job_type": "coverage" + "job_type": "coverage", + "coverage_tool": "gcov", + "coverage_format": "xml" + }, + { + # Latest stable Clang for the most accurate source-based + # coverage mapping (newer language features, fewer bugs in + # llvm-cov region inference). Pulled from apt.llvm.org since + # Ubuntu 24.04 default repos cap at clang-18. + "compiler_id": "clang-20-libcxx", + "compiler": "clang", + "cc": "clang-20", + "cxx": "clang++-20", + "compiler_version": 20, + "stdlib": "libcxx", + "configuration": "Debug", + "job_type": "coverage", + "coverage_tool": "llvm", + "coverage_format": "lcov" }, { "compiler_id": "clang-14-libstdcxx-gcc11", @@ -132,7 +152,8 @@ jobs: minimal_matrix = [ full_matrix[1], # gcc-13 (middle-ground gcc) full_matrix[2], # gcc-13 coverage - full_matrix[3] # clang-14 (mature, stable clang) + full_matrix[3], # clang-20 llvm-cov coverage + full_matrix[4] # clang-14 (mature, stable clang) ] # Determine which matrix to use based on the target branch @@ -215,6 +236,14 @@ jobs: print(f"Using MINIMAL matrix (3 configs) - feature branch/PR") matrix = minimal_matrix + # TEMP (coverage-llm branch): narrow the matrix to just the new + # clang-20 llvm-cov coverage row so we can iterate on it without + # burning runners on the rest. Guarded so it can't accidentally + # apply to dev/candidate/release - revert before merging anyway. + if ref not in main_branches and base_ref not in ["dev", "candidate", "release"]: + matrix = [e for e in matrix if e.get("coverage_tool") == "llvm"] + print(f"TEMP override: matrix narrowed to {len(matrix)} llvm-cov row(s)") + # Add runs_on based on job_type for entry in matrix: if entry.get("job_type") == "coverage": @@ -260,6 +289,19 @@ jobs: apt-get update apt-get install -y software-properties-common add-apt-repository ppa:ubuntu-toolchain-r/test -y + + # apt.llvm.org for Clang versions newer than what Ubuntu 24.04 ships + # (24.04 default repos cap at clang-18). The bootstrap script adds + # the LLVM apt source for the requested version and runs apt-get update. + if [ "${{ matrix.compiler }}" = "clang" ] && [ "${{ matrix.compiler_version }}" -ge 19 ]; then + apt-get install -y wget gnupg lsb-release + wget -qO /tmp/llvm.sh https://apt.llvm.org/llvm.sh + chmod +x /tmp/llvm.sh + # `all` installs clang + libllvm + lldb + lld + the llvm-N package + # (which provides llvm-profdata-N / llvm-cov-N for coverage runs). + /tmp/llvm.sh ${{ matrix.compiler_version }} all + fi + apt-get update apt-get install -y git python3 python-is-python3 pipx pipx ensurepath @@ -332,10 +374,16 @@ jobs: pipx install "conan>=2.0,<3" echo "$HOME/.local/bin" >> $GITHUB_PATH - # Install gcovr for coverage jobs + # Install coverage tooling if [ "${{ matrix.job_type }}" = "coverage" ]; then - pipx install "gcovr>=7,<9" apt-get install -y curl lcov + if [ "${{ matrix.coverage_tool }}" = "llvm" ]; then + # Native LLVM source-based coverage: llvm-profdata + llvm-cov. + # The clang-N package doesn't pull these in; the llvm-N package does. + apt-get install -y "llvm-${{ matrix.compiler_version }}" + else + pipx install "gcovr>=7,<9" + fi fi - name: Check environment @@ -348,10 +396,15 @@ jobs: which ${{ matrix.cxx }} && ${{ matrix.cxx }} --version || echo "${{ matrix.cxx }} not found" which ccache && ccache --version || echo "ccache not found" - # Check gcovr for coverage jobs + # Check coverage tooling if [ "${{ matrix.job_type }}" = "coverage" ]; then - which gcov && gcov --version || echo "gcov not found" - which gcovr && gcovr --version || echo "gcovr not found" + if [ "${{ matrix.coverage_tool }}" = "llvm" ]; then + which "llvm-profdata-${{ matrix.compiler_version }}" && "llvm-profdata-${{ matrix.compiler_version }}" --version || echo "llvm-profdata not found" + which "llvm-cov-${{ matrix.compiler_version }}" && "llvm-cov-${{ matrix.compiler_version }}" --version || echo "llvm-cov not found" + else + which gcov && gcov --version || echo "gcov not found" + which gcovr && gcovr --version || echo "gcovr not found" + fi fi echo "---- Full Environment ----" @@ -410,8 +463,9 @@ jobs: cache_version: ${{ env.CACHE_VERSION }} main_branch: ${{ env.MAIN_BRANCH_NAME }} stdlib: ${{ matrix.stdlib }} - # Coverage builds are slower due to instrumentation; use fewer parallel jobs to avoid flakiness - cmake-args: '-Dcoverage=ON -Dcoverage_format=xml -Dcoverage_test_parallelism=$(($(nproc)/2)) -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_CXX_FLAGS="-O0" -DCMAKE_C_FLAGS="-O0"' + # Coverage builds are slower due to instrumentation; use fewer parallel jobs to avoid flakiness. + # Use *_FLAGS_DEBUG so the build action's stdlib flag (e.g. -stdlib=libc++) in CMAKE_CXX_FLAGS isn't clobbered. + cmake-args: '-Dcoverage=ON -Dcoverage_tool=${{ matrix.coverage_tool }} -Dcoverage_format=${{ matrix.coverage_format }} -Dcoverage_test_parallelism=$(($(nproc)/2)) -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_CXX_FLAGS_DEBUG="-g -O0" -DCMAKE_C_FLAGS_DEBUG="-g -O0"' cmake-target: 'coverage' ccache_max_size: '100G' @@ -443,22 +497,26 @@ jobs: - name: Move coverage report if: matrix.job_type == 'coverage' shell: bash + env: + COVERAGE_FILE: ${{ matrix.coverage_tool == 'llvm' && 'coverage.lcov' || 'coverage.xml' }} run: | - mv "${{ env.build_dir }}/coverage.xml" ./ + mv "${{ env.build_dir }}/${COVERAGE_FILE}" ./ + echo "COVERAGE_FILE=${COVERAGE_FILE}" >> "$GITHUB_ENV" - name: Archive coverage report if: matrix.job_type == 'coverage' uses: actions/upload-artifact@v4 with: - name: coverage.xml - path: coverage.xml + name: ${{ env.COVERAGE_FILE }}-${{ matrix.compiler_id }} + path: ${{ env.COVERAGE_FILE }} retention-days: 30 - name: Upload coverage report if: matrix.job_type == 'coverage' uses: codecov/codecov-action@v5 with: - files: coverage.xml + files: ${{ env.COVERAGE_FILE }} + flags: ${{ matrix.coverage_tool }} fail_ci_if_error: true disable_search: true verbose: true diff --git a/cmake/CodeCoverageLLVM.cmake b/cmake/CodeCoverageLLVM.cmake new file mode 100644 index 000000000..b89098f61 --- /dev/null +++ b/cmake/CodeCoverageLLVM.cmake @@ -0,0 +1,149 @@ +#[===================================================================[ + Native LLVM source-based code coverage helper. + + Drives the -fprofile-instr-generate / -fcoverage-mapping pipeline: + 1. Run instrumented binary with LLVM_PROFILE_FILE=...%m-%p.profraw + 2. llvm-profdata merge -sparse -> coverage.profdata + 3. llvm-cov export/show/report -> final report + + Output filename per coverage_format: + lcov -> coverage.lcov + json -> coverage.json + txt | text -> coverage.txt + html | html-details -> /index.html +#]===================================================================] + +include(CMakeParseArguments) + +# Locate llvm-profdata / llvm-cov, preferring versioned variants matching the +# Clang we're building with so we don't accidentally pair clang-20 with +# llvm-cov-14 (profile format mismatch -> hard failure). +function(_find_llvm_cov_tools) + if(LLVM_PROFDATA_PATH AND LLVM_COV_PATH) + return() + endif() + + string(REGEX MATCH "^[0-9]+" _major "${CMAKE_CXX_COMPILER_VERSION}") + + set(_pd_names llvm-profdata) + set(_cov_names llvm-cov) + if(_major) + list(PREPEND _pd_names "llvm-profdata-${_major}") + list(PREPEND _cov_names "llvm-cov-${_major}") + endif() + + if(APPLE) + execute_process(COMMAND xcrun -f llvm-profdata + OUTPUT_VARIABLE _pd_xcrun OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET RESULT_VARIABLE _pd_rc) + if(_pd_rc EQUAL 0 AND _pd_xcrun) + set(LLVM_PROFDATA_PATH "${_pd_xcrun}" CACHE FILEPATH "llvm-profdata" FORCE) + endif() + execute_process(COMMAND xcrun -f llvm-cov + OUTPUT_VARIABLE _cov_xcrun OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET RESULT_VARIABLE _cov_rc) + if(_cov_rc EQUAL 0 AND _cov_xcrun) + set(LLVM_COV_PATH "${_cov_xcrun}" CACHE FILEPATH "llvm-cov" FORCE) + endif() + endif() + + if(NOT LLVM_PROFDATA_PATH) + find_program(LLVM_PROFDATA_PATH NAMES ${_pd_names}) + endif() + if(NOT LLVM_COV_PATH) + find_program(LLVM_COV_PATH NAMES ${_cov_names}) + endif() +endfunction() + +function(setup_target_for_coverage_llvm) + set(oneValueArgs NAME FORMAT BASE_DIRECTORY) + set(multiValueArgs EXCLUDE EXECUTABLE EXECUTABLE_ARGS DEPENDENCIES) + cmake_parse_arguments(Cov "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + _find_llvm_cov_tools() + if(NOT LLVM_PROFDATA_PATH) + message(FATAL_ERROR "llvm-profdata not found (needed for coverage_tool=llvm)") + endif() + if(NOT LLVM_COV_PATH) + message(FATAL_ERROR "llvm-cov not found (needed for coverage_tool=llvm)") + endif() + + if(NOT Cov_FORMAT) + set(Cov_FORMAT lcov) + endif() + + set(_profraw_dir "${PROJECT_BINARY_DIR}/${Cov_NAME}-profraw") + set(_profdata "${PROJECT_BINARY_DIR}/${Cov_NAME}.profdata") + + # Resolve binary path: accept either an absolute path or a bare target name + # (resolved against PROJECT_BINARY_DIR). + list(GET Cov_EXECUTABLE 0 _exec_name) + if(IS_ABSOLUTE "${_exec_name}") + set(_binary "${_exec_name}") + else() + set(_binary "${PROJECT_BINARY_DIR}/${_exec_name}") + endif() + + # llvm-cov takes a single -ignore-filename-regex; OR our excludes together. + set(_ignore_regex "") + foreach(EXC IN LISTS Cov_EXCLUDE) + if(_ignore_regex) + string(APPEND _ignore_regex "|") + endif() + string(APPEND _ignore_regex "${EXC}") + endforeach() + set(_filter "") + if(_ignore_regex) + set(_filter "-ignore-filename-regex='${_ignore_regex}'") + endif() + + # Pick llvm-cov subcommand + output file for the requested format. Each + # branch builds a single shell command string that we'll hand to bash -c. + if(Cov_FORMAT STREQUAL "lcov") + set(_output "${PROJECT_BINARY_DIR}/coverage.lcov") + set(_report_sh "${LLVM_COV_PATH} export -instr-profile='${_profdata}' -format=lcov ${_filter} '${_binary}' > '${_output}'") + elseif(Cov_FORMAT STREQUAL "json") + set(_output "${PROJECT_BINARY_DIR}/coverage.json") + set(_report_sh "${LLVM_COV_PATH} export -instr-profile='${_profdata}' -format=text ${_filter} '${_binary}' > '${_output}'") + elseif(Cov_FORMAT STREQUAL "txt" OR Cov_FORMAT STREQUAL "text") + set(_output "${PROJECT_BINARY_DIR}/coverage.txt") + set(_report_sh "${LLVM_COV_PATH} report -instr-profile='${_profdata}' ${_filter} '${_binary}' > '${_output}'") + elseif(Cov_FORMAT STREQUAL "html" OR Cov_FORMAT STREQUAL "html-details") + set(_output "${PROJECT_BINARY_DIR}/${Cov_NAME}/index.html") + set(_report_sh "${LLVM_COV_PATH} show -instr-profile='${_profdata}' -format=html -output-dir='${PROJECT_BINARY_DIR}/${Cov_NAME}' ${_filter} '${_binary}'") + else() + message(FATAL_ERROR "coverage_tool=llvm: unsupported coverage_format '${Cov_FORMAT}' (use lcov|json|txt|html)") + endif() + + set(_merge_sh "${LLVM_PROFDATA_PATH} merge -sparse -o '${_profdata}' '${_profraw_dir}'/*.profraw") + + if(CODE_COVERAGE_VERBOSE) + message(STATUS "[coverage:llvm] binary: ${_binary}") + message(STATUS "[coverage:llvm] profraw: ${_profraw_dir}") + message(STATUS "[coverage:llvm] profdata: ${_profdata}") + message(STATUS "[coverage:llvm] format: ${Cov_FORMAT}") + message(STATUS "[coverage:llvm] output: ${_output}") + if(_ignore_regex) + message(STATUS "[coverage:llvm] ignore: ${_ignore_regex}") + endif() + message(STATUS "[coverage:llvm] merge: ${_merge_sh}") + message(STATUS "[coverage:llvm] report: ${_report_sh}") + endif() + + # %m: hash of the binary, %p: pid. Wipe the dir up front so stale profraw + # files can't leak into a fresh merge. + add_custom_target(${Cov_NAME} + COMMAND ${CMAKE_COMMAND} -E rm -rf "${_profraw_dir}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${_profraw_dir}" + COMMAND ${CMAKE_COMMAND} -E env + "LLVM_PROFILE_FILE=${_profraw_dir}/rippled-%m-%p.profraw" + ${Cov_EXECUTABLE} ${Cov_EXECUTABLE_ARGS} + COMMAND bash -c "${_merge_sh}" + COMMAND bash -c "${_report_sh}" + BYPRODUCTS ${_output} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + DEPENDS ${Cov_DEPENDENCIES} + VERBATIM + COMMENT "Running llvm-cov (${Cov_FORMAT}) -> ${_output}" + ) +endfunction() diff --git a/cmake/RippledCov.cmake b/cmake/RippledCov.cmake index 3c48bb1c1..02ce3c6ad 100644 --- a/cmake/RippledCov.cmake +++ b/cmake/RippledCov.cmake @@ -11,6 +11,21 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") return() endif() +if(coverage_tool STREQUAL "llvm") + include(CodeCoverageLLVM) + + setup_target_for_coverage_llvm( + NAME coverage + FORMAT ${coverage_format} + EXECUTABLE rippled + EXECUTABLE_ARGS --unittest$<$:=${coverage_test}> --unittest-jobs ${coverage_test_parallelism} --quiet --unittest-log + EXCLUDE "src/test" "include/xrpl/beast/test" "include/xrpl/beast/unit_test" "${CMAKE_BINARY_DIR}/pb-xrpl.libpb" + DEPENDENCIES rippled + ) + return() +endif() + +# coverage_tool == "gcov" (default): existing gcovr-driven pipeline. include(CodeCoverage) # The instructions for these commands come from the `CodeCoverage` module, diff --git a/cmake/RippledInterface.cmake b/cmake/RippledInterface.cmake index 93a973ac7..44495ef49 100644 --- a/cmake/RippledInterface.cmake +++ b/cmake/RippledInterface.cmake @@ -28,15 +28,17 @@ target_compile_options (opts $<$,$>:-Wsuggest-override> $<$:-Wno-maybe-uninitialized> $<$:-fno-omit-frame-pointer> - $<$,$>:-g --coverage -fprofile-abs-path> - $<$,$>:-g --coverage> + $<$,$,$>:-g --coverage -fprofile-abs-path> + $<$,$,$>:-g --coverage> + $<$,$,$>:-g -fprofile-instr-generate -fcoverage-mapping> $<$:-pg> $<$,$>:-p>) target_link_libraries (opts INTERFACE - $<$,$>:-g --coverage -fprofile-abs-path> - $<$,$>:-g --coverage> + $<$,$,$>:-g --coverage -fprofile-abs-path> + $<$,$,$>:-g --coverage> + $<$,$,$>:-g -fprofile-instr-generate -fcoverage-mapping> $<$:-pg> $<$,$>:-p>) diff --git a/cmake/RippledSettings.cmake b/cmake/RippledSettings.cmake index 58877e188..68f6c49ea 100644 --- a/cmake/RippledSettings.cmake +++ b/cmake/RippledSettings.cmake @@ -29,8 +29,17 @@ if(is_gcc OR is_clang) "Unit tests parallelism for the purpose of coverage report.") set(coverage_format "html-details" CACHE STRING "Output format of the coverage report.") + set(coverage_tool "gcov" CACHE STRING + "Coverage instrumentation tool: 'gcov' (default, gcc/clang via --coverage + gcovr) or 'llvm' (clang only, native source-based coverage via -fprofile-instr-generate).") + set_property(CACHE coverage_tool PROPERTY STRINGS "gcov" "llvm") + if(NOT coverage_tool MATCHES "^(gcov|llvm)$") + message(FATAL_ERROR "coverage_tool must be 'gcov' or 'llvm', got '${coverage_tool}'") + endif() + if(coverage AND coverage_tool STREQUAL "llvm" AND NOT is_clang) + message(FATAL_ERROR "coverage_tool=llvm requires Clang (got ${CMAKE_CXX_COMPILER_ID})") + endif() set(coverage_extra_args "" CACHE STRING - "Additional arguments to pass to gcovr.") + "Additional arguments to pass to gcovr (gcov tool only).") set(coverage_test "" CACHE STRING "On gcc & clang, the specific unit test(s) to run for coverage. Default is all tests.") if(coverage_test AND NOT coverage)