Improve codecov builds:

- allow private token for jenkins/codecov
- add custom targets for gcc/clang to generate codecov reports
- use CMake coverage target in jenkins build
- optional coverage_test argument when configuring the build

CMakeLists.txt

@@ -81,6 +81,10 @@ if (NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
     "The most likely cause of this warning is trying to build rippled with a 32-bit OS.")
 endif ()
 
+if (APPLE AND NOT HOMEBREW)
+  find_program (HOMEBREW brew)
+endif ()
+
 #[===================================================================[
    read version from source
 #]===================================================================]
@@ -106,6 +110,11 @@ option (unity "Creates a build based on unity sources. This is the default" ON)
 if (is_gcc OR is_clang)
   option (coverage "Generates coverage info." OFF)
   option (profile "Add profiling flags" OFF)
+  set (coverage_test "" CACHE STRING
+    "On gcc & clang, the specific unit test(s) to run for coverage. Default is all tests.")
+  if (coverage_test AND NOT coverage)
+    set (coverage ON CACHE BOOL "gcc/clang only" FORCE)
+  endif ()
 else ()
   set (profile OFF CACHE BOOL "gcc/clang only" FORCE)
   set (coverage OFF CACHE BOOL "gcc/clang only" FORCE)
@@ -384,13 +393,15 @@ target_compile_options (opts
   INTERFACE
     $<$<AND:$<BOOL:${is_gcc}>,$<COMPILE_LANGUAGE:CXX>>:-Wsuggest-override>
     $<$<BOOL:${perf}>:-fno-omit-frame-pointer>
-    $<$<BOOL:${coverage}>:-fprofile-arcs -ftest-coverage>
+    $<$<AND:$<BOOL:${is_gcc}>,$<BOOL:${coverage}>>:-fprofile-arcs -ftest-coverage>
+    $<$<AND:$<BOOL:${is_clang}>,$<BOOL:${coverage}>>:-fprofile-instr-generate -fcoverage-mapping>
     $<$<BOOL:${profile}>:-pg>
     $<$<AND:$<BOOL:${is_gcc}>,$<BOOL:${profile}>>:-p>)
 
 target_link_libraries (opts
   INTERFACE
-    $<$<BOOL:${coverage}>:-fprofile-arcs -ftest-coverage>
+    $<$<AND:$<BOOL:${is_gcc}>,$<BOOL:${coverage}>>:-fprofile-arcs -ftest-coverage>
+    $<$<AND:$<BOOL:${is_clang}>,$<BOOL:${coverage}>>:-fprofile-instr-generate -fcoverage-mapping>
     $<$<BOOL:${profile}>:-pg>
     $<$<AND:$<BOOL:${is_gcc}>,$<BOOL:${profile}>>:-p>)
 
@@ -575,14 +586,11 @@ target_link_libraries (ripple_boost
 if (NOT DEFINED OPENSSL_ROOT_DIR)
   if (DEFINED ENV{OPENSSL_ROOT})
     set (OPENSSL_ROOT_DIR $ENV{OPENSSL_ROOT})
-  elseif (APPLE)
+  elseif (HOMEBREW)
-    find_program (homebrew brew)
+    execute_process (COMMAND ${HOMEBREW} --prefix openssl
-    if (homebrew)
-      execute_process (COMMAND ${homebrew} --prefix openssl
       OUTPUT_VARIABLE OPENSSL_ROOT_DIR
       OUTPUT_STRIP_TRAILING_WHITESPACE)
   endif ()
-  endif ()
   file (TO_CMAKE_PATH "${OPENSSL_ROOT_DIR}" OPENSSL_ROOT_DIR)
 endif ()
 
@@ -2331,6 +2339,87 @@ if (is_root_project)
   ")
 endif ()
 
+#[===================================================================[
+   coverage report target
+#]===================================================================]
+
+if (coverage)
+  if (is_clang)
+    if (APPLE)
+      execute_process (COMMAND xcrun -f llvm-profdata
+        OUTPUT_VARIABLE LLVM_PROFDATA
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
+    else ()
+      find_program (LLVM_PROFDATA llvm-profdata)
+    endif ()
+    if (NOT LLVM_PROFDATA)
+      message (WARNING "unable to find llvm-profdata - skipping coverage_report target")
+    endif ()
+
+    if (APPLE)
+      execute_process (COMMAND xcrun -f llvm-cov
+        OUTPUT_VARIABLE LLVM_COV
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
+    else ()
+      find_program (LLVM_COV llvm-cov)
+    endif ()
+    if (NOT LLVM_COV)
+      message (WARNING "unable to find llvm-cov - skipping coverage_report target")
+    endif ()
+
+    if (LLVM_COV AND LLVM_PROFDATA)
+      add_custom_target (coverage_report
+        USES_TERMINAL
+        COMMAND ${CMAKE_COMMAND} -E echo "Generating coverage - results will be in ${CMAKE_BINARY_DIR}/coverage/index.html."
+        COMMAND ${CMAKE_COMMAND} -E echo "Running rippled tests."
+        COMMAND rippled --unittest$<$<BOOL:${coverage_test}>:=${coverage_test}> --quiet --unittest-log
+        COMMAND ${LLVM_PROFDATA}
+          merge -sparse default.profraw -o rip.profdata
+        COMMAND ${LLVM_COV}
+          show -format=html -output-dir=${CMAKE_BINARY_DIR}/coverage
+          $<TARGET_FILE:rippled> -instr-profile=rip.profdata
+        BYPRODUCTS coverage/index.html)
+    endif ()
+  elseif (is_gcc)
+    find_program (LCOV lcov)
+    if (NOT LCOV)
+      message (WARNING "unable to find lcov - skipping coverage_report target")
+    endif ()
+
+    find_program (GENHTML genhtml)
+    if (NOT GENHTML)
+      message (WARNING "unable to find genhtml - skipping coverage_report target")
+    endif ()
+
+    if (LCOV AND GENHTML)
+      add_custom_target (coverage_report
+        USES_TERMINAL
+        COMMAND ${CMAKE_COMMAND} -E echo "Generating coverage- results will be in ${CMAKE_BINARY_DIR}/coverage/index.html."
+        # create baseline info file
+        COMMAND ${LCOV}
+          --no-external -d "${CMAKE_SOURCE_DIR}" -c -d . -i -o baseline.info
+          | grep -v "ignoring data for external file"
+        # run tests
+        COMMAND ${CMAKE_COMMAND} -E echo "Running rippled tests for coverage report."
+        COMMAND rippled --unittest$<$<BOOL:${coverage_test}>:=${coverage_test}> --quiet --unittest-log
+        # Create test coverage data file
+        COMMAND ${LCOV}
+          --no-external -d "${CMAKE_SOURCE_DIR}" -c -d . -o tests.info
+          | grep -v "ignoring data for external file"
+        # Combine baseline and test coverage data
+        COMMAND ${LCOV}
+          -a baseline.info -a tests.info -o lcov-all.info
+        # extract our files
+        COMMAND ${LCOV}
+          -e lcov-all.info "*/src/ripple/*" -o lcov.info
+        # generate HTML report
+        COMMAND ${GENHTML}
+          -o ${CMAKE_BINARY_DIR}/coverage lcov.info
+        BYPRODUCTS coverage/index.html)
+    endif ()
+  endif ()
+endif ()
+
 #[===================================================================[
    multiconfig misc
 #]===================================================================]

Jenkinsfile

@@ -111,8 +111,8 @@ try {
     stage ('Parallel Build') {
         String[][] variants = [
             ['gcc.Release'    ,'-Dassert=ON'     ,'MANUAL_TESTS=true'     ],
-            ['gcc.Debug'      ,'-Dcoverage=ON'                            ],
+            ['gcc.Debug'      ,'-Dcoverage=ON'   ,'TARGET=coverage_report', 'SKIP_TESTS=true'],
-            ['docs'                                                       ],
+            ['docs'           ,''                ,'TARGET=docs'           ],
             ['msvc.Debug'                                                 ],
             ['msvc.Debug'     ,''                ,'NINJA_BUILD=true'      ],
             ['msvc.Debug'     ,'-Dunity=OFF'                              ],
@@ -220,11 +220,26 @@ try {
                         env_vars.addAll(extra_env)
                     }
 
-                    withCredentials(
+                    // try to figure out codecov token to use. Look for
-                        [string(
+                    // MY_CODECOV_TOKEN id first so users can set that
-                            credentialsId: 'RIPPLED_CODECOV_TOKEN',
+                    // on job scope but then default to RIPPLED_CODECOV_TOKEN
-                            variable: 'CODECOV_TOKEN')])
+                    // which should be globally scoped
-                    {
+                    def codecov_token = ''
+                    try {
+                        withCredentials( [string( credentialsId: 'MY_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) {
+                            codecov_token = env.CODECOV_TOKEN
+                        }
+                    }
+                    catch (e) {
+                        // this might throw when MY_CODECOV_TOKEN doesn't exist
+                    }
+                    if (codecov_token == '') {
+                        withCredentials( [string( credentialsId: 'RIPPLED_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) {
+                            codecov_token = env.CODECOV_TOKEN
+                        }
+                    }
+                    env_vars.addAll(["CODECOV_TOKEN=${codecov_token}"])
+
                     withEnv(env_vars) {
                         myStage(bldlabel)
                         try {
@@ -261,10 +276,19 @@ try {
                             if (bldtype == 'docs') {
                                 publishHTML(
                                     allowMissing: true,
-                                        alwaysLinkToLastBuild: false,
+                                    alwaysLinkToLastBuild: true,
                                     keepAll: true,
                                     reportName:  'Doxygen',
-                                        reportDir:   'build/docs/html_doc',
+                                    reportDir:   "build/${bldlabel}/html_doc",
+                                    reportFiles: 'index.html')
+                            }
+                            if (isCoverage(cmake_extra)) {
+                                publishHTML(
+                                    allowMissing: true,
+                                    alwaysLinkToLastBuild: false,
+                                    keepAll: true,
+                                    reportName:  'Coverage',
+                                    reportDir:   "build/${bldlabel}/coverage",
                                     reportFiles: 'index.html')
                             }
                             def envs = ''
@@ -281,7 +305,6 @@ try {
                             }
                         } //try-catch-finally
                     } //withEnv
-                    } //withCredentials
                 } //node
             } //builds item
         } //for variants

bin/ci/ubuntu/build-and-test.sh

@@ -5,7 +5,7 @@
 # debugging.
 set -ex
 __dirname=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-echo "using CC: $CC"
+echo "using CC: ${CC}"
 "${CC}" --version
 export CC
 COMPNAME=$(basename $CC)
@@ -15,14 +15,14 @@ if [[ $CXX ]]; then
   export CXX
 fi
 : ${BUILD_TYPE:=Debug}
-echo "BUILD TYPE: $BUILD_TYPE"
+echo "BUILD TYPE: ${BUILD_TYPE}"
 
 : ${TARGET:=install}
-echo "BUILD TARGET: $TARGET"
+echo "BUILD TARGET: ${TARGET}"
 
 # Ensure APP defaults to rippled if it's not set.
 : ${APP:=rippled}
-echo "using APP: $APP"
+echo "using APP: ${APP}"
 
 JOBS=${NUM_PROCESSORS:-2}
 if [[ ${TRAVIS:-false} != "true" ]]; then
@@ -37,6 +37,12 @@ else
   time=
 fi
 
+if [[ -z "${MAX_TIME:-}" ]] ; then
+  timeout_cmd=""
+else
+  timeout_cmd="timeout ${MAX_TIME}"
+fi
+
 echo "cmake building ${APP}"
 : ${CMAKE_EXTRA_ARGS:=""}
 if [[ ${NINJA_BUILD:-} == true ]]; then
@@ -44,9 +50,10 @@ if [[ ${NINJA_BUILD:-} == true ]]; then
 fi
 
 coverage=false
-if [[ "${CMAKE_EXTRA_ARGS}" =~ -Dcoverage=((on)|(ON)) ]] ; then
+if [[ "${TARGET}" == "coverage_report" ]] ; then
     echo "coverage option detected."
     coverage=true
+    export PATH=$PATH:${LCOV_ROOT}/usr/bin
 fi
 
 #
@@ -77,9 +84,10 @@ fi
 mkdir -p "build/${BUILD_DIR}"
 pushd "build/${BUILD_DIR}"
 # generate
-$time cmake ../.. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${CMAKE_EXTRA_ARGS}
+${time} cmake ../.. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${CMAKE_EXTRA_ARGS}
 # build
-time DESTDIR=$(pwd)/_INSTALLED_ cmake --build . --target ${TARGET} -- $BUILDARGS
+export DESTDIR=$(pwd)/_INSTALLED_
+time ${timeout_cmd} cmake --build . --target ${TARGET} -- $BUILDARGS
 if [[ ${TARGET} == "docs" ]]; then
   ## mimic the standard test output for docs build
   ## to make controlling processes like jenkins happy
@@ -92,10 +100,10 @@ if [[ ${TARGET} == "docs" ]]; then
 fi
 popd
 export APP_PATH="$PWD/build/${BUILD_DIR}/${APP}"
-echo "using APP_PATH: $APP_PATH"
+echo "using APP_PATH: ${APP_PATH}"
 
 # See what we've actually built
-ldd $APP_PATH
+ldd ${APP_PATH}
 
 function join_by { local IFS="$1"; shift; echo "$*"; }
 
@@ -126,24 +134,15 @@ if [[ ${APP} == "rippled" ]]; then
   else
     APP_ARGS+=" --unittest --quiet --unittest-log"
   fi
-  # Only report on src/ripple files
-  export LCOV_FILES="*/src/ripple/*"
-  # Nothing to explicitly exclude
-  export LCOV_EXCLUDE_FILES="LCOV_NO_EXCLUDE"
   if [[ ${coverage} == false && ${PARALLEL_TESTS:-} == true ]]; then
     APP_ARGS+=" --unittest-jobs ${JOBS}"
   fi
-else
-  : ${LCOV_FILES:="*/src/*"}
-  # Don't exclude anything
-  : ${LCOV_EXCLUDE_FILES:="LCOV_NO_EXCLUDE"}
 fi
 
-if [[ $coverage == true ]]; then
+if [[ ${coverage} == true ]]; then
-  export PATH=$PATH:$LCOV_ROOT/usr/bin
+  # Push the results (lcov.info) to codecov
-
+  codecov -X gcov # don't even try and look for .gcov files ;)
-  # Create baseline coverage data file
+  find . -name "*.gcda" | xargs rm -f
-  lcov --no-external -c -i -d . -o baseline.info | grep -v "ignoring data for external file"
 fi
 
 if [[ ${SKIP_TESTS:-} == true ]]; then
@@ -151,45 +150,20 @@ if [[ ${SKIP_TESTS:-} == true ]]; then
   exit
 fi
 
-if [[ "${MAX_TIME:-}" == "" ]] ; then
+if [[ ${DEBUGGER:-true} == "true" && -v GDB_ROOT && -x ${GDB_ROOT}/bin/gdb ]]; then
-  tcmd=""
+  ${GDB_ROOT}/bin/gdb -v
-else
-  tcmd="timeout ${MAX_TIME}"
-fi
-
-if [[ ${DEBUGGER:-true} == "true" && -v GDB_ROOT && -x $GDB_ROOT/bin/gdb ]]; then
-  $GDB_ROOT/bin/gdb -v
   # Execute unit tests under gdb, printing a call stack
   # if we get a crash.
   export APP_ARGS
-  $tcmd $GDB_ROOT/bin/gdb -return-child-result -quiet -batch \
+  ${timeout_cmd} ${GDB_ROOT}/bin/gdb -return-child-result -quiet -batch \
                     -ex "set env MALLOC_CHECK_=3" \
                     -ex "set print thread-events off" \
                     -ex run \
                     -ex "thread apply all backtrace full" \
                     -ex "quit" \
-                    --args $APP_PATH $APP_ARGS
+                    --args ${APP_PATH} ${APP_ARGS}
 else
-  $tcmd $APP_PATH $APP_ARGS
+  ${timeout_cmd} ${APP_PATH} ${APP_ARGS}
-fi
-
-if [[ $coverage == true ]]; then
-  # Create test coverage data file
-  lcov --no-external -c -d . -o tests.info | grep -v "ignoring data for external file"
-
-  # Combine baseline and test coverage data
-  lcov -a baseline.info -a tests.info -o lcov-all.info
-
-  # Included files
-  lcov -e "lcov-all.info" "${LCOV_FILES}" -o lcov.pre.info
-
-  # Excluded files
-  lcov --remove lcov.pre.info "${LCOV_EXCLUDE_FILES}" -o lcov.info
-
-  # Push the results (lcov.info) to codecov
-  codecov -X gcov # don't even try and look for .gcov files ;)
-
-  find . -name "*.gcda" | xargs rm -f
 fi