Improve CI and packaging:

* add travis build for min cmake supported * add travis build for validator keys (uses xrpl_core) * add travis build for ipv6 (mac only) * add cmake target for validator keys via FetchContent * use validator keys target in package build
2025-12-01 08:25:51 +00:00 · 2019-08-27 08:23:11 -07:00
parent 5834fbbc5d
commit 008fc5155a
14 changed files with 197 additions and 104 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -94,3 +94,4 @@ Builds/VisualStudio2015/*.sdf
 .vs/
 CMakeSettings.json
 compile_commands.json
+.clangd
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ services:

 env:
  global:
-    - DOCKER_IMAGE="mellery451/rippled-ci-builder:2019-08-19"
+    - DOCKER_IMAGE="mellery451/rippled-ci-builder:2019-08-26"
    - CMAKE_EXTRA_ARGS="-Dwerr=ON"
    - NINJA_BUILD=true
    # change this if we get more VM capacity
@@ -191,6 +191,21 @@ matrix:
      env:
        - MATRIX_EVAL="CC=clang-9 && CXX=clang++-9"
        - BUILD_TYPE=Debug
+    # verify build with min version of cmake
+    - compiler: gcc-8
+      name: min cmake version
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Debug
+        - CMAKE_EXE=/opt/local/cmake-3.9/bin/cmake
+        - SKIP_TESTS=true
+    # validator keys project as subproj of rippled
+    - compiler: gcc-8
+      name: validator-keys
+      env:
+        - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+        - BUILD_TYPE=Debug
+        - TARGET=validator-keys
    # macos
    - &macos
      os: osx
@@ -198,6 +213,7 @@ matrix:
      name: xcode10, debug
      env:
        - BLD_CONFIG=Debug
+        - TEST_EXTRA_ARGS=""
        - BOOST_ROOT=${CACHE_DIR}/boost_1_70_0
        - >-
          CMAKE_ADD="
@@ -220,11 +236,16 @@ matrix:
        - mkdir -p build.macos && cd build.macos
        - cmake -G Ninja ${CMAKE_EXTRA_ARGS} -DCMAKE_BUILD_TYPE=${BLD_CONFIG} ..
        - travis_wait ${MAX_TIME_MIN} cmake --build . --parallel --verbose
-        - ./rippled --unittest --quiet --unittest-log --unittest-jobs ${NUM_PROCESSORS}
+        - ./rippled --unittest --quiet --unittest-log --unittest-jobs ${NUM_PROCESSORS} ${TEST_EXTRA_ARGS}
    - <<: *macos
      name: xcode10, release
      before_script:
        - export BLD_CONFIG=Release
+        - export CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -Dassert=ON"
+    - <<: *macos
+      name: ipv6 (macos)
+      before_script:
+        - export TEST_EXTRA_ARGS="--unittest-ipv6"
    - <<: *macos
      osx_image: xcode9.4
      name: xcode9, debug
--- a/Builds/CMake/CMakeFuncs.cmake
+++ b/Builds/CMake/CMakeFuncs.cmake
@@ -259,7 +259,7 @@ function (git_hash hash_val)
  if (NOT GIT_FOUND)
    return ()
  endif ()
-  set (_hash "unknown")
+  set (_hash "")
  set (_format "%H")
  if (ARGC GREATER_EQUAL 2)
    string (TOLOWER ${ARGV1} _short)
@@ -278,3 +278,21 @@ function (git_hash hash_val)
  endif ()
  set (${hash_val} "${_hash}" PARENT_SCOPE)
 endfunction ()
+
+function (git_branch branch_val)
+  if (NOT GIT_FOUND)
+    return ()
+  endif ()
+  set (_branch "")
+  execute_process (COMMAND ${GIT_EXECUTABLE} "rev-parse" "--abbrev-ref" "HEAD"
+                   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+                   RESULT_VARIABLE _git_exit_code
+                   OUTPUT_VARIABLE _temp_branch
+                   OUTPUT_STRIP_TRAILING_WHITESPACE
+                   ERROR_QUIET)
+  if (_git_exit_code EQUAL 0)
+    set (_branch ${_temp_branch})
+  endif ()
+  set (${branch_val} "${_branch}" PARENT_SCOPE)
+endfunction ()
+
--- a/Builds/containers/centos-builder/Dockerfile
+++ b/Builds/containers/centos-builder/Dockerfile
@@ -15,8 +15,13 @@ RUN chmod +x /tmp/centos_setup.sh && \
    chmod +x /tmp/install_cmake.sh && \
    chmod +x /tmp/extras.sh
 RUN /tmp/centos_setup.sh
-RUN /tmp/install_cmake.sh
+
+RUN /tmp/install_cmake.sh 3.15.2 /opt/local/cmake-3.15
+RUN ln -s /opt/local/cmake-3.15 /opt/local/cmake
 ENV PATH="/opt/local/cmake/bin:$PATH"
+# also install min supported cmake for testing
+RUN if [ "${CI_USE}" = true ] ; then /tmp/install_cmake.sh 3.9.0 /opt/local/cmake-3.9; fi
+
 RUN source scl_source enable devtoolset-7 python27 && \
     /tmp/build_deps.sh
 ENV BOOST_ROOT="/opt/local/boost/_INSTALLED_"
--- a/Builds/containers/packaging/dpkg/build_dpkg.sh
+++ b/Builds/containers/packaging/dpkg/build_dpkg.sh
@@ -11,7 +11,7 @@ RIPPLED_DPKG_VERSION=$(echo "${RIPPLED_VERSION}" | sed 's!-!~!g')
 # version here (hardcoded to 1). Does it ever need to change?
 RIPPLED_DPKG_FULL_VERSION="${RIPPLED_DPKG_VERSION}-1"

-cd rippled
+cd /opt/rippled_bld/pkg/rippled
 if [[ -n $(git status --porcelain) ]]; then
    git status
    error "Unstaged changes in this repo - please commit first"
--- a/Builds/containers/packaging/dpkg/debian/rules
+++ b/Builds/containers/packaging/dpkg/debian/rules
@@ -25,20 +25,11 @@ override_dh_auto_configure:

 override_dh_auto_build:
 	cd bld && \
-	cmake --build . --parallel -- -v
+	cmake --build . --target rippled --target validator-keys --parallel -- -v

 override_dh_auto_install:
 	cd bld && DESTDIR=../debian/tmp cmake --build . --target install -- -v
-	rm -rf bld_vl
-	mkdir -p bld_vl
-	cd bld_vl && \
-	cmake ../../validator-keys-tool -G Ninja \
-		-DCMAKE_BUILD_TYPE=Release \
-		-DCMAKE_PREFIX_PATH=${PWD}/debian/tmp/opt/ripple/ \
-		-Dstatic=true \
-		-DCMAKE_VERBOSE_MAKEFILE=ON && \
-	cmake --build . --parallel -- -v
-	install -D bld_vl/validator-keys debian/tmp/opt/ripple/bin/validator-keys
+	install -D bld/validator-keys/validator-keys debian/tmp/opt/ripple/bin/validator-keys
 	install -D Builds/containers/shared/update-rippled.sh debian/tmp/opt/ripple/bin/update-rippled.sh
 	install -D Builds/containers/shared/update-rippled-cron debian/tmp/opt/ripple/etc/update-rippled-cron
 	install -D Builds/containers/shared/rippled-logrotate debian/tmp/etc/logrotate.d/rippled
--- a/Builds/containers/packaging/rpm/build_rpm.sh
+++ b/Builds/containers/packaging/rpm/build_rpm.sh
@@ -25,14 +25,14 @@ if [[ $RPM_PATCH ]]; then
    export RPM_PATCH
 fi

-cd rippled
+cd /opt/rippled_bld/pkg/rippled
 if [[ -n $(git status --porcelain) ]]; then
    git status
    error "Unstaged changes in this repo - please commit first"
 fi
 git archive --format tar.gz --prefix rippled/ -o ../rpmbuild/SOURCES/rippled.tar.gz HEAD
+# TODO include validator-keys sources
 cd ..
-tar -zc --exclude-vcs -f ./rpmbuild/SOURCES/validator-keys.tar.gz validator-keys-tool/

 source /opt/rh/devtoolset-8/enable

--- a/Builds/containers/packaging/rpm/rippled.spec
+++ b/Builds/containers/packaging/rpm/rippled.spec
@@ -11,7 +11,6 @@ Summary:        rippled daemon
 License:        MIT
 URL:            http://ripple.com/
 Source0:        rippled.tar.gz
-Source1:        validator-keys.tar.gz

 BuildRequires:  protobuf-static openssl-static cmake zlib-static ninja-build

@@ -27,26 +26,14 @@ Requires: openssl-static, zlib-static
 core library for development of standalone applications that sign transactions.

 %prep
-%setup -c -n rippled -a 1
+%setup -c -n rippled

 %build
 cd rippled
 mkdir -p bld.release
 cd bld.release
 cmake .. -G Ninja -DCMAKE_INSTALL_PREFIX=%{_prefix} -DCMAKE_BUILD_TYPE=Release -Dstatic=true -DCMAKE_VERBOSE_MAKEFILE=ON -Dlocal_protobuf=ON
-# build VK
-cd ../../validator-keys-tool
-mkdir -p bld.release
-cd bld.release
-# Install a copy of the rippled artifacts into a local VK build dir so that it
-# can use them to build against (VK needs xrpl_core lib to build). We install
-# into a local build dir instead of buildroot because we want VK to have
-# relative paths embedded in debug info, otherwise check-buildroot (rpmbuild)
-# will complain
-mkdir xrpl_dir
-DESTDIR="%{_builddir}/validator-keys-tool/bld.release/xrpl_dir"  cmake --build ../../rippled/bld.release --target install -- -v
-cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=%{_builddir}/validator-keys-tool/bld.release/xrpl_dir/opt/ripple -Dstatic=true -DCMAKE_VERBOSE_MAKEFILE=ON
-cmake --build . --parallel -- -v
+cmake --build . --parallel --target rippled --target validator-keys -- -v

 %pre
 test -e /etc/pki/tls || { mkdir -p /etc/pki; ln -s /usr/lib/ssl /etc/pki/tls; }
@@ -59,7 +46,7 @@ install -d ${RPM_BUILD_ROOT}/usr/local/bin
 ln -s %{_prefix}/etc/rippled.cfg ${RPM_BUILD_ROOT}/etc/opt/ripple/rippled.cfg
 ln -s %{_prefix}/etc/validators.txt ${RPM_BUILD_ROOT}/etc/opt/ripple/validators.txt
 ln -s %{_prefix}/bin/rippled ${RPM_BUILD_ROOT}/usr/local/bin/rippled
-install -D validator-keys-tool/bld.release/validator-keys ${RPM_BUILD_ROOT}%{_bindir}/validator-keys
+install -D rippled/bld.release/validator-keys/validator-keys ${RPM_BUILD_ROOT}%{_bindir}/validator-keys
 install -D ./rippled/Builds/containers/shared/rippled.service ${RPM_BUILD_ROOT}/usr/lib/systemd/system/rippled.service
 install -D ./rippled/Builds/containers/packaging/rpm/50-rippled.preset ${RPM_BUILD_ROOT}/usr/lib/systemd/system-preset/50-rippled.preset
 install -D ./rippled/Builds/containers/shared/update-rippled.sh ${RPM_BUILD_ROOT}%{_bindir}/update-rippled.sh
@@ -109,6 +96,9 @@ chown -R root:$GROUP_NAME %{_prefix}/etc/update-rippled-cron
 %{_prefix}/lib/cmake/ripple

 %changelog
+* Wed Aug 28 2019 Mike Ellery <mellery451@gmail.com>
+- Switch to subproject build for validator-keys
+
 * Wed May 15 2019 Mike Ellery <mellery451@gmail.com>
 - Make validator-keys use local rippled build for core lib

--- a/Builds/containers/shared/install_cmake.sh
+++ b/Builds/containers/shared/install_cmake.sh
@@ -1,16 +1,34 @@
 #!/usr/bin/env bash
-set -ex
+set -e

-cd /tmp
-CM_INSTALLER=cmake-3.15.2-Linux-x86_64.sh
-CM_VER_DIR=/opt/local/cmake-3.15
-wget https://cmake.org/files/v3.15/$CM_INSTALLER
-chmod a+x $CM_INSTALLER
-mkdir -p $CM_VER_DIR
-ln -s $CM_VER_DIR /opt/local/cmake
-./$CM_INSTALLER --prefix=$CM_VER_DIR --exclude-subdir
-rm -f /tmp/$CM_INSTALLER
+IFS=. read cm_maj cm_min cm_rel <<<"$1"
+: ${cm_rel:-0}
+CMAKE_ROOT=${2:-"${HOME}/cmake"}

-export PATH="/opt/local/cmake/bin:${PATH}"
+function cmake_version ()
+{
+    if [[ -d ${CMAKE_ROOT} ]] ; then
+        local perms=$(test $(uname) = "Linux" && echo "/111" || echo "+111")
+        local installed=$(find ${CMAKE_ROOT} -perm ${perms} -type f -name cmake)
+        if [[ "${installed}" != "" ]] ; then
+            echo "$(${installed} --version | head -1)"
+        fi
+    fi
+}
+
+installed=$(cmake_version)
+if [[ "${installed}" != "" && ${installed} =~ ${cm_maj}.${cm_min}.${cm_rel} ]] ; then
+    echo "cmake already installed: ${installed}"
+    exit
+fi
+
+pkgname="cmake-${cm_maj}.${cm_min}.${cm_rel}-$(uname)-x86_64.tar.gz"
+tmppkg="/tmp/cmake.tar.gz"
+wget --quiet https://cmake.org/files/v${cm_maj}.${cm_min}/${pkgname} -O ${tmppkg}
+mkdir -p ${CMAKE_ROOT}
+cd ${CMAKE_ROOT}
+tar --strip-components 1 -xf ${tmppkg}
+rm -f ${tmppkg}
+echo "installed: $(cmake_version)"


--- a/Builds/containers/shared/update_sources.sh
+++ b/Builds/containers/shared/update_sources.sh
@@ -8,13 +8,6 @@ function error {
 cd /opt/rippled_bld/pkg/rippled
 export RIPPLED_VERSION=$(egrep -i -o "\b(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9a-z\-]+(\.[0-9a-z\-]+)*)?(\+[0-9a-z\-]+(\.[0-9a-z\-]+)*)?\b" src/ripple/protocol/impl/BuildInfo.cpp)

-cd ..
-git clone https://github.com/ripple/validator-keys-tool.git
-cd validator-keys-tool
-git checkout origin/master
-git submodule update --init --recursive
-cd ..
-
 : ${PKG_OUTDIR:=/opt/rippled_bld/pkg/out}
 export PKG_OUTDIR
 if [ ! -d ${PKG_OUTDIR} ]; then
--- a/Builds/containers/ubuntu-builder/Dockerfile
+++ b/Builds/containers/ubuntu-builder/Dockerfile
@@ -14,8 +14,13 @@ RUN chmod +x /tmp/ubuntu_setup.sh && \
    chmod +x /tmp/install_boost.sh && \
    chmod +x /tmp/install_cmake.sh
 RUN /tmp/ubuntu_setup.sh
-RUN /tmp/install_cmake.sh
+
+RUN /tmp/install_cmake.sh 3.15.2 /opt/local/cmake-3.15
+RUN ln -s /opt/local/cmake-3.15 /opt/local/cmake
 ENV PATH="/opt/local/cmake/bin:$PATH"
+# also install min supported cmake for testing
+RUN if [ "${CI_USE}" = true ] ; then /tmp/install_cmake.sh 3.9.0 /opt/local/cmake-3.9; fi
+
 RUN /tmp/build_deps.sh
 ENV PLANTUML_JAR="/opt/plantuml/plantuml.jar"
 ENV BOOST_ROOT="/opt/local/boost/_INSTALLED_"
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2889,3 +2889,23 @@ else ()
  message (STATUS "doxygen executable not found -- skipping docs target")
 endif ()

+if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.11)
+  git_branch (current_branch)
+  # default to tracking VK develop branch unless we are on master/release
+  if (NOT (current_branch STREQUAL "master" OR current_branch STREQUAL "release"))
+    set (current_branch "develop")
+  endif ()
+  message (STATUS "tracking ValidatorKeys branch: ${current_branch}")
+
+  FetchContent_Declare (
+    validator_keys_src
+    GIT_REPOSITORY https://github.com/ripple/validator-keys-tool.git
+    GIT_TAG        "${current_branch}"
+  )
+  FetchContent_GetProperties (validator_keys_src)
+  if (NOT validator_keys_src_POPULATED)
+    message (STATUS "Pausing to download ValidatorKeys...")
+    FetchContent_Populate (validator_keys_src)
+  endif ()
+  add_subdirectory (${validator_keys_src_SOURCE_DIR} ${CMAKE_BINARY_DIR}/validator-keys)
+endif ()
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ The server software that powers the XRP Ledger is called `rippled` and is availa


 ## Source Code
-[![travis-ci.org: Build Status](https://travis-ci.com/ripple/rippled.svg?branch=develop)](https://travis-ci.org/ripple/rippled)
+[![travis-ci.com: Build Status](https://travis-ci.com/ripple/rippled.svg?branch=develop)](https://travis-ci.com/ripple/rippled)
 [![codecov.io: Code Coverage](https://codecov.io/gh/ripple/rippled/branch/develop/graph/badge.svg)](https://codecov.io/gh/ripple/rippled)

 ### Repository Contents
--- a/bin/ci/ubuntu/build-and-test.sh
+++ b/bin/ci/ubuntu/build-and-test.sh
@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -ex

+function version_ge() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"; }
+
 __dirname=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 echo "using CC: ${CC}"
 "${CC}" --version
@@ -18,15 +20,15 @@ echo "BUILD TYPE: ${BUILD_TYPE}"
 : ${TARGET:=install}
 echo "BUILD TARGET: ${TARGET}"

-# Ensure APP defaults to rippled if it's not set.
-: ${APP:=rippled}
-echo "using APP: ${APP}"
-
 JOBS=${NUM_PROCESSORS:-2}
 if [[ ${TRAVIS:-false} != "true" ]]; then
    JOBS=$((JOBS+1))
 fi

+if [[ ! -z "${CMAKE_EXE:-}" ]] ; then
+    export PATH="$(dirname ${CMAKE_EXE}):$PATH"
+fi
+
 if [ -x /usr/bin/time ] ; then
    : ${TIME:="Duration: %E"}
    export TIME
@@ -35,13 +37,7 @@ else
    time=
 fi

-if [[ -z "${MAX_TIME:-}" ]] ; then
-    timeout_cmd=""
-else
-    timeout_cmd="timeout ${MAX_TIME}"
-fi
-
-echo "cmake building ${APP}"
+echo "Building rippled"
 : ${CMAKE_EXTRA_ARGS:=""}
 if [[ ${NINJA_BUILD:-} == true ]]; then
    CMAKE_EXTRA_ARGS+=" -G Ninja"
@@ -53,19 +49,39 @@ if [[ "${TARGET}" == "coverage_report" ]] ; then
    coverage=true
 fi

+cmake --version
+CMAKE_VER=$(cmake --version | cut -d " " -f 3 | head -1)
+
 #
 # allow explicit setting of the name of the build
 # dir, otherwise default to the compiler.build_type
 #
 : "${BUILD_DIR:=${COMPNAME}.${BUILD_TYPE}}"
-BUILDARGS="--target ${TARGET} --parallel"
+BUILDARGS="--target ${TARGET}"
+BUILDTOOLARGS=""
+if version_ge $CMAKE_VER "3.12.0" ; then
+    BUILDARGS+=" --parallel"
+fi
+
 if [[ ${NINJA_BUILD:-} == false ]]; then
+    if version_ge $CMAKE_VER "3.12.0" ; then
        BUILDARGS+=" ${JOBS}"
+    else
+        BUILDTOOLARGS+=" -j ${JOBS}"
+    fi
 fi

 if [[ ${VERBOSE_BUILD:-} == true ]]; then
    CMAKE_EXTRA_ARGS+=" -DCMAKE_VERBOSE_MAKEFILE=ON"
+    if version_ge $CMAKE_VER "3.14.0" ; then
        BUILDARGS+=" --verbose"
+    else
+        if [[ ${NINJA_BUILD:-} == false ]]; then
+            BUILDTOOLARGS+=" verbose=1"
+        else
+            BUILDTOOLARGS+=" -v"
+        fi
+    fi
 fi

 if [[ ${USE_CCACHE:-} == true ]]; then
@@ -78,11 +94,14 @@ fi

 mkdir -p "build/${BUILD_DIR}"
 pushd "build/${BUILD_DIR}"
+
 # generate
 ${time} cmake ../.. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${CMAKE_EXTRA_ARGS}
 # build
 export DESTDIR=$(pwd)/_INSTALLED_
-time eval ${timeout_cmd} cmake --build . ${BUILDARGS}
+
+${time} eval cmake --build . ${BUILDARGS} -- ${BUILDTOOLARGS}
+
 if [[ ${TARGET} == "docs" ]]; then
    ## mimic the standard test output for docs build
    ## to make controlling processes like jenkins happy
@@ -94,12 +113,22 @@ if [[ ${TARGET} == "docs" ]]; then
    exit
 fi
 popd
-export APP_PATH="$PWD/build/${BUILD_DIR}/${APP}"
+
+if [[ "${TARGET}" == "validator-keys" ]] ; then
+    export APP_PATH="$PWD/build/${BUILD_DIR}/validator-keys/validator-keys"
+else
+    export APP_PATH="$PWD/build/${BUILD_DIR}/rippled"
+fi
 echo "using APP_PATH: ${APP_PATH}"

 # See what we've actually built
 ldd ${APP_PATH}

+: ${APP_ARGS:=}
+
+if [[ "${TARGET}" == "validator-keys" ]] ; then
+    APP_ARGS="--unittest"
+else
    function join_by { local IFS="$1"; shift; echo "$*"; }

    # This is a list of manual tests
@@ -131,8 +160,6 @@ if [[ ${TRAVIS:-false} != "true" ]]; then
        )
    fi

-: ${APP_ARGS:=}
-if [[ ${APP} == "rippled" ]]; then
    if [[ ${MANUAL_TESTS:-} == true ]]; then
        APP_ARGS+=" --unittest=$(join_by , "${manual_tests[@]}")"
    else
@@ -141,6 +168,10 @@ if [[ ${APP} == "rippled" ]]; then
    if [[ ${coverage} == false && ${PARALLEL_TESTS:-} == true ]]; then
        APP_ARGS+=" --unittest-jobs ${JOBS}"
    fi
+
+    if [[ ${IPV6_TESTS:-} == true ]]; then
+        APP_ARGS+=" --unittest-ipv6"
+    fi
 fi

 if [[ ${coverage} == true && $CC =~ ^gcc ]]; then
@@ -171,14 +202,14 @@ set +e
 echo "Running tests for ${APP_PATH}"
 if [[ ${MANUAL_TESTS:-} == true && ${PARALLEL_TESTS:-} != true ]]; then
    for t in "${manual_tests[@]}" ; do
-        ${timeout_cmd} ${APP_PATH} --unittest=${t}
+        ${APP_PATH} --unittest=${t}
        TEST_STAT=$?
        if [[ $TEST_STAT -ne 0 ]] ; then
            break
        fi
    done
 else
-    ${timeout_cmd} ${APP_PATH} ${APP_ARGS}
+    ${APP_PATH} ${APP_ARGS}
    TEST_STAT=$?
 fi
 set -e