Streamline pkg and travis CI:

* use tagged containers for pkg build * update build images * continue to build container images in pipeline, but allow failure (non-block) * limit travis macos cache * add vs2019 windows to travis * remove xcode 9 travis build * remove clang5/6 from CI and update min version of Clang required in cmake * break windows CI build into stages to reduce timeouts * update datelib * add if condition to travis builds to allow commit message to limit builds by platform
2025-11-28 22:45:49 +00:00 · 2019-11-07 13:17:05 -08:00
parent cd9732b47a
commit eb016456a1
35 changed files with 862 additions and 699 deletions
--- a/Builds/containers/centos-builder/Dockerfile
+++ b/Builds/containers/centos-builder/Dockerfile
@@ -16,8 +16,8 @@ RUN chmod +x /tmp/centos_setup.sh && \
    chmod +x /tmp/extras.sh
 RUN /tmp/centos_setup.sh

-RUN /tmp/install_cmake.sh 3.15.2 /opt/local/cmake-3.15
-RUN ln -s /opt/local/cmake-3.15 /opt/local/cmake
+RUN /tmp/install_cmake.sh 3.16.1 /opt/local/cmake-3.16
+RUN ln -s /opt/local/cmake-3.16 /opt/local/cmake
 ENV PATH="/opt/local/cmake/bin:$PATH"
 # also install min supported cmake for testing
 RUN if [ "${CI_USE}" = true ] ; then /tmp/install_cmake.sh 3.9.0 /opt/local/cmake-3.9; fi
--- a/Builds/containers/centos-builder/centos_setup.sh
+++ b/Builds/containers/centos-builder/centos_setup.sh
@@ -7,7 +7,7 @@ yum -y upgrade
 yum -y update
 yum -y install epel-release centos-release-scl
 yum -y install \
-    wget curl time gcc-c++ time yum-utils autoconf pkgconfig \
+    wget curl time gcc-c++ time yum-utils autoconf automake pkgconfig libtool \
    libstdc++-static rpm-build gnupg which make cmake \
    devtoolset-7 devtoolset-7-gdb devtoolset-7-libasan-devel devtoolset-7-libtsan-devel devtoolset-7-libubsan-devel \
    devtoolset-8 devtoolset-8-gdb devtoolset-8-binutils devtoolset-8-libstdc++-devel \
--- a/Builds/containers/centos-builder/extras.sh
+++ b/Builds/containers/centos-builder/extras.sh
@@ -3,50 +3,31 @@ set -ex

 if [ "${CI_USE}" = true ] ; then
    cd /tmp
-    wget https://ftp.gnu.org/gnu/gdb/gdb-8.2.tar.xz
-    tar xf gdb-8.2.tar.xz
-    cd gdb-8.2
-    ./configure CFLAGS="-w -O2" CXXFLAGS="-std=gnu++11 -g -O2 -w" --prefix=/opt/local/gdb-8.2
+    wget https://ftp.gnu.org/gnu/gdb/gdb-8.3.1.tar.xz
+    tar xf gdb-8.3.1.tar.xz
+    cd gdb-8.3
+    ./configure CFLAGS="-w -O2" CXXFLAGS="-std=gnu++11 -g -O2 -w" --prefix=/opt/local/gdb-8.3
    make -j$(nproc)
    make install
-    ln -s /opt/local/gdb-8.2 /opt/local/gdb
+    ln -s /opt/local/gdb-8.3 /opt/local/gdb
    cd ..
-    rm -f gdb-8.2.tar.xz
-    rm -rf gdb-8.2
+    rm -f gdb-8.3.tar.xz
+    rm -rf gdb-8.3

    # clang from source
-    RELEASE=tags/RELEASE_701/final
-    INSTALL=/opt/llvm-7.0.1/
-    mkdir -p /tmp/clang-src
-    cd /tmp/clang-src
-    TOPDIR=`pwd`
-    svn co -q http://llvm.org/svn/llvm-project/llvm/${RELEASE} llvm
-    cd ${TOPDIR}/llvm/tools
-    svn co -q http://llvm.org/svn/llvm-project/cfe/${RELEASE} clang
-    cd ${TOPDIR}/llvm/tools/clang/tools
-    svn co -q http://llvm.org/svn/llvm-project/clang-tools-extra/${RELEASE} extra
-    cd ${TOPDIR}/llvm/tools
-    svn co -q http://llvm.org/svn/llvm-project/lld/${RELEASE} lld
-    cd ${TOPDIR}/llvm/tools
-    svn co -q http://llvm.org/svn/llvm-project/polly/${RELEASE} polly
-    cd ${TOPDIR}/llvm/projects
-    svn co -q http://llvm.org/svn/llvm-project/compiler-rt/${RELEASE} compiler-rt
-    cd ${TOPDIR}/llvm/projects
-    svn co -q http://llvm.org/svn/llvm-project/openmp/${RELEASE} openmp
-    cd ${TOPDIR}/llvm/projects
-    svn co -q http://llvm.org/svn/llvm-project/libcxx/${RELEASE} libcxx
-    svn co -q http://llvm.org/svn/llvm-project/libcxxabi/${RELEASE} libcxxabi
-    cd ${TOPDIR}/llvm/projects
-    ## config/build
-    cd ${TOPDIR}
+    cd /tmp
+    git clone https://github.com/llvm/llvm-project.git
+    cd llvm-project
+    git checkout llvmorg-9.0.0
+    INSTALL=/opt/llvm-9/
    mkdir mybuilddir && cd mybuilddir
+    # TODO figure out necessary options
    cmake ../llvm -G Ninja \
      -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_ENABLE_PROJECTS='clang;clang-tools-extra;libcxx;libcxxabi;lldb;compiler-rt;lld;polly' \
      -DCMAKE_INSTALL_PREFIX=${INSTALL} \
-      -DLLVM_LIBDIR_SUFFIX=64 \
-      -DLLVM_ENABLE_EH=ON \
-      -DLLVM_ENABLE_RTTI=ON
+      -DLLVM_LIBDIR_SUFFIX=64
    cmake --build . --parallel --target install
    cd /tmp
-    rm -rf clang-src
+    rm -rf llvm-project
 fi
--- a/Builds/containers/gitlab-ci/build_container.sh
+++ b/Builds/containers/gitlab-ci/build_container.sh
@@ -9,24 +9,15 @@ else
    echo "invalid package type"
    exit 1
 fi
-if docker pull "${ARTIFACTORY_HUB}/${container_name}:${CI_COMMIT_SHA}"; then
-    echo "${pkgtype} container for ${CI_COMMIT_SHA} already exists" \
-        "- skipping container build!"
-    exit 0
-else
-    echo "no existing ${pkgtype} container for this branch - searching history."
-    for CID_PREV in $(git log --pretty=%H -n30) ; do
-        if docker pull "${ARTIFACTORY_HUB}/${container_name}:${CID_PREV}"; then
-            echo "found container for previous commit ${CID_PREV}" \
-                "- using as cache."
-            docker tag \
-               "${ARTIFACTORY_HUB}/${container_name}:${CID_PREV}" \
-               "${container_name}:${CID_PREV}"
-            CMAKE_EXTRA="-D${pkgtype}_cache_from=${container_name}:${CID_PREV}"
-            break
-        fi
-    done
+
+if docker pull "${ARTIFACTORY_HUB}/${container_name}:latest_${CI_COMMIT_REF_SLUG}"; then
+    echo "found container for latest - using as cache."
+    docker tag \
+       "${ARTIFACTORY_HUB}/${container_name}:latest_${CI_COMMIT_REF_SLUG}" \
+       "${container_name}:latest_${CI_COMMIT_REF_SLUG}"
+    CMAKE_EXTRA="-D${pkgtype}_cache_from=${container_name}:latest_${CI_COMMIT_REF_SLUG}"
 fi
+
 cmake --version
 test -d build && rm -rf build
 mkdir -p build/container && cd build/container
@@ -34,8 +25,4 @@ eval time \
    cmake -Dpackages_only=ON -DCMAKE_VERBOSE_MAKEFILE=ON ${CMAKE_EXTRA} \
    -G Ninja ../..
 time cmake --build . --target "${pkgtype}_container" -- -v
-docker tag \
-    "${container_name}:${CI_COMMIT_SHA}" \
-    "${ARTIFACTORY_HUB}/${container_name}:${CI_COMMIT_SHA}"
-time docker push "${ARTIFACTORY_HUB}/${container_name}:${CI_COMMIT_SHA}"

--- a/Builds/containers/gitlab-ci/build_package.sh
+++ b/Builds/containers/gitlab-ci/build_package.sh
@@ -2,22 +2,27 @@
 set -ex
 pkgtype=$1
 if [ "${pkgtype}" = "rpm" ] ; then
-    container_name="${RPM_CONTAINER_NAME}"
+    container_name="${RPM_CONTAINER_FULLNAME}"
+    container_tag="${RPM_CONTAINER_TAG}"
 elif [ "${pkgtype}" = "dpkg" ] ; then
-    container_name="${DPKG_CONTAINER_NAME}"
+    container_name="${DPKG_CONTAINER_FULLNAME}"
+    container_tag="${DPKG_CONTAINER_TAG}"
 else
    echo "invalid package type"
    exit 1
 fi
-time docker pull "${ARTIFACTORY_HUB}/${container_name}:${CI_COMMIT_SHA}"
+time docker pull "${ARTIFACTORY_HUB}/${container_name}"
 docker tag \
-  "${ARTIFACTORY_HUB}/${container_name}:${CI_COMMIT_SHA}" \
-  "${container_name}:${CI_COMMIT_SHA}"
+  "${ARTIFACTORY_HUB}/${container_name}" \
+  "${container_name}"
 docker images
 test -d build && rm -rf build
 mkdir -p build/${pkgtype} && cd build/${pkgtype}
 time cmake \
-  -Dpackages_only=ON -Dhave_package_container=ON -DCMAKE_VERBOSE_MAKEFILE=ON \
+  -Dpackages_only=ON \
+  -Dcontainer_label="${container_tag}" \
+  -Dhave_package_container=ON \
+  -DCMAKE_VERBOSE_MAKEFILE=ON \
  -G Ninja ../..
 time cmake --build . --target ${pkgtype} -- -v

--- a/Builds/containers/gitlab-ci/pkgbuild.yml
+++ b/Builds/containers/gitlab-ci/pkgbuild.yml
@@ -8,8 +8,15 @@
 # NOTE: these are sensible defaults for Ripple pipelines. These
 # can be overridden by project or group variables as needed.
 variables:
+  # these containers are built manually using the rippled
+  # cmake build (container targets) and tagged/pushed so they 
+  # can be used here
+  RPM_CONTAINER_TAG: "2020-01-08"
  RPM_CONTAINER_NAME: "rippled-rpm-builder"
+  RPM_CONTAINER_FULLNAME: "${RPM_CONTAINER_NAME}:${RPM_CONTAINER_TAG}"
+  DPKG_CONTAINER_TAG: "2020-01-08"
  DPKG_CONTAINER_NAME: "rippled-dpkg-builder"
+  DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}"
  ARTIFACTORY_HOST: "artifactory.ops.ripple.com"
  ARTIFACTORY_HUB: "${ARTIFACTORY_HOST}:6555"
  GIT_SIGN_PUBKEYS_URL: "https://gitlab.ops.ripple.com/snippets/11/raw"
@@ -19,7 +26,6 @@ variables:
  #   IS_PRIMARY_REPO = "true"

 stages:
-  - build_containers
  - build_packages
  - sign_packages
  - smoketest
@@ -31,6 +37,7 @@ stages:
  - push_to_prod
  - verify_from_prod
  - get_final_hashes
+  - build_containers

 .dind_template: &dind_param
  before_script:
@@ -65,35 +72,6 @@ stages:
  script:
    - . ./Builds/containers/gitlab-ci/smoketest.sh repo

-#########################################################################
-##                                                                     ##
-##  stage: build_containers                                            ##
-##                                                                     ##
-##  build containers from docker definitions. These containers are     ##
-##  subsequently used to build the rpm and deb packages.               ##
-##                                                                     ##
-#########################################################################
-
-build_centos_container:
-  stage: build_containers
-  <<: *dind_param
-  cache:
-    key: containers
-    paths:
-      - .nih_c
-  script:
-    - . ./Builds/containers/gitlab-ci/build_container.sh rpm
-
-build_ubuntu_container:
-  stage: build_containers
-  <<: *dind_param
-  cache:
-    key: containers
-    paths:
-      - .nih_c
-  script:
-    - . ./Builds/containers/gitlab-ci/build_container.sh dpkg
-
 #########################################################################
 ##                                                                     ##
 ##  stage: build_packages                                              ##
@@ -104,31 +82,19 @@ build_ubuntu_container:

 rpm_build:
  stage: build_packages
-  dependencies:
-    - build_centos_container
  <<: *dind_param
  artifacts:
    paths:
      - build/rpm/packages/
-  cache:
-    key: rpm
-    paths:
-      - .nih_c/pkgbuild
  script:
    - . ./Builds/containers/gitlab-ci/build_package.sh rpm

 dpkg_build:
  stage: build_packages
-  dependencies:
-    - build_ubuntu_container
  <<: *dind_param
  artifacts:
    paths:
      - build/dpkg/packages/
-  cache:
-    key: dpkg
-    paths:
-      - .nih_c/pkgbuild
  script:
    - . ./Builds/containers/gitlab-ci/build_package.sh dpkg

@@ -146,7 +112,7 @@ rpm_sign:
    - rpm_build
  image:
    name: centos:7
-    # <<: *dind_param
+  <<: *only_primary
  before_script:
  - |
    # Make sure GnuPG is installed
@@ -175,7 +141,7 @@ dpkg_sign:
    - dpkg_build
  image:
    name: ubuntu:18.04
-    # <<: *dind_param
+  <<: *only_primary
  before_script:
  - |
    # make sure we have GnuPG
@@ -210,6 +176,7 @@ dpkg_sign:
 centos_7_smoketest:
  stage: smoketest
  dependencies:
+    - rpm_build
    - rpm_sign
  image:
    name: centos:7
@@ -218,6 +185,7 @@ centos_7_smoketest:
 fedora_29_smoketest:
  stage: smoketest
  dependencies:
+    - rpm_build
    - rpm_sign
  image:
    name: fedora:29
@@ -226,6 +194,7 @@ fedora_29_smoketest:
 fedora_28_smoketest:
  stage: smoketest
  dependencies:
+    - rpm_build
    - rpm_sign
  image:
    name: fedora:28
@@ -234,6 +203,7 @@ fedora_28_smoketest:
 fedora_27_smoketest:
  stage: smoketest
  dependencies:
+    - rpm_build
    - rpm_sign
  image:
    name: fedora:27
@@ -245,6 +215,7 @@ fedora_27_smoketest:
 ubuntu_19_smoketest:
  stage: smoketest
  dependencies:
+    - dpkg_build
    - dpkg_sign
  image:
    name: ubuntu:19.04
@@ -253,6 +224,7 @@ ubuntu_19_smoketest:
 ubuntu_18_smoketest:
  stage: smoketest
  dependencies:
+    - dpkg_build
    - dpkg_sign
  image:
    name: ubuntu:18.04
@@ -261,6 +233,7 @@ ubuntu_18_smoketest:
 ubuntu_16_smoketest:
  stage: smoketest
  dependencies:
+    - dpkg_build
    - dpkg_sign
  image:
    name: ubuntu:16.04
@@ -269,6 +242,7 @@ ubuntu_16_smoketest:
 debian_9_smoketest:
  stage: smoketest
  dependencies:
+    - dpkg_build
    - dpkg_sign
  image:
    name: debian:9
@@ -621,3 +595,28 @@ get_prod_hashes:
  script:
    - . ./Builds/containers/gitlab-ci/push_to_artifactory.sh "GET" ".checksums"

+#########################################################################
+##                                                                     ##
+##  stage: build_containers                                            ##
+##                                                                     ##
+##  build containers from docker definitions. These containers are NOT ##
+##  used for the package build. This step is only used to ensure that  ##
+##  the package build targets and files are still working properly.    ##
+##                                                                     ##
+#########################################################################
+
+build_centos_container:
+  stage: build_containers
+  <<: *dind_param
+  script:
+    - . ./Builds/containers/gitlab-ci/build_container.sh rpm
+  allow_failure: true
+
+build_ubuntu_container:
+  stage: build_containers
+  <<: *dind_param
+  script:
+    - . ./Builds/containers/gitlab-ci/build_container.sh dpkg
+  allow_failure: true
+
+
--- a/Builds/containers/gitlab-ci/tag_docker_image.sh
+++ b/Builds/containers/gitlab-ci/tag_docker_image.sh
@@ -4,17 +4,17 @@ docker login -u rippled \
    -p ${ARTIFACTORY_DEPLOY_KEY_RIPPLED} "${ARTIFACTORY_HUB}"
 # this gives us rippled_version :
 source build/rpm/packages/build_vars
-docker pull "${ARTIFACTORY_HUB}/${RPM_CONTAINER_NAME}:${CI_COMMIT_SHA}"
-docker pull "${ARTIFACTORY_HUB}/${DPKG_CONTAINER_NAME}:${CI_COMMIT_SHA}"
+docker pull "${ARTIFACTORY_HUB}/${RPM_CONTAINER_FULLNAME}"
+docker pull "${ARTIFACTORY_HUB}/${DPKG_CONTAINER_FULLNAME}"
 # tag/push two labels...one using the current rippled version and one just using "latest"
 for label in ${rippled_version} latest ; do
    docker tag \
-        "${ARTIFACTORY_HUB}/${RPM_CONTAINER_NAME}:${CI_COMMIT_SHA}" \
+        "${ARTIFACTORY_HUB}/${RPM_CONTAINER_FULLNAME}" \
        "${ARTIFACTORY_HUB}/${RPM_CONTAINER_NAME}:${label}_${CI_COMMIT_REF_SLUG}"
    docker tag \
-        "${ARTIFACTORY_HUB}/${DPKG_CONTAINER_NAME}:${CI_COMMIT_SHA}" \
+        "${ARTIFACTORY_HUB}/${DPKG_CONTAINER_FULLNAME}" \
        "${ARTIFACTORY_HUB}/${DPKG_CONTAINER_NAME}:${label}_${CI_COMMIT_REF_SLUG}"
-    docker push "${ARTIFACTORY_HUB}/${RPM_CONTAINER_NAME}"
-    docker push "${ARTIFACTORY_HUB}/${DPKG_CONTAINER_NAME}"
+    docker push "${ARTIFACTORY_HUB}/${RPM_CONTAINER_FULLNAME}"
+    docker push "${ARTIFACTORY_HUB}/${DPKG_CONTAINER_FULLNAME}"
 done

--- a/Builds/containers/packaging/dpkg/debian/rules
+++ b/Builds/containers/packaging/dpkg/debian/rules
@@ -19,7 +19,7 @@ override_dh_auto_configure:
 	cmake .. -G Ninja \
 		-DCMAKE_INSTALL_PREFIX=/opt/ripple \
 		-DCMAKE_BUILD_TYPE=Release \
-		-Dstatic=true \
+		-Dstatic=ON \
 		-Dvalidator_keys=ON \
 		-DCMAKE_VERBOSE_MAKEFILE=ON

@@ -33,5 +33,8 @@ override_dh_auto_install:
 	install -D Builds/containers/shared/update-rippled.sh debian/tmp/opt/ripple/bin/update-rippled.sh
 	install -D Builds/containers/shared/update-rippled-cron debian/tmp/opt/ripple/etc/update-rippled-cron
 	install -D Builds/containers/shared/rippled-logrotate debian/tmp/etc/logrotate.d/rippled
+	rm -rf debian/tmp/opt/ripple/lib64/cmake/date
 	rm -rf bld
 	rm -rf bld_vl
+
+
--- a/Builds/containers/packaging/rpm/rippled.spec
+++ b/Builds/containers/packaging/rpm/rippled.spec
@@ -41,6 +41,7 @@ test -e /etc/pki/tls || { mkdir -p /etc/pki; ln -s /usr/lib/ssl /etc/pki/tls; }
 %install
 rm -rf $RPM_BUILD_ROOT
 DESTDIR=$RPM_BUILD_ROOT cmake --build rippled/bld.release --target install -- -v
+rm -rf ${RPM_BUILD_ROOT}/%{_prefix}/lib64/cmake/date
 install -d ${RPM_BUILD_ROOT}/etc/opt/ripple
 install -d ${RPM_BUILD_ROOT}/usr/local/bin
 ln -s %{_prefix}/etc/rippled.cfg ${RPM_BUILD_ROOT}/etc/opt/ripple/rippled.cfg
--- a/Builds/containers/shared/build_deps.sh
+++ b/Builds/containers/shared/build_deps.sh
@@ -22,7 +22,7 @@ build_boost "1.70.0" true
 # installed in opt, so won't be used
 # unless specified by OPENSSL_ROOT_DIR
 cd /tmp
-OPENSSL_VER=1.1.1
+OPENSSL_VER=1.1.1d
 wget https://www.openssl.org/source/openssl-${OPENSSL_VER}.tar.gz
 tar xf openssl-${OPENSSL_VER}.tar.gz
 cd openssl-${OPENSSL_VER}
@@ -36,6 +36,18 @@ rm -f openssl-${OPENSSL_VER}.tar.gz
 rm -rf openssl-${OPENSSL_VER}
 LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/opt/local/openssl/lib /opt/local/openssl/bin/openssl version -a

+cd /tmp
+wget https://libarchive.org/downloads/libarchive-3.4.1.tar.gz
+tar xzf libarchive-3.4.1.tar.gz
+cd libarchive-3.4.1
+mkdir _bld && cd _bld
+cmake -DCMAKE_BUILD_TYPE=Release ..
+make -j$(nproc)
+make install
+cd ../..
+rm -f libarchive-3.4.1.tar.gz
+rm -rf libarchive-3.4.1
+
 cd /tmp
 wget https://github.com/protocolbuffers/protobuf/releases/download/v3.10.1/protobuf-all-3.10.1.tar.gz
 tar xf protobuf-all-3.10.1.tar.gz
@@ -50,13 +62,24 @@ rm -f protobuf-all-3.10.1.tar.gz
 rm -rf protobuf-3.10.1

 cd /tmp
-wget https://github.com/c-ares/c-ares/releases/download/cares-1_15_0/c-ares-1.15.0.tar.gz
+wget https://c-ares.haxx.se/download/c-ares-1.15.0.tar.gz
 tar xf c-ares-1.15.0.tar.gz
 cd c-ares-1.15.0
-./configure
+mkdir _bld && cd _bld
+cmake \
+  -DHAVE_LIBNSL=OFF \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCARES_STATIC=ON \
+  -DCARES_SHARED=OFF \
+  -DCARES_INSTALL=ON \
+  -DCARES_STATIC_PIC=ON \
+  -DCARES_BUILD_TOOLS=OFF \
+  -DCARES_BUILD_TESTS=OFF \
+  -DCARES_BUILD_CONTAINER_TESTS=OFF \
+  ..
 make -j$(nproc)
 make install
-cd ..
+cd ../..
 rm -f c-ares-1.15.0.tar.gz
 rm -rf c-ares-1.15.0

@@ -64,13 +87,26 @@ cd /tmp
 wget https://github.com/grpc/grpc/archive/v1.25.0.tar.gz
 tar xf v1.25.0.tar.gz
 cd grpc-1.25.0
+mkdir _bld && cd _bld
+cmake \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DBUILD_SHARED_LIBS=OFF \
+  -DgRPC_ZLIB_PROVIDER=package \
+  -DgRPC_CARES_PROVIDER=package \
+  -DgRPC_SSL_PROVIDER=package \
+  -DgRPC_PROTOBUF_PROVIDER=package \
+  -DProtobuf_USE_STATIC_LIBS=ON \
+  ..
 make -j$(nproc)
 make install
-cd ..
+cd ../..
 rm -f xf v1.25.0.tar.gz
 rm -rf grpc-1.25.0

 if [ "${CI_USE}" = true ] ; then
+
+    build_boost "1.71.0" false
+
    cd /tmp
    wget https://github.com/doxygen/doxygen/archive/Release_1_8_16.tar.gz
    tar xf Release_1_8_16.tar.gz
@@ -96,15 +132,15 @@ if [ "${CI_USE}" = true ] ; then
    rm -r lcov-1.14 lcov-1.14.tar.gz

    cd /tmp
-    wget https://github.com/ccache/ccache/releases/download/v3.7.3/ccache-3.7.3.tar.gz
-    tar xf ccache-3.7.3.tar.gz
-    cd ccache-3.7.3
+    wget https://github.com/ccache/ccache/releases/download/v3.7.6/ccache-3.7.6.tar.gz
+    tar xf ccache-3.7.6.tar.gz
+    cd ccache-3.7.6
    ./configure --prefix=/usr/local
    make
    make install
    cd ..
-    rm -f ccache-3.7.3.tar.gz
-    rm -rf ccache-3.7.3
+    rm -f ccache-3.7.6.tar.gz
+    rm -rf ccache-3.7.6

    pip install requests
    pip install https://github.com/codecov/codecov-python/archive/master.zip
--- a/Builds/containers/shared/install_boost.sh
+++ b/Builds/containers/shared/install_boost.sh
@@ -9,6 +9,7 @@
 set -exu

 odir=$(pwd)
+: ${BOOST_TOOLSET:=msvc-14.1}

 if [[ -d "$BOOST_ROOT/lib" || -d "${BOOST_ROOT}/stage/lib" ]] ; then
    echo "Using cached boost at $BOOST_ROOT"
@@ -21,7 +22,8 @@ ext="${fn##*.}"
 wget --quiet $BOOST_URL -O /tmp/boost.tar.${ext}
 cd $(dirname $BOOST_ROOT)
 rm -fr ${BOOST_ROOT}
-tar xf /tmp/boost.tar.${ext}
+mkdir ${BOOST_ROOT}
+tar xf /tmp/boost.tar.${ext} -C ${BOOST_ROOT} --strip-components 1
 cd $BOOST_ROOT

 BLDARGS=()
@@ -60,7 +62,7 @@ if [[ -z ${COMSPEC:-} ]]; then
 else
    BLDARGS+=(runtime-link="static,shared")
    BLDARGS+=(--layout=versioned)
-    BLDARGS+=(--toolset="msvc-14.1")
+    BLDARGS+=(--toolset="${BOOST_TOOLSET}")
    BLDARGS+=(address-model=64)
    BLDARGS+=(architecture=x86)
    BLDARGS+=(link=static)
--- a/Builds/containers/ubuntu-builder/Dockerfile
+++ b/Builds/containers/ubuntu-builder/Dockerfile
@@ -15,8 +15,8 @@ RUN chmod +x /tmp/ubuntu_setup.sh && \
    chmod +x /tmp/install_cmake.sh
 RUN /tmp/ubuntu_setup.sh

-RUN /tmp/install_cmake.sh 3.15.2 /opt/local/cmake-3.15
-RUN ln -s /opt/local/cmake-3.15 /opt/local/cmake
+RUN /tmp/install_cmake.sh 3.16.1 /opt/local/cmake-3.16
+RUN ln -s /opt/local/cmake-3.16 /opt/local/cmake
 ENV PATH="/opt/local/cmake/bin:$PATH"
 # also install min supported cmake for testing
 RUN if [ "${CI_USE}" = true ] ; then /tmp/install_cmake.sh 3.9.0 /opt/local/cmake-3.9; fi
--- a/Builds/containers/ubuntu-builder/ubuntu_setup.sh
+++ b/Builds/containers/ubuntu-builder/ubuntu_setup.sh
@@ -30,7 +30,7 @@ apt-get -y clean
 apt-get -y update

 apt-get -y --fix-missing install \
-    make cmake ninja-build autoconf libtool pkg-config \
+    make cmake ninja-build autoconf automake libtool pkg-config libtool \
    openssl libssl-dev \
    liblzma-dev libbz2-dev zlib1g-dev \
    libjemalloc-dev \
@@ -158,36 +158,6 @@ update-alternatives --install \
 update-alternatives --auto clang

 if [ "${CI_USE}" = true ] ; then
-    apt-get -y install \
-        clang-5.0 libclang-common-5.0-dev libclang-5.0-dev libllvm5.0 llvm-5.0 \
-        llvm-5.0-dev llvm-5.0-runtime clang-format-5.0 python-clang-5.0 \
-        lld-5.0 libfuzzer-5.0-dev
-    update-alternatives --install \
-        /usr/bin/clang clang /usr/bin/clang-5.0 10 \
-        --slave /usr/bin/clang++ clang++ /usr/bin/clang++-5.0 \
-        --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-5.0 \
-        --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-5.0 \
-        --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-5.0 \
-        --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-5.0 \
-        --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-5.0 \
-        --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-5.0 \
-        --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-5.0
-
-    apt-get -y install \
-        clang-6.0 libclang-common-6.0-dev libclang-6.0-dev libllvm6.0 llvm-6.0 \
-        llvm-6.0-dev llvm-6.0-runtime clang-format-6.0 python-clang-6.0 \
-        lld-6.0 libfuzzer-6.0-dev
-    update-alternatives --install \
-        /usr/bin/clang clang /usr/bin/clang-6.0 12 \
-        --slave /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 \
-        --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-6.0 \
-        --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-6.0 \
-        --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-6.0 \
-        --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-6.0 \
-        --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-6.0 \
-        --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-6.0 \
-        --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-6.0
-
    apt-get -y install \
        clang-9 libclang-common-9-dev libclang-9-dev libllvm9 llvm-9 \
        llvm-9-dev llvm-9-runtime clang-format-9 python-clang-9 \