Travis CI improvements:

FIXES: #2527 * define custom docker image for travis-linux builds based on package build image * add macos builds * add windows builds (currently allowed to fail) * improve build and shell scripts as required for the CI envs * add asio timer latency workaround * omit several manual tests from TravisCI which cause memory exhaustion
2025-12-05 16:57:53 +00:00 · 2019-05-20 10:53:24 -07:00
parent 87e9ee5ce9
commit 13a4fefe34
18 changed files with 1300 additions and 591 deletions
--- a/Builds/containers/centos-builder/Dockerfile
+++ b/Builds/containers/centos-builder/Dockerfile
@@ -8,20 +8,22 @@ COPY centos-builder/centos_setup.sh /tmp/
 COPY shared/build_deps.sh /tmp/
 COPY shared/install_cmake.sh /tmp/
 COPY centos-builder/extras.sh /tmp/
+COPY shared/install_boost.sh /tmp/
 RUN chmod +x /tmp/centos_setup.sh && \
    chmod +x /tmp/build_deps.sh && \
+    chmod +x /tmp/install_boost.sh && \
    chmod +x /tmp/install_cmake.sh && \
    chmod +x /tmp/extras.sh
 RUN /tmp/centos_setup.sh
 RUN /tmp/install_cmake.sh
 ENV PATH="/opt/local/cmake/bin:$PATH"
-RUN source scl_source enable devtoolset-6 python27 && \
+RUN source scl_source enable devtoolset-7 python27 && \
     /tmp/build_deps.sh
+ENV BOOST_ROOT="/opt/local/boost/_INSTALLED_"
 ENV PLANTUML_JAR="/opt/plantuml/plantuml.jar"
-ENV BOOST_ROOT="/opt/local/boost"
 ENV OPENSSL_ROOT="/opt/local/openssl"
 ENV GDB_ROOT="/opt/local/gdb"
-RUN source scl_source enable devtoolset-6 python27 && \
+RUN source scl_source enable devtoolset-7 python27 && \
     /tmp/extras.sh

 # prep files for package building
--- a/Builds/containers/centos-builder/centos_setup.sh
+++ b/Builds/containers/centos-builder/centos_setup.sh
@@ -9,14 +9,8 @@ yum -y install epel-release centos-release-scl
 yum -y install \
    wget curl time gcc-c++ time yum-utils \
    libstdc++-static rpm-build gnupg which make cmake \
-    devtoolset-4 devtoolset-4-gdb devtoolset-4-libasan-devel devtoolset-4-libtsan-devel devtoolset-4-libubsan-devel \
-    devtoolset-6 devtoolset-6-gdb devtoolset-6-libasan-devel devtoolset-6-libtsan-devel devtoolset-6-libubsan-devel \
    devtoolset-7 devtoolset-7-gdb devtoolset-7-libasan-devel devtoolset-7-libtsan-devel devtoolset-7-libubsan-devel \
-    llvm-toolset-7 llvm-toolset-7-runtime llvm-toolset-7-build llvm-toolset-7-clang \
-    llvm-toolset-7-clang-analyzer llvm-toolset-7-clang-devel llvm-toolset-7-clang-libs \
-    llvm-toolset-7-clang-tools-extra llvm-toolset-7-compiler-rt llvm-toolset-7-lldb \
-    llvm-toolset-7-lldb-devel llvm-toolset-7-python-lldb \
-    flex flex-devel bison bison-devel \
+    flex flex-devel bison bison-devel parallel \
    ncurses ncurses-devel ncurses-libs graphviz graphviz-devel \
    lzip p7zip bzip2 bzip2-devel lzma-sdk lzma-sdk-devel xz-devel \
    zlib zlib-devel zlib-static texinfo openssl openssl-static \
@@ -28,8 +22,17 @@ yum -y install \
    ninja-build git svn \
    protobuf protobuf-static protobuf-c-devel \
    protobuf-compiler protobuf-devel \
-    swig ccache perl-Digest-MD5 python2-pip
+    swig perl-Digest-MD5 python2-pip

-# TODO need permanent link
-yum -y install ftp://ftp.pbone.net/mirror/archive.fedoraproject.org/fedora-secondary/updates/26/i386/Packages/p/python2-six-1.10.0-9.fc26.noarch.rpm
+if [ "${CI_USE}" = true ] ; then
+    # TODO need permanent link
+    yum -y install ftp://ftp.pbone.net/mirror/archive.fedoraproject.org/fedora-secondary/updates/26/i386/Packages/p/python2-six-1.10.0-9.fc26.noarch.rpm

+    yum -y install \
+        devtoolset-4 devtoolset-4-gdb devtoolset-4-libasan-devel devtoolset-4-libtsan-devel devtoolset-4-libubsan-devel \
+        devtoolset-6 devtoolset-6-gdb devtoolset-6-libasan-devel devtoolset-6-libtsan-devel devtoolset-6-libubsan-devel \
+        llvm-toolset-7 llvm-toolset-7-runtime llvm-toolset-7-build llvm-toolset-7-clang \
+        llvm-toolset-7-clang-analyzer llvm-toolset-7-clang-devel llvm-toolset-7-clang-libs \
+        llvm-toolset-7-clang-tools-extra llvm-toolset-7-compiler-rt llvm-toolset-7-lldb \
+        llvm-toolset-7-lldb-devel llvm-toolset-7-python-lldb 
+fi
--- a/Builds/containers/shared/build_deps.sh
+++ b/Builds/containers/shared/build_deps.sh
@@ -6,20 +6,15 @@ function build_boost()
    local boost_ver=$1
    local do_link=$2
    local boost_path=$(echo "${boost_ver}" | sed -e 's!\.!_!g')
-    cd /tmp
-    wget https://dl.bintray.com/boostorg/release/${boost_ver}/source/boost_${boost_path}.tar.bz2
    mkdir -p /opt/local
    cd /opt/local
-    tar xf /tmp/boost_${boost_path}.tar.bz2
+    BOOST_ROOT=/opt/local/boost_${boost_path}
+    BOOST_URL="https://dl.bintray.com/boostorg/release/${boost_ver}/source/boost_${boost_path}.tar.bz2"
+    BOOST_BUILD_ALL=true
+    . /tmp/install_boost.sh
    if [ "$do_link" = true ] ; then
        ln -s ./boost_${boost_path} boost
    fi
-    cd boost_${boost_path}
-    ./bootstrap.sh
-    ./b2 -j$(nproc)
-    ./b2 stage
-    cd ..
-    rm -f /tmp/boost_${boost_path}.tar.bz2
 }

 build_boost "1.70.0" true
@@ -39,7 +34,7 @@ make install
 cd ..
 rm -f openssl-${OPENSSL_VER}.tar.gz
 rm -rf openssl-${OPENSSL_VER}
-LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/local/openssl/lib /opt/local/openssl/bin/openssl version -a
+LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/opt/local/openssl/lib /opt/local/openssl/bin/openssl version -a

 if [ "${CI_USE}" = true ] ; then
    cd /tmp
@@ -59,20 +54,25 @@ if [ "${CI_USE}" = true ] ; then
    wget -O /opt/plantuml/plantuml.jar https://downloads.sourceforge.net/project/plantuml/plantuml.jar

    cd /tmp
-    wget https://github.com/linux-test-project/lcov/releases/download/v1.13/lcov-1.13.tar.gz
-    tar xfz lcov-1.13.tar.gz
-    cd lcov-1.13
+    wget https://github.com/linux-test-project/lcov/releases/download/v1.14/lcov-1.14.tar.gz
+    tar xfz lcov-1.14.tar.gz
+    cd lcov-1.14
    make install PREFIX=/usr/local
    cd ..
-    rm -r lcov-1.13 lcov-1.13.tar.gz
+    rm -r lcov-1.14 lcov-1.14.tar.gz
+
+    cd /tmp
+    wget https://github.com/ccache/ccache/releases/download/v3.7.1/ccache-3.7.1.tar.gz
+    tar xf ccache-3.7.1.tar.gz
+    cd ccache-3.7.1
+    ./configure --prefix=/usr/local
+    make
+    make install
+    cd ..
+    rm -f ccache-3.7.1.tar.gz
+    rm -rf ccache-3.7.1

    pip install requests
    pip install https://github.com/codecov/codecov-python/archive/master.zip
-
-    set +e
-    mkdir -p /opt/local/nih_cache
-    mkdir -p /opt/jenkins
-    set -e
 fi

-
--- a/Builds/containers/shared/install_boost.sh
+++ b/Builds/containers/shared/install_boost.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Assumptions:
+# 1) BOOST_ROOT and BOOST_URL are already defined,
+# and contain valid values.
+# 2) The last namepart of BOOST_ROOT matches the
+# folder name internal to boost's .tar.gz
+# When testing you can force a boost build by clearing travis caches:
+# https://travis-ci.org/ripple/rippled/caches
+set -exu
+
+odir=$(pwd)
+
+if [[ -d "$BOOST_ROOT/lib" || -d "${BOOST_ROOT}/stage/lib" ]] ; then
+    echo "Using cached boost at $BOOST_ROOT"
+    exit
+fi
+
+#fetch/unpack:
+fn=$(basename -- "$BOOST_URL")
+ext="${fn##*.}"
+wget --quiet $BOOST_URL -O /tmp/boost.tar.${ext}
+cd $(dirname $BOOST_ROOT)
+rm -fr ${BOOST_ROOT}
+tar xf /tmp/boost.tar.${ext}
+cd $BOOST_ROOT
+
+BLDARGS=()
+if [[ ${BOOST_BUILD_ALL:-false} == "true" ]]; then
+    # we never need boost-python...so even for ALL
+    # option we can skip it
+    BLDARGS+=(--without-python)
+else
+    BLDARGS+=(--with-chrono)
+    BLDARGS+=(--with-context)
+    BLDARGS+=(--with-coroutine)
+    BLDARGS+=(--with-date_time)
+    BLDARGS+=(--with-filesystem)
+    BLDARGS+=(--with-program_options)
+    BLDARGS+=(--with-regex)
+    BLDARGS+=(--with-serialization)
+    BLDARGS+=(--with-system)
+    BLDARGS+=(--with-atomic)
+    BLDARGS+=(--with-thread)
+fi
+BLDARGS+=(-j$((2*${NUM_PROCESSORS:-2})))
+BLDARGS+=(--prefix=${BOOST_ROOT}/_INSTALLED_)
+BLDARGS+=(-d0) # suppress messages/output
+
+if [[ -z ${COMSPEC:-} ]]; then
+    if [[ "$(uname)" == "Darwin" ]] ; then
+        BLDARGS+=(cxxflags="-std=c++14 -fvisibility=default")
+    else
+        BLDARGS+=(cxxflags="-std=c++14")
+        BLDARGS+=(runtime-link="static,shared")
+    fi
+    BLDARGS+=(--layout=tagged)
+    ./bootstrap.sh
+    ./b2 "${BLDARGS[@]}" stage
+    ./b2 "${BLDARGS[@]}" install
+else
+    BLDARGS+=(runtime-link="static,shared")
+    BLDARGS+=(--layout=versioned)
+    BLDARGS+=(--toolset="msvc-14.1")
+    BLDARGS+=(address-model=64)
+    BLDARGS+=(architecture=x86)
+    BLDARGS+=(link=static)
+    BLDARGS+=(threading=multi)
+    cmd /E:ON /D /S /C"bootstrap.bat"
+    ./b2.exe "${BLDARGS[@]}" stage
+    ./b2.exe "${BLDARGS[@]}" install
+fi
+
+if [[ ${CI:-false} == "true" ]]; then
+    # save some disk space...these are mostly
+    # obj files and don't need to be kept in CI contexts
+    rm -rf bin.v2
+fi
+
+cd $odir
+
--- a/Builds/containers/ubuntu-builder/Dockerfile
+++ b/Builds/containers/ubuntu-builder/Dockerfile
@@ -1,22 +1,24 @@
 ARG DIST_TAG=16.04
-ARG GIT_COMMIT=unknown
-
 FROM ubuntu:$DIST_TAG
+ARG GIT_COMMIT=unknown
+ARG CI_USE=false
 LABEL git-commit=$GIT_COMMIT

 # install/setup prerequisites:
 COPY ubuntu-builder/ubuntu_setup.sh /tmp/
 COPY shared/build_deps.sh /tmp/
 COPY shared/install_cmake.sh /tmp/
+COPY shared/install_boost.sh /tmp/
 RUN chmod +x /tmp/ubuntu_setup.sh && \
    chmod +x /tmp/build_deps.sh && \
+    chmod +x /tmp/install_boost.sh && \
    chmod +x /tmp/install_cmake.sh
 RUN /tmp/ubuntu_setup.sh
 RUN /tmp/install_cmake.sh
 ENV PATH="/opt/local/cmake/bin:$PATH"
 RUN /tmp/build_deps.sh
 ENV PLANTUML_JAR="/opt/plantuml/plantuml.jar"
-ENV BOOST_ROOT="/opt/local/boost"
+ENV BOOST_ROOT="/opt/local/boost/_INSTALLED_"
 ENV OPENSSL_ROOT="/opt/local/openssl"

 # prep files for package building
--- a/Builds/containers/ubuntu-builder/ubuntu_setup.sh
+++ b/Builds/containers/ubuntu-builder/ubuntu_setup.sh
@@ -24,29 +24,27 @@ apt-get -y upgrade
 if [[ ${VERSION_ID} =~ ^18\. ]] ; then
    apt-add-repository -y multiverse
    apt-add-repository -y universe
-elif [[ ${VERSION_ID} =~ ^16\. ]] ; then
-    add-apt-repository -y ppa:ubuntu-toolchain-r/test
 fi
+add-apt-repository -y ppa:ubuntu-toolchain-r/test
 apt-get -y clean
 apt-get -y update

-apt-get -y --fix-missing install  \
-    make cmake ninja-build ccache \
+apt-get -y --fix-missing install \
+    make cmake ninja-build \
    protobuf-compiler libprotobuf-dev openssl libssl-dev \
    liblzma-dev libbz2-dev zlib1g-dev \
    libjemalloc-dev \
    python-pip \
    gdb gdbserver \
    libstdc++6 \
-    flex bison \
+    flex bison parallel \
    libicu-dev texinfo \
    java-common javacc \
-    gcc-7 g++-7 \
-    gcc-8 g++-8 \
    dpkg-dev debhelper devscripts fakeroot \
    debmake git-buildpackage dh-make gitpkg debsums gnupg \
    dh-buildinfo dh-make dh-systemd

+apt-get -y install gcc-7 g++-7
 update-alternatives --install \
    /usr/bin/gcc gcc /usr/bin/gcc-7 40 \
    --slave /usr/bin/g++ g++ /usr/bin/g++-7 \
@@ -56,6 +54,8 @@ update-alternatives --install \
    --slave /usr/bin/gcov gcov /usr/bin/gcov-7 \
    --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-7 \
    --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-7
+
+apt-get -y install gcc-8 g++-8
 update-alternatives --install \
    /usr/bin/gcc gcc /usr/bin/gcc-8 20 \
    --slave /usr/bin/g++ g++ /usr/bin/g++-8 \
@@ -71,6 +71,30 @@ update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-7 40
 update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-8 20
 update-alternatives --auto cpp

+if [ "${CI_USE}" = true ] ; then
+    apt-get -y install gcc-6 g++-6
+    update-alternatives --install \
+        /usr/bin/gcc gcc /usr/bin/gcc-6 10 \
+        --slave /usr/bin/g++ g++ /usr/bin/g++-6 \
+        --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-6 \
+        --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-6 \
+        --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-6 \
+        --slave /usr/bin/gcov gcov /usr/bin/gcov-6 \
+        --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-6 \
+        --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-6
+
+    apt-get -y install gcc-9 g++-9
+    update-alternatives --install \
+        /usr/bin/gcc gcc /usr/bin/gcc-9 15 \
+        --slave /usr/bin/g++ g++ /usr/bin/g++-9 \
+        --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-9 \
+        --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-9 \
+        --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-9 \
+        --slave /usr/bin/gcov gcov /usr/bin/gcov-9 \
+        --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-9 \
+        --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-9
+fi
+
 if [[ ${VERSION_ID} =~ ^18\. ]] ; then
    apt-get -y install binutils
 elif [[ ${VERSION_ID} =~ ^16\. ]] ; then
@@ -82,39 +106,109 @@ if [[ ${VERSION_ID} =~ ^18\. ]] ; then
    cat << EOF > /etc/apt/sources.list.d/llvm.list
 deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main
 deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic main
-deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-6.0 main
-deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-6.0 main
 deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main
 deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main
+deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main
+deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main
 EOF
 elif [[ ${VERSION_ID} =~ ^16\. ]] ; then
    cat << EOF > /etc/apt/sources.list.d/llvm.list
 deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial main
 deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial main
-deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main
-deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main
 deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main
 deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main
+deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main
+deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main
 EOF
 fi
 apt-get -y update

-apt-get -y install  \
-    clang-7 libclang-common-7-dev libclang-7-dev libllvm7 lldb-7 llvm-7 \
-    llvm-7-dev llvm-7-runtime clang-format-7 python-clang-7 python-lldb-7 \
-    liblldb-7-dev lld-7 libfuzzer-7-dev libc++-7-dev
+apt-get -y install \
+    clang-7 libclang-common-7-dev libclang-7-dev libllvm7 llvm-7 \
+    llvm-7-dev llvm-7-runtime clang-format-7 python-clang-7 \
+    lld-7 libfuzzer-7-dev libc++-7-dev
 update-alternatives --install \
-  /usr/bin/clang clang /usr/bin/clang-7 40 \
-  --slave /usr/bin/clang++ clang++ /usr/bin/clang++-7 \
-  --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-7 \
-  --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-7 \
-  --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-7 \
-  --slave /usr/bin/lldb lldb /usr/bin/lldb-7 \
-  --slave /usr/bin/lldb-server lldb-server /usr/bin/lldb-server-7 \
-  --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-7 \
-  --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-7 \
-  --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-7
+    /usr/bin/clang clang /usr/bin/clang-7 40 \
+    --slave /usr/bin/clang++ clang++ /usr/bin/clang++-7 \
+    --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-7 \
+    --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-7 \
+    --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-7 \
+    --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-7 \
+    --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-7 \
+    --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-7 \
+    --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-7
+apt-get -y install \
+    clang-8 libclang-common-8-dev libclang-8-dev libllvm8 llvm-8 \
+    llvm-8-dev llvm-8-runtime clang-format-8 python-clang-8 \
+    lld-8 libfuzzer-8-dev libc++-8-dev
+update-alternatives --install \
+    /usr/bin/clang clang /usr/bin/clang-8 20 \
+    --slave /usr/bin/clang++ clang++ /usr/bin/clang++-8 \
+    --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-8 \
+    --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-8 \
+    --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-8 \
+    --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-8 \
+    --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-8 \
+    --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-8 \
+    --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-8
 update-alternatives --auto clang

+if [ "${CI_USE}" = true ] ; then
+    apt-get -y install \
+        clang-5.0 libclang-common-5.0-dev libclang-5.0-dev libllvm5.0 llvm-5.0 \
+        llvm-5.0-dev llvm-5.0-runtime clang-format-5.0 python-clang-5.0 \
+        lld-5.0 libfuzzer-5.0-dev
+    update-alternatives --install \
+        /usr/bin/clang clang /usr/bin/clang-5.0 10 \
+        --slave /usr/bin/clang++ clang++ /usr/bin/clang++-5.0 \
+        --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-5.0 \
+        --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-5.0 \
+        --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-5.0 \
+        --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-5.0 \
+        --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-5.0 \
+        --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-5.0 \
+        --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-5.0
+
+    apt-get -y install \
+        clang-6.0 libclang-common-6.0-dev libclang-6.0-dev libllvm6.0 llvm-6.0 \
+        llvm-6.0-dev llvm-6.0-runtime clang-format-6.0 python-clang-6.0 \
+        lld-6.0 libfuzzer-6.0-dev
+    update-alternatives --install \
+        /usr/bin/clang clang /usr/bin/clang-6.0 12 \
+        --slave /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 \
+        --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-6.0 \
+        --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-6.0 \
+        --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-6.0 \
+        --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-6.0 \
+        --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-6.0 \
+        --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-6.0 \
+        --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-6.0
+
+    apt-get -y install \
+        clang-9 libclang-common-9-dev libclang-9-dev libllvm9 llvm-9 \
+        llvm-9-dev llvm-9-runtime clang-format-9 python-clang-9 \
+        lld-9 libfuzzer-9-dev libc++-9-dev
+    update-alternatives --install \
+        /usr/bin/clang clang /usr/bin/clang-9 20 \
+        --slave /usr/bin/clang++ clang++ /usr/bin/clang++-9 \
+        --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-9 \
+        --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-9 \
+        --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-9 \
+        --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-9 \
+        --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-9 \
+        --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-9 \
+        --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-9
+
+    # only install latest lldb
+    apt-get -y install lldb-9 python-lldb-9 liblldb-9-dev
+    update-alternatives --install \
+        /usr/bin/lldb lldb /usr/bin/lldb-9 50 \
+        --slave /usr/bin/lldb-server lldb-server /usr/bin/lldb-server-9 \
+        --slave /usr/bin/lldb-argdumper lldb-argdumper /usr/bin/lldb-argdumper-9 \
+        --slave /usr/bin/lldb-instr lldb-instr /usr/bin/lldb-instr-9 \
+        --slave /usr/bin/lldb-mi lldb-mi /usr/bin/lldb-mi-9
+    update-alternatives --auto clang
+fi
+
 apt-get -y autoremove