Build packages with Ubuntu 18.04

Restores Ubuntu 18.04 packages
Update docker images to use Conan

Builds/CMake/RippledCompiler.cmake

@@ -124,7 +124,10 @@ else ()
       #   * static option set and
       #   * NOT APPLE (AppleClang does not support static libc/c++) and
       #   * NOT san (sanitizers typically don't work with static libc/c++)
-      $<$<AND:$<BOOL:${static}>,$<NOT:$<BOOL:${APPLE}>>,$<NOT:$<BOOL:${san}>>>:-static-libstdc++>)
+      $<$<AND:$<BOOL:${static}>,$<NOT:$<BOOL:${APPLE}>>,$<NOT:$<BOOL:${san}>>>:
+      -static-libstdc++
+      -static-libgcc
+      >)
 endif ()
 
 if (use_gold AND is_gcc)

Builds/CMake/RippledRelease.cmake

@@ -64,16 +64,16 @@ if (is_root_project)
     #[===================================================================[
         dpkg
     #]===================================================================]
-    # currently use ubuntu 16.04 as a base b/c it has one of
+    # currently use ubuntu 18.04 as a base b/c it has one of
     # the lower versions of libc among ubuntu and debian releases.
     # we could change this in the future and build with some other deb
     # based system.
     add_custom_target (dpkg_container
       docker build
         --pull
-        --build-arg DIST_TAG=20.04
+        --build-arg DIST_TAG=18.04
         --build-arg GIT_COMMIT=${commit_hash}
-        -t rippleci/rippled-dpkg-builder:${container_label}
+        -t rippled-dpkg-builder:${container_label}
         $<$<BOOL:${dpkg_cache_from}>:--cache-from=${dpkg_cache_from}>
         -f ubuntu-builder/Dockerfile .
       WORKING_DIRECTORY  ${CMAKE_CURRENT_SOURCE_DIR}/Builds/containers
@@ -119,7 +119,7 @@ if (is_root_project)
       docker run
         -v ${CMAKE_CURRENT_SOURCE_DIR}:/opt/rippled_bld/pkg/rippled
         -v ${CMAKE_CURRENT_BINARY_DIR}/packages:/opt/rippled_bld/pkg/out
-        -t rippleci/rippled-dpkg-builder:${container_label}
+        -t rippled-dpkg-builder:${container_label}
         /bin/bash -c "cp -fpu rippled/Builds/containers/packaging/dpkg/build_dpkg.sh . && ./build_dpkg.sh"
       VERBATIM
       USES_TERMINAL
@@ -159,7 +159,7 @@ if (is_root_project)
     add_custom_target (ci_container
       docker build
         --pull
-        --build-arg DIST_TAG=20.04
+        --build-arg DIST_TAG=18.04
         --build-arg GIT_COMMIT=${commit_hash}
         --build-arg CI_USE=true
         -t rippled-ci-builder:${container_label}

Builds/containers/gitlab-ci/pkgbuild.yml

@@ -12,10 +12,10 @@ variables:
   # cmake build (container targets) and tagged/pushed so they
   # can be used here
   RPM_CONTAINER_TAG: "2023-02-13"
-  RPM_CONTAINER_NAME: "rippleci/rippled-rpm-builder"
+  RPM_CONTAINER_NAME: "rippled-rpm-builder"
   RPM_CONTAINER_FULLNAME: "${RPM_CONTAINER_NAME}:${RPM_CONTAINER_TAG}"
-  DPKG_CONTAINER_TAG: "2023-02-13"
+  DPKG_CONTAINER_TAG: "2023-03-20"
-  DPKG_CONTAINER_NAME: "rippleci/rippled-dpkg-builder"
+  DPKG_CONTAINER_NAME: "rippled-dpkg-builder"
   DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}"
   ARTIFACTORY_HOST: "artifactory.ops.ripple.com"
   ARTIFACTORY_HUB: "${ARTIFACTORY_HOST}:6555"
@@ -190,7 +190,7 @@ rocky_8_smoketest:
   dependencies:
     - rpm_build
   image:
-    name: rockylinux/rockylinux:8
+    name: artifactory.ops.ripple.com/rockylinux/rockylinux:8
   <<: *run_local_smoketest
 
 fedora_37_smoketest:
@@ -209,6 +209,14 @@ fedora_38_smoketest:
     name: artifactory.ops.ripple.com/fedora:38
   <<: *run_local_smoketest
 
+ubuntu_18_smoketest:
+  stage: smoketest
+  dependencies:
+    - dpkg_build
+  image:
+    name: artifactory.ops.ripple.com/ubuntu:18.04
+  <<: *run_local_smoketest
+
 ubuntu_20_smoketest:
   stage: smoketest
   dependencies:
@@ -225,6 +233,14 @@ ubuntu_22_smoketest:
     name: artifactory.ops.ripple.com/ubuntu:22.04
   <<: *run_local_smoketest
 
+debian_10_smoketest:
+  stage: smoketest
+  dependencies:
+    - dpkg_build
+  image:
+    name: artifactory.ops.ripple.com/debian:10
+  <<: *run_local_smoketest
+
 debian_11_smoketest:
   stage: smoketest
   dependencies:
@@ -329,12 +345,13 @@ centos_7_verify_repo_test:
   <<: *only_primary
   <<: *run_repo_smoketest
 
+
 rocky_8_verify_repo_test:
   stage: verify_from_test
   variables:
     RPM_REPO: "rippled-rpm-test-mirror"
   image:
-    name: rockylinux/rockylinux:8
+    name: artifactory.ops.ripple.com/rockylinux/rockylinux:8
   dependencies:
     - rpm_sign
   <<: *only_primary
@@ -362,6 +379,18 @@ fedora_38_verify_repo_test:
   <<: *only_primary
   <<: *run_repo_smoketest
 
+ubuntu_18_verify_repo_test:
+  stage: verify_from_test
+  variables:
+    DISTRO: "bionic"
+    DEB_REPO: "rippled-deb-test-mirror"
+  image:
+    name: artifactory.ops.ripple.com/ubuntu:18.04
+  dependencies:
+    - dpkg_sign
+  <<: *only_primary
+  <<: *run_repo_smoketest
+
 ubuntu_20_verify_repo_test:
   stage: verify_from_test
   variables:
@@ -386,6 +415,18 @@ ubuntu_22_verify_repo_test:
   <<: *only_primary
   <<: *run_repo_smoketest
 
+debian_10_verify_repo_test:
+  stage: verify_from_test
+  variables:
+    DISTRO: "buster"
+    DEB_REPO: "rippled-deb-test-mirror"
+  image:
+    name: artifactory.ops.ripple.com/debian:10
+  dependencies:
+    - dpkg_sign
+  <<: *only_primary
+  <<: *run_repo_smoketest
+
 debian_11_verify_repo_test:
   stage: verify_from_test
   variables:
@@ -468,7 +509,7 @@ rocky_8_verify_repo_prod:
   variables:
     RPM_REPO: "rippled-rpm"
   image:
-    name: rockylinux/rockylinux:8
+    name: artifactory.ops.ripple.com/rockylinux/rockylinux:8
   dependencies:
     - rpm_sign
   <<: *only_primary
@@ -496,6 +537,18 @@ fedora_38_verify_repo_prod:
   <<: *only_primary
   <<: *run_repo_smoketest
 
+ubuntu_18_verify_repo_prod:
+  stage: verify_from_prod
+  variables:
+    DISTRO: "bionic"
+    DEB_REPO: "rippled-deb"
+  image:
+    name: artifactory.ops.ripple.com/ubuntu:18.04
+  dependencies:
+    - dpkg_sign
+  <<: *only_primary
+  <<: *run_repo_smoketest
+
 ubuntu_20_verify_repo_prod:
   stage: verify_from_prod
   variables:
@@ -520,6 +573,18 @@ ubuntu_22_verify_repo_prod:
   <<: *only_primary
   <<: *run_repo_smoketest
 
+debian_10_verify_repo_prod:
+  stage: verify_from_prod
+  variables:
+    DISTRO: "buster"
+    DEB_REPO: "rippled-deb"
+  image:
+    name: artifactory.ops.ripple.com/debian:10
+  dependencies:
+    - dpkg_sign
+  <<: *only_primary
+  <<: *run_repo_smoketest
+
 debian_11_verify_repo_prod:
   stage: verify_from_prod
   variables:

Builds/containers/gitlab-ci/push_to_artifactory.sh

@@ -20,7 +20,7 @@ RIPPLED_REPORTING_DBG_PKG=$(ls rippled-reporting-dbgsym_*.*deb)
 # TODO - where to upload src tgz?
 RIPPLED_SRC=$(ls rippled_*.orig.tar.gz)
 DEB_MATRIX=";deb.component=${COMPONENT};deb.architecture=amd64"
-for dist in bullseye focal jammy; do
+for dist in buster bullseye bionic focal jammy; do
     DEB_MATRIX="${DEB_MATRIX};deb.distribution=${dist}"
 done
 echo "{ \"debs\": {" > "${TOPDIR}/files.info"

Builds/containers/packaging/dpkg/build_dpkg.sh

@@ -45,8 +45,6 @@ CHANGELOG
 # PATH must be preserved for our more modern cmake in /opt/local
 # TODO : consider allowing lintian to run in future ?
 export DH_BUILD_DDEBS=1
-export CC=gcc-11
-export CXX=g++-11
 debuild --no-lintian --preserve-envvar PATH --preserve-env -us -uc
 rc=$?; if [[ $rc != 0 ]]; then
     error "error building dpkg"

Builds/containers/packaging/dpkg/debian/rules

@@ -23,6 +23,8 @@ override_dh_auto_configure:
 	conan install . \
 		--install-folder bld/rippled  \
 		--build missing \
+		--build boost \
+		--build sqlite3 \
 		--settings build_type=Release
 
 	cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
@@ -38,8 +40,10 @@ override_dh_auto_configure:
 	conan install . \
 		--install-folder bld/rippled-reporting \
 		--build missing \
+		--build boost \
+		--build sqlite3 \
+		--build libuv \
 		--settings build_type=Release \
-		--settings compiler.cppstd=17 \
 		--options reporting=True
 
 	cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \

Builds/containers/ubuntu-builder/Dockerfile

@@ -1,22 +1,13 @@
-ARG DIST_TAG=20.04
+ARG DIST_TAG=18.04
 FROM ubuntu:$DIST_TAG
 ARG GIT_COMMIT=unknown
 ARG CI_USE=false
 LABEL git-commit=$GIT_COMMIT
 
-# install/setup prerequisites:
+WORKDIR /root
-COPY ubuntu-builder/ubuntu_setup.sh /tmp/
+COPY ubuntu-builder/ubuntu_setup.sh .
-COPY shared/install_cmake.sh /tmp/
+RUN ./ubuntu_setup.sh && rm ubuntu_setup.sh
-RUN chmod +x /tmp/ubuntu_setup.sh && \
-    chmod +x /tmp/install_cmake.sh
-RUN /tmp/ubuntu_setup.sh
 
-RUN /tmp/install_cmake.sh 3.16.3 /opt/local/cmake-3.16
-RUN ln -s /opt/local/cmake-3.16 /opt/local/cmake
-ENV PATH="/opt/local/cmake/bin:$PATH"
-
-# prep files for package building
-RUN update-alternatives --set gcc /usr/bin/gcc-11
 RUN mkdir -m 777 -p /opt/rippled_bld/pkg/
 WORKDIR /opt/rippled_bld/pkg
 

Builds/containers/ubuntu-builder/ubuntu_setup.sh

@@ -1,104 +1,76 @@
 #!/usr/bin/env bash
-set -ex
 
-source /etc/os-release
+set -o errexit
+set -o nounset
+set -o xtrace
 
-if [[ ${VERSION_ID} =~ ^20\. || ${VERSION_ID} =~ ^22\. ]] ; then
+# Parameters
-    echo "setup for ${PRETTY_NAME}"
-else
-    echo "${VERSION} not supported"
-    exit 1
-fi
 
-export DEBIAN_FRONTEND="noninteractive"
+gcc_version=${GCC_VERSION:-10}
-echo "Acquire::Retries 3;" > /etc/apt/apt.conf.d/80-retries
+cmake_version=${CMAKE_VERSION:-3.25.1}
-echo "Acquire::http::Pipeline-Depth 0;" >> /etc/apt/apt.conf.d/80-retries
+conan_version=${CONAN_VERSION:-1.59}
-echo "Acquire::http::No-Cache true;" >> /etc/apt/apt.conf.d/80-retries
-echo "Acquire::BrokenProxy    true;" >> /etc/apt/apt.conf.d/80-retries
-apt-get update -o Acquire::CompressionTypes::Order::=gz
 
-apt-get -y update
+apt update
-apt-get -y install apt-utils
+# Iteratively build the list of packages to install so that we can interleave
-apt-get -y install software-properties-common wget curl ca-certificates
+# the lines with comments explaining their inclusion.
-apt-get -y install python3-pip
+dependencies=''
-apt-get -y upgrade
+# - to identify the Ubuntu version
-add-apt-repository -y ppa:ubuntu-toolchain-r/test
+dependencies+=' lsb-release'
-apt-get -y clean
+# - for add-apt-repository
-apt-get -y update
+dependencies+=' software-properties-common'
+# - to download CMake
+dependencies+=' curl'
+# - to build CMake
+dependencies+=' libssl-dev'
+# - Python headers for Boost.Python
+dependencies+=' python3-dev'
+# - to install Conan
+dependencies+=' python3-pip'
+# - to download rippled
+dependencies+=' git'
+# - CMake generators (but not CMake itself)
+dependencies+=' make ninja-build'
+apt install --yes ${dependencies}
 
-apt-get -y --fix-missing install \
+add-apt-repository --yes ppa:ubuntu-toolchain-r/test
-    make cmake ninja-build autoconf automake libtool pkg-config libtool \
+apt install --yes gcc-${gcc_version} g++-${gcc_version} \
-    openssl libssl-dev \
+  debhelper debmake debsums gnupg dh-buildinfo dh-make dh-systemd cmake \
-    liblzma-dev libbz2-dev zlib1g-dev \
+  ninja-build zlib1g-dev make cmake ninja-build autoconf automake \
-    libjemalloc-dev \
+  pkg-config apt-transport-https
-    gdb gdbserver \
-    libstdc++6 \
-    flex bison parallel \
-    libicu-dev texinfo \
-    java-common javacc \
-    dpkg-dev debhelper devscripts fakeroot \
-    debmake git-buildpackage dh-make gitpkg debsums gnupg \
-    dh-buildinfo dh-make  \
-    apt-transport-https
 
-if [[ ${VERSION_ID} =~ ^20\. ]] ; then
+# Give us nice unversioned aliases for gcc and company.
-apt-get install -y \
-    dh-systemd
-fi
-
-apt-get -y install gcc-11 g++-11
 update-alternatives --install \
-    /usr/bin/gcc gcc /usr/bin/gcc-11 20 \
+  /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 \
-    --slave /usr/bin/g++ g++ /usr/bin/g++-11 \
+  --slave /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} \
-    --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \
+  --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-${gcc_version} \
-    --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-11 \
+  --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-${gcc_version} \
-    --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 \
+  --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-${gcc_version} \
-    --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \
+  --slave /usr/bin/gcov gcov /usr/bin/gcov-${gcc_version} \
-    --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-11 \
+  --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-${gcc_version} \
-    --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-11
+  --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-${gcc_version}
 update-alternatives --auto gcc
 
-update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 20
+# Download and unpack CMake.
-update-alternatives --auto cpp
+cmake_slug="cmake-${cmake_version}"
+curl --location --remote-name \
+  "https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_slug}.tar.gz"
+tar xzf ${cmake_slug}.tar.gz
+rm ${cmake_slug}.tar.gz
 
-wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+# Build and install CMake.
+cd ${cmake_slug}
+./bootstrap --parallel=$(nproc)
+make --jobs $(nproc)
+make install
+cd ..
+rm --recursive --force ${cmake_slug}
 
-if [[ ${VERSION_ID} =~ ^20\. ]] ; then
+# Install Conan.
-    cat << EOF > /etc/apt/sources.list.d/llvm.list
+pip3 install conan==${conan_version}
-deb http://apt.llvm.org/focal/ llvm-toolchain-focal main
-deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal main
-deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main
-deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main
-deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main
-deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main
-EOF
-    apt-get -y install binutils clang-12
-fi
 
-
+conan profile new --detect gcc
-apt-get -y update
+conan profile update settings.compiler=gcc gcc
-if [[ ${VERSION_ID} =~ ^20\. ]] ; then
+conan profile update settings.compiler.version=${gcc_version} gcc
-for v in 12 14; do
+conan profile update settings.compiler.libcxx=libstdc++11 gcc
-    apt-get -y install \
+conan profile update env.CC=/usr/bin/gcc gcc
-        clang-$v libclang-common-$v-dev libclang-$v-dev libllvm$v llvm-$v \
+conan profile update env.CXX=/usr/bin/g++ gcc
-        llvm-$v-dev llvm-$v-runtime clang-format-$v python3-clang-$v \
-        lld-$v libfuzzer-$v-dev libc++-$v-dev python-is-python3
-    update-alternatives --install \
-        /usr/bin/clang clang /usr/bin/clang-$v 40 \
-        --slave /usr/bin/clang++ clang++ /usr/bin/clang++-$v \
-        --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-$v \
-        --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-$v \
-        --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-$v \
-        --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-$v \
-        --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-$v \
-        --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-$v \
-        --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-$v
-    done
-fi
-
-pip install "conan<2" && \
-    conan profile new default --detect && \
-    conan profile update settings.compiler.cppstd=20 default && \
-    conan profile update settings.compiler.libcxx=libstdc++11 default
-
-apt-get -y autoremove