Build packages with Ubuntu 18.04

Restores Ubuntu 18.04 packages Update docker images to use Conan
2025-11-04 10:45:50 +00:00 · 2023-03-21 02:18:01 -07:00
parent f7b3ddd87b
commit da18c86cbf
8 changed files with 153 additions and 120 deletions
--- a/Builds/CMake/RippledCompiler.cmake
+++ b/Builds/CMake/RippledCompiler.cmake
@@ -124,7 +124,10 @@ else ()
      #   * static option set and
      #   * NOT APPLE (AppleClang does not support static libc/c++) and
      #   * NOT san (sanitizers typically don't work with static libc/c++)
-      $<$<AND:$<BOOL:${static}>,$<NOT:$<BOOL:${APPLE}>>,$<NOT:$<BOOL:${san}>>>:-static-libstdc++>)
+      $<$<AND:$<BOOL:${static}>,$<NOT:$<BOOL:${APPLE}>>,$<NOT:$<BOOL:${san}>>>:
+      -static-libstdc++
+      -static-libgcc
+      >)
 endif ()

 if (use_gold AND is_gcc)
--- a/Builds/CMake/RippledRelease.cmake
+++ b/Builds/CMake/RippledRelease.cmake
@@ -64,16 +64,16 @@ if (is_root_project)
    #[===================================================================[
        dpkg
    #]===================================================================]
-    # currently use ubuntu 16.04 as a base b/c it has one of
+    # currently use ubuntu 18.04 as a base b/c it has one of
    # the lower versions of libc among ubuntu and debian releases.
    # we could change this in the future and build with some other deb
    # based system.
    add_custom_target (dpkg_container
      docker build
        --pull
-        --build-arg DIST_TAG=20.04
+        --build-arg DIST_TAG=18.04
        --build-arg GIT_COMMIT=${commit_hash}
-        -t rippleci/rippled-dpkg-builder:${container_label}
+        -t rippled-dpkg-builder:${container_label}
        $<$<BOOL:${dpkg_cache_from}>:--cache-from=${dpkg_cache_from}>
        -f ubuntu-builder/Dockerfile .
      WORKING_DIRECTORY  ${CMAKE_CURRENT_SOURCE_DIR}/Builds/containers
@@ -119,7 +119,7 @@ if (is_root_project)
      docker run
        -v ${CMAKE_CURRENT_SOURCE_DIR}:/opt/rippled_bld/pkg/rippled
        -v ${CMAKE_CURRENT_BINARY_DIR}/packages:/opt/rippled_bld/pkg/out
-        -t rippleci/rippled-dpkg-builder:${container_label}
+        -t rippled-dpkg-builder:${container_label}
        /bin/bash -c "cp -fpu rippled/Builds/containers/packaging/dpkg/build_dpkg.sh . && ./build_dpkg.sh"
      VERBATIM
      USES_TERMINAL
@@ -159,7 +159,7 @@ if (is_root_project)
    add_custom_target (ci_container
      docker build
        --pull
-        --build-arg DIST_TAG=20.04
+        --build-arg DIST_TAG=18.04
        --build-arg GIT_COMMIT=${commit_hash}
        --build-arg CI_USE=true
        -t rippled-ci-builder:${container_label}
--- a/Builds/containers/gitlab-ci/pkgbuild.yml
+++ b/Builds/containers/gitlab-ci/pkgbuild.yml
@@ -12,10 +12,10 @@ variables:
  # cmake build (container targets) and tagged/pushed so they
  # can be used here
  RPM_CONTAINER_TAG: "2023-02-13"
-  RPM_CONTAINER_NAME: "rippleci/rippled-rpm-builder"
+  RPM_CONTAINER_NAME: "rippled-rpm-builder"
  RPM_CONTAINER_FULLNAME: "${RPM_CONTAINER_NAME}:${RPM_CONTAINER_TAG}"
-  DPKG_CONTAINER_TAG: "2023-02-13"
-  DPKG_CONTAINER_NAME: "rippleci/rippled-dpkg-builder"
+  DPKG_CONTAINER_TAG: "2023-03-20"
+  DPKG_CONTAINER_NAME: "rippled-dpkg-builder"
  DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}"
  ARTIFACTORY_HOST: "artifactory.ops.ripple.com"
  ARTIFACTORY_HUB: "${ARTIFACTORY_HOST}:6555"
@@ -190,7 +190,7 @@ rocky_8_smoketest:
  dependencies:
    - rpm_build
  image:
-    name: rockylinux/rockylinux:8
+    name: artifactory.ops.ripple.com/rockylinux/rockylinux:8
  <<: *run_local_smoketest

 fedora_37_smoketest:
@@ -209,6 +209,14 @@ fedora_38_smoketest:
    name: artifactory.ops.ripple.com/fedora:38
  <<: *run_local_smoketest

+ubuntu_18_smoketest:
+  stage: smoketest
+  dependencies:
+    - dpkg_build
+  image:
+    name: artifactory.ops.ripple.com/ubuntu:18.04
+  <<: *run_local_smoketest
+
 ubuntu_20_smoketest:
  stage: smoketest
  dependencies:
@@ -225,6 +233,14 @@ ubuntu_22_smoketest:
    name: artifactory.ops.ripple.com/ubuntu:22.04
  <<: *run_local_smoketest

+debian_10_smoketest:
+  stage: smoketest
+  dependencies:
+    - dpkg_build
+  image:
+    name: artifactory.ops.ripple.com/debian:10
+  <<: *run_local_smoketest
+
 debian_11_smoketest:
  stage: smoketest
  dependencies:
@@ -329,12 +345,13 @@ centos_7_verify_repo_test:
  <<: *only_primary
  <<: *run_repo_smoketest

+
 rocky_8_verify_repo_test:
  stage: verify_from_test
  variables:
    RPM_REPO: "rippled-rpm-test-mirror"
  image:
-    name: rockylinux/rockylinux:8
+    name: artifactory.ops.ripple.com/rockylinux/rockylinux:8
  dependencies:
    - rpm_sign
  <<: *only_primary
@@ -362,6 +379,18 @@ fedora_38_verify_repo_test:
  <<: *only_primary
  <<: *run_repo_smoketest

+ubuntu_18_verify_repo_test:
+  stage: verify_from_test
+  variables:
+    DISTRO: "bionic"
+    DEB_REPO: "rippled-deb-test-mirror"
+  image:
+    name: artifactory.ops.ripple.com/ubuntu:18.04
+  dependencies:
+    - dpkg_sign
+  <<: *only_primary
+  <<: *run_repo_smoketest
+
 ubuntu_20_verify_repo_test:
  stage: verify_from_test
  variables:
@@ -386,6 +415,18 @@ ubuntu_22_verify_repo_test:
  <<: *only_primary
  <<: *run_repo_smoketest

+debian_10_verify_repo_test:
+  stage: verify_from_test
+  variables:
+    DISTRO: "buster"
+    DEB_REPO: "rippled-deb-test-mirror"
+  image:
+    name: artifactory.ops.ripple.com/debian:10
+  dependencies:
+    - dpkg_sign
+  <<: *only_primary
+  <<: *run_repo_smoketest
+
 debian_11_verify_repo_test:
  stage: verify_from_test
  variables:
@@ -468,7 +509,7 @@ rocky_8_verify_repo_prod:
  variables:
    RPM_REPO: "rippled-rpm"
  image:
-    name: rockylinux/rockylinux:8
+    name: artifactory.ops.ripple.com/rockylinux/rockylinux:8
  dependencies:
    - rpm_sign
  <<: *only_primary
@@ -496,6 +537,18 @@ fedora_38_verify_repo_prod:
  <<: *only_primary
  <<: *run_repo_smoketest

+ubuntu_18_verify_repo_prod:
+  stage: verify_from_prod
+  variables:
+    DISTRO: "bionic"
+    DEB_REPO: "rippled-deb"
+  image:
+    name: artifactory.ops.ripple.com/ubuntu:18.04
+  dependencies:
+    - dpkg_sign
+  <<: *only_primary
+  <<: *run_repo_smoketest
+
 ubuntu_20_verify_repo_prod:
  stage: verify_from_prod
  variables:
@@ -520,6 +573,18 @@ ubuntu_22_verify_repo_prod:
  <<: *only_primary
  <<: *run_repo_smoketest

+debian_10_verify_repo_prod:
+  stage: verify_from_prod
+  variables:
+    DISTRO: "buster"
+    DEB_REPO: "rippled-deb"
+  image:
+    name: artifactory.ops.ripple.com/debian:10
+  dependencies:
+    - dpkg_sign
+  <<: *only_primary
+  <<: *run_repo_smoketest
+
 debian_11_verify_repo_prod:
  stage: verify_from_prod
  variables:
--- a/Builds/containers/gitlab-ci/push_to_artifactory.sh
+++ b/Builds/containers/gitlab-ci/push_to_artifactory.sh
@@ -20,7 +20,7 @@ RIPPLED_REPORTING_DBG_PKG=$(ls rippled-reporting-dbgsym_*.*deb)
 # TODO - where to upload src tgz?
 RIPPLED_SRC=$(ls rippled_*.orig.tar.gz)
 DEB_MATRIX=";deb.component=${COMPONENT};deb.architecture=amd64"
-for dist in bullseye focal jammy; do
+for dist in buster bullseye bionic focal jammy; do
    DEB_MATRIX="${DEB_MATRIX};deb.distribution=${dist}"
 done
 echo "{ \"debs\": {" > "${TOPDIR}/files.info"
--- a/Builds/containers/packaging/dpkg/build_dpkg.sh
+++ b/Builds/containers/packaging/dpkg/build_dpkg.sh
@@ -45,8 +45,6 @@ CHANGELOG
 # PATH must be preserved for our more modern cmake in /opt/local
 # TODO : consider allowing lintian to run in future ?
 export DH_BUILD_DDEBS=1
-export CC=gcc-11
-export CXX=g++-11
 debuild --no-lintian --preserve-envvar PATH --preserve-env -us -uc
 rc=$?; if [[ $rc != 0 ]]; then
    error "error building dpkg"
--- a/Builds/containers/packaging/dpkg/debian/rules
+++ b/Builds/containers/packaging/dpkg/debian/rules
@@ -23,6 +23,8 @@ override_dh_auto_configure:
 	conan install . \
 		--install-folder bld/rippled  \
 		--build missing \
+		--build boost \
+		--build sqlite3 \
 		--settings build_type=Release

 	cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
@@ -38,8 +40,10 @@ override_dh_auto_configure:
 	conan install . \
 		--install-folder bld/rippled-reporting \
 		--build missing \
+		--build boost \
+		--build sqlite3 \
+		--build libuv \
 		--settings build_type=Release \
-		--settings compiler.cppstd=17 \
 		--options reporting=True

 	cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
--- a/Builds/containers/ubuntu-builder/Dockerfile
+++ b/Builds/containers/ubuntu-builder/Dockerfile
@@ -1,22 +1,13 @@
-ARG DIST_TAG=20.04
+ARG DIST_TAG=18.04
 FROM ubuntu:$DIST_TAG
 ARG GIT_COMMIT=unknown
 ARG CI_USE=false
 LABEL git-commit=$GIT_COMMIT

-# install/setup prerequisites:
-COPY ubuntu-builder/ubuntu_setup.sh /tmp/
-COPY shared/install_cmake.sh /tmp/
-RUN chmod +x /tmp/ubuntu_setup.sh && \
-    chmod +x /tmp/install_cmake.sh
-RUN /tmp/ubuntu_setup.sh
+WORKDIR /root
+COPY ubuntu-builder/ubuntu_setup.sh .
+RUN ./ubuntu_setup.sh && rm ubuntu_setup.sh

-RUN /tmp/install_cmake.sh 3.16.3 /opt/local/cmake-3.16
-RUN ln -s /opt/local/cmake-3.16 /opt/local/cmake
-ENV PATH="/opt/local/cmake/bin:$PATH"
-
-# prep files for package building
-RUN update-alternatives --set gcc /usr/bin/gcc-11
 RUN mkdir -m 777 -p /opt/rippled_bld/pkg/
 WORKDIR /opt/rippled_bld/pkg

--- a/Builds/containers/ubuntu-builder/ubuntu_setup.sh
+++ b/Builds/containers/ubuntu-builder/ubuntu_setup.sh
@@ -1,104 +1,76 @@
 #!/usr/bin/env bash
-set -ex

-source /etc/os-release
+set -o errexit
+set -o nounset
+set -o xtrace

-if [[ ${VERSION_ID} =~ ^20\. || ${VERSION_ID} =~ ^22\. ]] ; then
-    echo "setup for ${PRETTY_NAME}"
-else
-    echo "${VERSION} not supported"
-    exit 1
-fi
+# Parameters

-export DEBIAN_FRONTEND="noninteractive"
-echo "Acquire::Retries 3;" > /etc/apt/apt.conf.d/80-retries
-echo "Acquire::http::Pipeline-Depth 0;" >> /etc/apt/apt.conf.d/80-retries
-echo "Acquire::http::No-Cache true;" >> /etc/apt/apt.conf.d/80-retries
-echo "Acquire::BrokenProxy    true;" >> /etc/apt/apt.conf.d/80-retries
-apt-get update -o Acquire::CompressionTypes::Order::=gz
+gcc_version=${GCC_VERSION:-10}
+cmake_version=${CMAKE_VERSION:-3.25.1}
+conan_version=${CONAN_VERSION:-1.59}

-apt-get -y update
-apt-get -y install apt-utils
-apt-get -y install software-properties-common wget curl ca-certificates
-apt-get -y install python3-pip
-apt-get -y upgrade
-add-apt-repository -y ppa:ubuntu-toolchain-r/test
-apt-get -y clean
-apt-get -y update
+apt update
+# Iteratively build the list of packages to install so that we can interleave
+# the lines with comments explaining their inclusion.
+dependencies=''
+# - to identify the Ubuntu version
+dependencies+=' lsb-release'
+# - for add-apt-repository
+dependencies+=' software-properties-common'
+# - to download CMake
+dependencies+=' curl'
+# - to build CMake
+dependencies+=' libssl-dev'
+# - Python headers for Boost.Python
+dependencies+=' python3-dev'
+# - to install Conan
+dependencies+=' python3-pip'
+# - to download rippled
+dependencies+=' git'
+# - CMake generators (but not CMake itself)
+dependencies+=' make ninja-build'
+apt install --yes ${dependencies}

-apt-get -y --fix-missing install \
-    make cmake ninja-build autoconf automake libtool pkg-config libtool \
-    openssl libssl-dev \
-    liblzma-dev libbz2-dev zlib1g-dev \
-    libjemalloc-dev \
-    gdb gdbserver \
-    libstdc++6 \
-    flex bison parallel \
-    libicu-dev texinfo \
-    java-common javacc \
-    dpkg-dev debhelper devscripts fakeroot \
-    debmake git-buildpackage dh-make gitpkg debsums gnupg \
-    dh-buildinfo dh-make  \
-    apt-transport-https
+add-apt-repository --yes ppa:ubuntu-toolchain-r/test
+apt install --yes gcc-${gcc_version} g++-${gcc_version} \
+  debhelper debmake debsums gnupg dh-buildinfo dh-make dh-systemd cmake \
+  ninja-build zlib1g-dev make cmake ninja-build autoconf automake \
+  pkg-config apt-transport-https

-if [[ ${VERSION_ID} =~ ^20\. ]] ; then
-apt-get install -y \
-    dh-systemd
-fi
-
-apt-get -y install gcc-11 g++-11
+# Give us nice unversioned aliases for gcc and company.
 update-alternatives --install \
-    /usr/bin/gcc gcc /usr/bin/gcc-11 20 \
-    --slave /usr/bin/g++ g++ /usr/bin/g++-11 \
-    --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \
-    --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-11 \
-    --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 \
-    --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \
-    --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-11 \
-    --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-11
+  /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 \
+  --slave /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} \
+  --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-${gcc_version} \
+  --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-${gcc_version} \
+  --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-${gcc_version} \
+  --slave /usr/bin/gcov gcov /usr/bin/gcov-${gcc_version} \
+  --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-${gcc_version} \
+  --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-${gcc_version}
 update-alternatives --auto gcc

-update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 20
-update-alternatives --auto cpp
+# Download and unpack CMake.
+cmake_slug="cmake-${cmake_version}"
+curl --location --remote-name \
+  "https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_slug}.tar.gz"
+tar xzf ${cmake_slug}.tar.gz
+rm ${cmake_slug}.tar.gz

-wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+# Build and install CMake.
+cd ${cmake_slug}
+./bootstrap --parallel=$(nproc)
+make --jobs $(nproc)
+make install
+cd ..
+rm --recursive --force ${cmake_slug}

-if [[ ${VERSION_ID} =~ ^20\. ]] ; then
-    cat << EOF > /etc/apt/sources.list.d/llvm.list
-deb http://apt.llvm.org/focal/ llvm-toolchain-focal main
-deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal main
-deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main
-deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main
-deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main
-deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main
-EOF
-    apt-get -y install binutils clang-12
-fi
+# Install Conan.
+pip3 install conan==${conan_version}

-
-apt-get -y update
-if [[ ${VERSION_ID} =~ ^20\. ]] ; then
-for v in 12 14; do
-    apt-get -y install \
-        clang-$v libclang-common-$v-dev libclang-$v-dev libllvm$v llvm-$v \
-        llvm-$v-dev llvm-$v-runtime clang-format-$v python3-clang-$v \
-        lld-$v libfuzzer-$v-dev libc++-$v-dev python-is-python3
-    update-alternatives --install \
-        /usr/bin/clang clang /usr/bin/clang-$v 40 \
-        --slave /usr/bin/clang++ clang++ /usr/bin/clang++-$v \
-        --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-$v \
-        --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-$v \
-        --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-$v \
-        --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-$v \
-        --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-$v \
-        --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-$v \
-        --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-$v
-    done
-fi
-
-pip install "conan<2" && \
-    conan profile new default --detect && \
-    conan profile update settings.compiler.cppstd=20 default && \
-    conan profile update settings.compiler.libcxx=libstdc++11 default
-
-apt-get -y autoremove
+conan profile new --detect gcc
+conan profile update settings.compiler=gcc gcc
+conan profile update settings.compiler.version=${gcc_version} gcc
+conan profile update settings.compiler.libcxx=libstdc++11 gcc
+conan profile update env.CC=/usr/bin/gcc gcc
+conan profile update env.CXX=/usr/bin/g++ gcc