Update Ubuntu build image (#4650)

Builds/CMake/RippledRelease.cmake

@@ -49,7 +49,7 @@ if (is_root_project)
       docker run
         -v ${CMAKE_CURRENT_SOURCE_DIR}:/opt/rippled_bld/pkg/rippled
         -v ${CMAKE_CURRENT_BINARY_DIR}/packages:/opt/rippled_bld/pkg/out
-        -t rippleci/rippled-rpm-builder:${container_label}
+        -t rippled-rpm-builder:${container_label}
         /bin/bash -c "cp -fpu rippled/Builds/containers/packaging/rpm/build_rpm.sh . && ./build_rpm.sh"
       VERBATIM
       USES_TERMINAL

Builds/containers/gitlab-ci/pkgbuild.yml

@@ -14,7 +14,7 @@ variables:
   RPM_CONTAINER_TAG: "2023-02-13"
   RPM_CONTAINER_NAME: "rippled-rpm-builder"
   RPM_CONTAINER_FULLNAME: "${RPM_CONTAINER_NAME}:${RPM_CONTAINER_TAG}"
-  DPKG_CONTAINER_TAG: "2023-03-20"
+  DPKG_CONTAINER_TAG: "2023-07-31"
   DPKG_CONTAINER_NAME: "rippled-dpkg-builder"
   DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}"
   ARTIFACTORY_HOST: "artifactory.ops.ripple.com"

Builds/containers/packaging/dpkg/debian/rippled-reporting.install

@@ -1,4 +1,4 @@
-bld/rippled-reporting/rippled-reporting opt/rippled-reporting/bin
+build.rippled-reporting/rippled-reporting opt/rippled-reporting/bin
 cfg/rippled-reporting.cfg opt/rippled-reporting/etc
 debian/tmp/opt/rippled-reporting/etc/validators.txt opt/rippled-reporting/etc
 

Builds/containers/packaging/dpkg/debian/rules

@@ -15,69 +15,56 @@ override_dh_systemd_start:
 	dh_systemd_start --no-restart-on-upgrade
 
 override_dh_auto_configure:
-	apt install --yes gcc-11 g++-11
+	/root/.pyenv/shims/conan export external/snappy snappy/1.1.10@
-	update-alternatives --install \
+	/root/.pyenv/shims/conan export external/soci soci/4.0.3@
-		/usr/bin/gcc gcc /usr/bin/gcc-11 100 \
-		--slave /usr/bin/g++ g++ /usr/bin/g++-11 \
-		--slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \
-		--slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-11 \
-		--slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 \
-		--slave /usr/bin/gcov gcov /usr/bin/gcov-11 \
-		--slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-11 \
-		--slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-11
-	update-alternatives --set gcc /usr/bin/gcc-11
-	env
-	rm -rf bld
-	conan profile update settings.compiler.cppstd=20 gcc
-	conan profile update settings.compiler.version=11 gcc
-	conan export external/snappy snappy/1.1.9@
 
-	conan install . \
+	mkdir build.rippled
-		--profile gcc \
-		--install-folder bld/rippled  \
-		--build missing \
-		--build boost \
-		--build sqlite3 \
-		--settings build_type=Release
 
-	cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
+	cd build.rippled && \
-		-G Ninja \
+		/root/.pyenv/shims/conan install .. \
-		-DCMAKE_BUILD_TYPE=Release \
+			--profile gcc \
-		-DCMAKE_INSTALL_PREFIX=/opt/ripple \
+			--output-folder . \
-		-Dstatic=ON \
+			--build missing \
-		-Dunity=OFF \
+			--settings build_type=Release
-		-DCMAKE_VERBOSE_MAKEFILE=ON \
-		-Dvalidator_keys=ON \
-		-B bld/rippled
 
-	conan install . \
+	cd build.rippled && \
-		--profile gcc \
+		cmake .. \
-		--install-folder bld/rippled-reporting \
+			-DCMAKE_BUILD_TYPE=Release \
-		--build missing \
+			-Dvalidator_keys=ON \
-		--build boost \
+			-DCMAKE_VERBOSE_MAKEFILE=ON \
-		--build sqlite3 \
+			-DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake
-		--build libuv \
-		--settings build_type=Release \
-		--settings compiler.cppstd=17 \
-		--options reporting=True
 
-	cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
+	mkdir build.rippled-reporting
-		-G Ninja \
+
-		-DCMAKE_BUILD_TYPE=Release \
+	cd build.rippled-reporting && \
-		-DCMAKE_INSTALL_PREFIX=/opt/rippled-reporting \
+		/root/.pyenv/shims/conan install .. \
-		-Dstatic=ON \
+			--profile gcc \
-		-Dunity=OFF \
+			--output-folder . \
-		-DCMAKE_VERBOSE_MAKEFILE=ON \
+			--settings compiler.cppstd=17 \
-		-Dreporting=ON \
+			--settings build_type=Release \
-		-B bld/rippled-reporting
+			--build missing \
+			--build boost \
+			--build sqlite3 \
+			--build libuv \
+			--options reporting=True
+
+	cd build.rippled-reporting && \
+		cmake .. \
+			-DCMAKE_BUILD_TYPE=Release \
+			-Dvalidator_keys=ON \
+			-Dstatic=ON \
+			-Dunity=OFF \
+			-Dreporting=ON \
+			-DCMAKE_VERBOSE_MAKEFILE=ON \
+			-DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake
 
 override_dh_auto_build:
-	cmake --build bld/rippled --target rippled --target validator-keys -j${nproc}
+	cmake --build build.rippled --target rippled --target validator-keys --parallel 8
-	cmake --build bld/rippled-reporting --target rippled -j${nproc}
+	cmake --build build.rippled-reporting --target rippled --parallel 8
 
 override_dh_auto_install:
-	cmake --install bld/rippled --prefix debian/tmp/opt/ripple
+	cmake --install build.rippled --prefix debian/tmp/opt/ripple
-	install -D bld/rippled/validator-keys/validator-keys debian/tmp/opt/ripple/bin/validator-keys
+	install -D build.rippled/validator-keys/validator-keys debian/tmp/opt/ripple/bin/validator-keys
 	install -D Builds/containers/shared/update-rippled.sh debian/tmp/opt/ripple/bin/update-rippled.sh
 	install -D bin/getRippledInfo debian/tmp/opt/ripple/bin/getRippledInfo
 	install -D Builds/containers/shared/update-rippled-cron debian/tmp/opt/ripple/etc/update-rippled-cron

Builds/containers/packaging/rpm/rippled.spec

@@ -36,41 +36,44 @@ History server for XRP Ledger
 %setup -c -n rippled
 
 %build
-rm -rf ~/.conan/profiles/default
+
+source /opt/rh/devtoolset-11/enable
+source /opt/rh/rh-python38/enable
+
+pip install "conan<2"
+
+conan profile new default --detect
+conan profile update settings.compiler.cppstd=20 default
+conan profile update settings.compiler.libcxx=libstdc++11 default
+
+cd rippled
+
+conan export external/snappy snappy/1.1.10@
+conan export external/soci soci/4.0.3@
+
+mkdir -p bld.rippled
+pushd bld.rippled
 
 cp /opt/libcstd/libstdc++.so.6.0.22 /usr/lib64
 cp /opt/libcstd/libstdc++.so.6.0.22 /lib64
 ln -sf /usr/lib64/libstdc++.so.6.0.22 /usr/lib64/libstdc++.so.6
 ln -sf /lib64/libstdc++.so.6.0.22 /usr/lib64/libstdc++.so.6
 
-source /opt/rh/rh-python38/enable
-pip install "conan<2"
-conan profile new default --detect
-conan profile update settings.compiler.libcxx=libstdc++11 default
-conan profile update settings.compiler.cppstd=20 default
-
-cd rippled
-
-mkdir -p bld.rippled
-conan export external/snappy snappy/1.1.9@
-
-pushd bld.rippled
 conan install .. \
-     --settings build_type=Release \
+    --profile default \
-     --output-folder . \
+    --output-folder . \
-     --build missing
+    --build missing \
+    --settings build_type=Release
 
-cmake -G Ninja \
+cmake .. \
-     -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
+    -DCMAKE_BUILD_TYPE=Release \
-     -DCMAKE_INSTALL_PREFIX=%{_prefix} \
+    -Dvalidator_keys=ON \
-     -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=%{_prefix} \
-     -Dunity=OFF \
+    -DCMAKE_VERBOSE_MAKEFILE=ON \
-     -Dstatic=ON \
+    -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake
-     -Dvalidator_keys=ON \
-     -DCMAKE_VERBOSE_MAKEFILE=ON \
-     ..
 
 cmake --build . --parallel $(nproc) --target rippled --target validator-keys
+
 popd
 
 mkdir -p bld.rippled-reporting
@@ -83,16 +86,16 @@ conan install .. \
      --settings compiler.cppstd=17 \
      --options reporting=True
 
-cmake -G Ninja \
+cmake .. \
-     -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
+    -G Ninja \
-     -DCMAKE_INSTALL_PREFIX=%{_prefix} \
+    -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
-     -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=%{_prefix} \
-     -Dunity=OFF \
+    -DCMAKE_BUILD_TYPE=Release \
-     -Dstatic=ON \
+    -Dunity=OFF \
-     -Dvalidator_keys=ON \
+    -Dstatic=ON \
-     -Dreporting=ON \
+    -Dvalidator_keys=ON \
-     -DCMAKE_VERBOSE_MAKEFILE=ON \
+    -Dreporting=ON \
-     ..
+    -DCMAKE_VERBOSE_MAKEFILE=ON \
 
 cmake --build . --parallel $(nproc) --target rippled
 
@@ -103,7 +106,7 @@ test -e /etc/pki/tls || { mkdir -p /etc/pki; ln -s /usr/lib/ssl /etc/pki/tls; }
 rm -rf $RPM_BUILD_ROOT
 DESTDIR=$RPM_BUILD_ROOT cmake --build rippled/bld.rippled --target install #-- -v
 mkdir -p $RPM_BUILD_ROOT
-rm -rf ${RPM_BUILD_ROOT}/%{_prefix}/lib64/
+rm -rf ${RPM_BUILD_ROOT}%{_prefix}/lib64/
 install -d ${RPM_BUILD_ROOT}/etc/opt/ripple
 install -d ${RPM_BUILD_ROOT}/usr/local/bin
 
@@ -130,9 +133,9 @@ install -D rippled/bld.rippled-reporting/rippled-reporting ${RPM_BUILD_ROOT}%{_b
 install -D ./rippled/cfg/rippled-reporting.cfg ${RPM_BUILD_ROOT}%{_prefix}/etc/rippled-reporting.cfg
 install -D ./rippled/cfg/validators-example.txt ${RPM_BUILD_ROOT}%{_prefix}/etc/validators.txt
 install -D ./rippled/Builds/containers/packaging/rpm/50-rippled-reporting.preset ${RPM_BUILD_ROOT}/usr/lib/systemd/system-preset/50-rippled-reporting.preset
-ln -s %{_prefix}/bin/rippled-reporting ${RPM_BUILD_ROOT}/usr/local/bin/rippled-reporting
+ln -sf %{_prefix}/bin/rippled-reporting ${RPM_BUILD_ROOT}/usr/local/bin/rippled-reporting
-ln -s %{_prefix}/etc/rippled-reporting.cfg ${RPM_BUILD_ROOT}/etc/opt/rippled-reporting/rippled-reporting.cfg
+ln -sf %{_prefix}/etc/rippled-reporting.cfg ${RPM_BUILD_ROOT}/etc/opt/rippled-reporting/rippled-reporting.cfg
-ln -s %{_prefix}/etc/validators.txt ${RPM_BUILD_ROOT}/etc/opt/rippled-reporting/validators.txt
+ln -sf %{_prefix}/etc/validators.txt ${RPM_BUILD_ROOT}/etc/opt/rippled-reporting/validators.txt
 install -d $RPM_BUILD_ROOT/var/log/rippled-reporting
 install -d $RPM_BUILD_ROOT/var/lib/rippled-reporting
 install -D ./rippled/Builds/containers/shared/rippled-reporting.service ${RPM_BUILD_ROOT}/usr/lib/systemd/system/rippled-reporting.service

Builds/containers/ubuntu-builder/Dockerfile

@@ -6,8 +6,15 @@ LABEL git-commit=$GIT_COMMIT
 
 WORKDIR /root
 COPY ubuntu-builder/ubuntu_setup.sh .
+COPY ubuntu-builder/ubuntu_setup2.sh .
+
 RUN ./ubuntu_setup.sh && rm ubuntu_setup.sh
 
+RUN ./ubuntu_setup2.sh && rm ubuntu_setup2.sh
+
+COPY ubuntu-builder/ubuntu_setup3.sh .
+RUN ./ubuntu_setup3.sh && rm ubuntu_setup3.sh
+
 RUN mkdir -m 777 -p /opt/rippled_bld/pkg/
 WORKDIR /opt/rippled_bld/pkg
 

Builds/containers/ubuntu-builder/ubuntu_setup.sh

@@ -6,37 +6,33 @@ set -o xtrace
 
 # Parameters
 
-gcc_version=${GCC_VERSION:-10}
+gcc_version=${GCC_VERSION:-11}
-cmake_version=${CMAKE_VERSION:-3.25.1}
+
-conan_version=${CONAN_VERSION:-1.59}
+export DEBIAN_FRONTEND=noninteractive
 
 apt update
 # Iteratively build the list of packages to install so that we can interleave
 # the lines with comments explaining their inclusion.
 dependencies=''
-# - to identify the Ubuntu version
-dependencies+=' lsb-release'
 # - for add-apt-repository
 dependencies+=' software-properties-common'
 # - to download CMake
 dependencies+=' curl'
 # - to build CMake
 dependencies+=' libssl-dev'
-# - Python headers for Boost.Python
+# - for Python
-dependencies+=' python3-dev'
+dependencies+=' libbz2-dev liblzma-dev libsqlite3-dev'
-# - to install Conan
-dependencies+=' python3-pip'
 # - to download rippled
 dependencies+=' git'
 # - CMake generators (but not CMake itself)
 dependencies+=' make ninja-build'
-apt install --yes ${dependencies}
+apt-get install --yes ${dependencies}
 
 add-apt-repository --yes ppa:ubuntu-toolchain-r/test
-apt install --yes gcc-${gcc_version} g++-${gcc_version} \
+apt-get install --yes gcc-${gcc_version} g++-${gcc_version}
-  debhelper debmake debsums gnupg dh-buildinfo dh-make dh-systemd cmake \
+apt-get install --yes build-essential libssl-dev zlib1g-dev \
-  ninja-build zlib1g-dev make cmake ninja-build autoconf automake \
+libbz2-dev libreadline-dev libsqlite3-dev curl \
-  pkg-config apt-transport-https
+libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
 
 # Give us nice unversioned aliases for gcc and company.
 update-alternatives --install \
@@ -49,28 +45,3 @@ update-alternatives --install \
   --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-${gcc_version} \
   --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-${gcc_version}
 update-alternatives --auto gcc
-
-# Download and unpack CMake.
-cmake_slug="cmake-${cmake_version}"
-curl --location --remote-name \
-  "https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_slug}.tar.gz"
-tar xzf ${cmake_slug}.tar.gz
-rm ${cmake_slug}.tar.gz
-
-# Build and install CMake.
-cd ${cmake_slug}
-./bootstrap --parallel=$(nproc)
-make --jobs $(nproc)
-make install
-cd ..
-rm --recursive --force ${cmake_slug}
-
-# Install Conan.
-pip3 install conan==${conan_version}
-
-conan profile new --detect gcc
-conan profile update settings.compiler=gcc gcc
-conan profile update settings.compiler.version=${gcc_version} gcc
-conan profile update settings.compiler.libcxx=libstdc++11 gcc
-conan profile update env.CC=/usr/bin/gcc gcc
-conan profile update env.CXX=/usr/bin/g++ gcc

Builds/containers/ubuntu-builder/ubuntu_setup2.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o xtrace
+
+# Parameters
+
+gcc_version=${GCC_VERSION:-11}
+cmake_version=${CMAKE_VERSION:-3.25.1}
+cmake_sha256=1c511d09516af493694ed9baf13c55947a36389674d657a2d5e0ccedc6b291d8
+conan_version=${CONAN_VERSION:-1.60}
+
+curl https://pyenv.run | bash
+export PYENV_ROOT="$HOME/.pyenv"
+command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"
+eval "$(pyenv init -)"
+
+pyenv install 3.11.2
+pyenv global 3.11.2
+
+# Download and unpack CMake.
+cmake_slug="cmake-${cmake_version}"
+cmake_archive="${cmake_slug}.tar.gz"
+curl --location --remote-name \
+  "https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_archive}"
+echo "${cmake_sha256}  ${cmake_archive}" | sha256sum --check
+tar -xzf ${cmake_archive}
+rm ${cmake_archive}
+
+# Build and install CMake.
+cd ${cmake_slug}
+./bootstrap --parallel=$(nproc)
+make --jobs $(nproc)
+make install
+cd ..
+rm --recursive --force ${cmake_slug}
+
+# Install Conan.
+pip install --upgrade pip
+pip install conan==${conan_version}
+
+conan profile new --detect gcc
+conan profile update settings.compiler=gcc gcc
+conan profile update settings.compiler.version=${gcc_version} gcc
+conan profile update settings.compiler.libcxx=libstdc++11 gcc
+conan profile update settings.compiler.cppstd=20 gcc
+conan profile update env.CC=/usr/bin/gcc gcc
+conan profile update env.CXX=/usr/bin/g++ gcc

Builds/containers/ubuntu-builder/ubuntu_setup3.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o xtrace
+
+apt-get install --yes build-essential fakeroot devscripts cmake debhelper dh-systemd