From fa1b93bfd879f93e0b8fcf369b86fb18d39390b4 Mon Sep 17 00:00:00 2001 From: Niq Dudfield Date: Fri, 10 Oct 2025 10:57:46 +0700 Subject: [PATCH] build: migrate to conan 2 (#585) Migrates the build system from Conan 1 to Conan 2 --- .github/actions/xahau-ga-build/action.yml | 4 + .../actions/xahau-ga-dependencies/action.yml | 5 +- .github/workflows/xahau-ga-macos.yml | 52 +++++++-- .github/workflows/xahau-ga-nix.yml | 36 ++++-- BUILD.md | 72 ++++++++---- build-core.sh | 27 ++++- conanfile.py | 103 ++++++++++-------- external/wasmedge/conanfile.py | 11 +- release-builder.sh | 60 +++++++--- 9 files changed, 256 insertions(+), 114 deletions(-) diff --git a/.github/actions/xahau-ga-build/action.yml b/.github/actions/xahau-ga-build/action.yml index 3387f0391..66fe45d62 100644 --- a/.github/actions/xahau-ga-build/action.yml +++ b/.github/actions/xahau-ga-build/action.yml @@ -94,6 +94,10 @@ runs: fi # Run CMake configure + # Note: conanfile.py hardcodes 'build/generators' as the output path. + # If we're in a 'build' folder, Conan detects this and uses just 'generators/' + # If we're in '.build' (non-standard), Conan adds the full 'build/generators/' + # So we get: .build/build/generators/ with our non-standard folder name cmake .. \ -G "${{ inputs.generator }}" \ $CCACHE_ARGS \ diff --git a/.github/actions/xahau-ga-dependencies/action.yml b/.github/actions/xahau-ga-dependencies/action.yml index d295e20eb..cb14e4e57 100644 --- a/.github/actions/xahau-ga-dependencies/action.yml +++ b/.github/actions/xahau-ga-dependencies/action.yml @@ -78,8 +78,9 @@ runs: - name: Export custom recipes shell: bash run: | - conan export external/snappy snappy/1.1.10@xahaud/stable - conan export external/soci soci/4.0.3@xahaud/stable + conan export external/snappy --version 1.1.10 --user xahaud --channel stable + conan export external/soci --version 4.0.3 --user xahaud --channel stable + conan export external/wasmedge --version 0.11.2 --user xahaud --channel stable - name: Install dependencies shell: bash diff --git a/.github/workflows/xahau-ga-macos.yml b/.github/workflows/xahau-ga-macos.yml index efb1a2001..66c7e6877 100644 --- a/.github/workflows/xahau-ga-macos.yml +++ b/.github/workflows/xahau-ga-macos.yml @@ -32,9 +32,9 @@ jobs: - name: Install Conan run: | - brew install conan@1 - # Add Conan 1 to the PATH for this job - echo "$(brew --prefix conan@1)/bin" >> $GITHUB_PATH + brew install conan + # Verify Conan 2 is installed + conan --version - name: Install Coreutils run: | @@ -60,12 +60,20 @@ jobs: - name: Install CMake run: | - if which cmake > /dev/null 2>&1; then - echo "cmake executable exists" - cmake --version - else - brew install cmake - fi + # Install CMake 3.x to match local dev environments + # With Conan 2 and the policy args passed to CMake, newer versions + # can have issues with dependencies that require cmake_minimum_required < 3.5 + brew uninstall cmake --ignore-dependencies 2>/dev/null || true + + # Download and install CMake 3.31.7 directly + curl -L https://github.com/Kitware/CMake/releases/download/v3.31.7/cmake-3.31.7-macos-universal.tar.gz -o cmake.tar.gz + tar -xzf cmake.tar.gz + + # Move the entire CMake.app to /Applications + sudo mv cmake-3.31.7-macos-universal/CMake.app /Applications/ + + echo "/Applications/CMake.app/Contents/bin" >> $GITHUB_PATH + /Applications/CMake.app/Contents/bin/cmake --version - name: Install ccache run: brew install ccache @@ -92,8 +100,30 @@ jobs: - name: Configure Conan run: | - conan profile new default --detect || true # Ignore error if profile exists - conan profile update settings.compiler.cppstd=20 default + # Create the default profile directory if it doesn't exist + mkdir -p ~/.conan2/profiles + + # Detect compiler version + COMPILER_VERSION=$(clang --version | grep -oE 'version [0-9]+' | grep -oE '[0-9]+') + + # Create profile with our specific settings + cat > ~/.conan2/profiles/default <=2.0" - name: Configure ccache uses: ./.github/actions/xahau-configure-ccache @@ -54,18 +54,30 @@ jobs: - name: Configure Conan run: | - conan profile new default --detect || true # Ignore error if profile exists - conan profile update settings.compiler.cppstd=20 default - conan profile update settings.compiler=${{ matrix.compiler }} default - conan profile update settings.compiler.libcxx=libstdc++11 default - conan profile update env.CC=/usr/bin/${{ matrix.cc }} default - conan profile update env.CXX=/usr/bin/${{ matrix.cxx }} default - conan profile update conf.tools.build:compiler_executables='{"c": "/usr/bin/${{ matrix.cc }}", "cpp": "/usr/bin/${{ matrix.cxx }}"}' default + # Create the default profile directory if it doesn't exist + mkdir -p ~/.conan2/profiles + + # Create profile with our specific settings + cat > ~/.conan2/profiles/default <` header. @@ -65,13 +65,24 @@ can't build earlier Boost versions. 1. (Optional) If you've never used Conan, use autodetect to set up a default profile. ``` - conan profile new default --detect + conan profile detect --force ``` 2. Update the compiler settings. + For Conan 2, you can edit the profile directly at `~/.conan2/profiles/default`, + or use the Conan CLI. Ensure C++20 is set: + ``` - conan profile update settings.compiler.cppstd=20 default + conan profile show + ``` + + Look for `compiler.cppstd=20` in the output. If it's not set, edit the profile: + + ``` + # Edit ~/.conan2/profiles/default and ensure these settings exist: + [settings] + compiler.cppstd=20 ``` Linux developers will commonly have a default Conan [profile][] that compiles @@ -80,7 +91,9 @@ can't build earlier Boost versions. then you will need to choose the `libstdc++11` ABI. ``` - conan profile update settings.compiler.libcxx=libstdc++11 default + # In ~/.conan2/profiles/default, ensure: + [settings] + compiler.libcxx=libstdc++11 ``` On Windows, you should use the x64 native build tools. @@ -91,7 +104,9 @@ can't build earlier Boost versions. architecture. ``` - conan profile update settings.arch=x86_64 default + # In ~/.conan2/profiles/default, ensure: + [settings] + arch=x86_64 ``` 3. (Optional) If you have multiple compilers installed on your platform, @@ -100,16 +115,18 @@ can't build earlier Boost versions. in the generated CMake toolchain file. ``` - conan profile update 'conf.tools.build:compiler_executables={"c": "", "cpp": ""}' default + # In ~/.conan2/profiles/default, add under [conf] section: + [conf] + tools.build:compiler_executables={"c": "", "cpp": ""} ``` - It should choose the compiler for dependencies as well, - but not all of them have a Conan recipe that respects this setting (yet). - For the rest, you can set these environment variables: + For setting environment variables for dependencies: ``` - conan profile update env.CC= default - conan profile update env.CXX= default + # In ~/.conan2/profiles/default, add under [buildenv] section: + [buildenv] + CC= + CXX= ``` 4. Export our [Conan recipe for Snappy](./external/snappy). @@ -117,14 +134,20 @@ can't build earlier Boost versions. which allows you to statically link it with GCC, if you want. ``` - conan export external/snappy snappy/1.1.10@xahaud/stable + conan export external/snappy --version 1.1.10 --user xahaud --channel stable ``` 5. Export our [Conan recipe for SOCI](./external/soci). It patches their CMake to correctly import its dependencies. ``` - conan export external/soci soci/4.0.3@xahaud/stable + conan export external/soci --version 4.0.3 --user xahaud --channel stable + ``` + +6. Export our [Conan recipe for WasmEdge](./external/wasmedge). + + ``` + conan export external/wasmedge --version 0.11.2 --user xahaud --channel stable ``` ### Build and Test @@ -259,23 +282,26 @@ and can be helpful for detecting `#include` omissions. If you have trouble building dependencies after changing Conan settings, try removing the Conan cache. +For Conan 2: ``` -rm -rf ~/.conan/data +rm -rf ~/.conan2/p +``` + +Or clear the entire Conan 2 cache: +``` +conan cache clean "*" ``` -### no std::result_of +### macOS compilation with Apple Clang 17+ -If your compiler version is recent enough to have removed `std::result_of` as -part of C++20, e.g. Apple Clang 15.0, then you might need to add a preprocessor -definition to your build. +If you're on macOS with Apple Clang 17 or newer, you need to add a compiler flag to work around a compilation error in gRPC dependencies. + +Edit `~/.conan2/profiles/default` and add under the `[conf]` section: ``` -conan profile update 'options.boost:extra_b2_flags="define=BOOST_ASIO_HAS_STD_INVOKE_RESULT"' default -conan profile update 'env.CFLAGS="-DBOOST_ASIO_HAS_STD_INVOKE_RESULT"' default -conan profile update 'env.CXXFLAGS="-DBOOST_ASIO_HAS_STD_INVOKE_RESULT"' default -conan profile update 'conf.tools.build:cflags+=["-DBOOST_ASIO_HAS_STD_INVOKE_RESULT"]' default -conan profile update 'conf.tools.build:cxxflags+=["-DBOOST_ASIO_HAS_STD_INVOKE_RESULT"]' default +[conf] +tools.build:cxxflags=["-Wno-missing-template-arg-list-after-template-kw"] ``` diff --git a/build-core.sh b/build-core.sh index 7b1eb78dc..ce78c7625 100755 --- a/build-core.sh +++ b/build-core.sh @@ -12,6 +12,13 @@ echo "-- GITHUB_REPOSITORY: $1" echo "-- GITHUB_SHA: $2" echo "-- GITHUB_RUN_NUMBER: $4" +# Use mounted filesystem for temp files to avoid container space limits +export TMPDIR=/io/tmp +export TEMP=/io/tmp +export TMP=/io/tmp +mkdir -p /io/tmp +echo "=== Using temp directory: /io/tmp ===" + umask 0000; cd /io/ && @@ -43,10 +50,17 @@ export LDFLAGS="-static-libstdc++" git config --global --add safe.directory /io && git checkout src/ripple/protocol/impl/BuildInfo.cpp && sed -i s/\"0.0.0\"/\"$(date +%Y).$(date +%-m).$(date +%-d)-$(git rev-parse --abbrev-ref HEAD)$(if [ -n "$4" ]; then echo "+$4"; fi)\"/g src/ripple/protocol/impl/BuildInfo.cpp && -conan export external/snappy snappy/1.1.10@xahaud/stable && -conan export external/soci soci/4.0.3@xahaud/stable && +conan export external/snappy --version 1.1.10 --user xahaud --channel stable && +conan export external/soci --version 4.0.3 --user xahaud --channel stable && +conan export external/wasmedge --version 0.11.2 --user xahaud --channel stable && cd release-build && -conan install .. --output-folder . --build missing --settings build_type=$BUILD_TYPE && +# Install dependencies - tool_requires in conanfile.py handles glibc 2.28 compatibility +# for build tools (protoc, grpc plugins, b2) in HBB environment +# The tool_requires('b2/5.3.2') in conanfile.py should force b2 to build from source +# with the correct toolchain, avoiding the GLIBCXX_3.4.29 issue +echo "=== Installing dependencies ===" && +conan install .. --output-folder . --build missing --settings build_type=$BUILD_TYPE \ + -o with_wasmedge=False -o tool_requires_b2=True && cmake .. -G Ninja \ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ @@ -56,10 +70,13 @@ cmake .. -G Ninja \ -Dxrpld=TRUE \ -Dtests=TRUE && ccache -z && -ninja -j $3 && +ninja -j $3 && echo "=== Re-running final link with verbose output ===" && rm -f rippled && ninja -v rippled && ccache -s && -strip -s rippled && +strip -s rippled && mv rippled xahaud && +echo "=== Full ldd output ===" && +ldd xahaud && +echo "=== Running libcheck ===" && libcheck xahaud && echo "Build host: `hostname`" > release.info && echo "Build date: `date`" >> release.info && diff --git a/conanfile.py b/conanfile.py index 0a5bc0b4a..4b98b84db 100644 --- a/conanfile.py +++ b/conanfile.py @@ -21,22 +21,20 @@ class Xrpl(ConanFile): 'static': [True, False], 'tests': [True, False], 'unity': [True, False], + 'with_wasmedge': [True, False], + 'tool_requires_b2': [True, False], } requires = [ - 'boost/1.86.0', 'date/3.0.1', 'libarchive/3.6.0', - 'lz4/1.9.3', + 'lz4/1.9.4', 'grpc/1.50.1', 'nudb/2.0.8', 'openssl/1.1.1u', - 'protobuf/3.21.9', - 'snappy/1.1.10@xahaud/stable', + 'protobuf/3.21.12', 'soci/4.0.3@xahaud/stable', - 'sqlite3/3.42.0', - 'zlib/1.2.13', - 'wasmedge/0.11.2', + 'zlib/1.3.1', ] default_options = { @@ -50,42 +48,44 @@ class Xrpl(ConanFile): 'static': True, 'tests': True, 'unity': False, + 'with_wasmedge': True, + 'tool_requires_b2': False, - 'cassandra-cpp-driver:shared': False, - 'date:header_only': True, - 'grpc:shared': False, - 'grpc:secure': True, - 'libarchive:shared': False, - 'libarchive:with_acl': False, - 'libarchive:with_bzip2': False, - 'libarchive:with_cng': False, - 'libarchive:with_expat': False, - 'libarchive:with_iconv': False, - 'libarchive:with_libxml2': False, - 'libarchive:with_lz4': True, - 'libarchive:with_lzma': False, - 'libarchive:with_lzo': False, - 'libarchive:with_nettle': False, - 'libarchive:with_openssl': False, - 'libarchive:with_pcreposix': False, - 'libarchive:with_xattr': False, - 'libarchive:with_zlib': False, - 'libpq:shared': False, - 'lz4:shared': False, - 'openssl:shared': False, - 'protobuf:shared': False, - 'protobuf:with_zlib': True, - 'rocksdb:enable_sse': False, - 'rocksdb:lite': False, - 'rocksdb:shared': False, - 'rocksdb:use_rtti': True, - 'rocksdb:with_jemalloc': False, - 'rocksdb:with_lz4': True, - 'rocksdb:with_snappy': True, - 'snappy:shared': False, - 'soci:shared': False, - 'soci:with_sqlite3': True, - 'soci:with_boost': True, + 'cassandra-cpp-driver/*:shared': False, + 'date/*:header_only': True, + 'grpc/*:shared': False, + 'grpc/*:secure': True, + 'libarchive/*:shared': False, + 'libarchive/*:with_acl': False, + 'libarchive/*:with_bzip2': False, + 'libarchive/*:with_cng': False, + 'libarchive/*:with_expat': False, + 'libarchive/*:with_iconv': False, + 'libarchive/*:with_libxml2': False, + 'libarchive/*:with_lz4': True, + 'libarchive/*:with_lzma': False, + 'libarchive/*:with_lzo': False, + 'libarchive/*:with_nettle': False, + 'libarchive/*:with_openssl': False, + 'libarchive/*:with_pcreposix': False, + 'libarchive/*:with_xattr': False, + 'libarchive/*:with_zlib': False, + 'libpq/*:shared': False, + 'lz4/*:shared': False, + 'openssl/*:shared': False, + 'protobuf/*:shared': False, + 'protobuf/*:with_zlib': True, + 'rocksdb/*:enable_sse': False, + 'rocksdb/*:lite': False, + 'rocksdb/*:shared': False, + 'rocksdb/*:use_rtti': True, + 'rocksdb/*:with_jemalloc': False, + 'rocksdb/*:with_lz4': True, + 'rocksdb/*:with_snappy': True, + 'snappy/*:shared': False, + 'soci/*:shared': False, + 'soci/*:with_sqlite3': True, + 'soci/*:with_boost': True, } def set_version(self): @@ -96,11 +96,28 @@ class Xrpl(ConanFile): match = next(m for m in matches if m) self.version = match.group(1) + def build_requirements(self): + # These provide build tools (protoc, grpc plugins) that run during build + self.tool_requires('protobuf/3.21.12') + self.tool_requires('grpc/1.50.1') + # Explicitly require b2 (e.g. for building from source for glibc compatibility) + if self.options.tool_requires_b2: + self.tool_requires('b2/5.3.2') + def configure(self): if self.settings.compiler == 'apple-clang': - self.options['boost'].visibility = 'global' + self.options['boost/*'].visibility = 'global' def requirements(self): + # Force sqlite3 version to avoid conflicts with soci + self.requires('sqlite3/3.42.0', override=True) + # Force our custom snappy build for all dependencies + self.requires('snappy/1.1.10@xahaud/stable', override=True) + # Force boost version for all dependencies to avoid conflicts + self.requires('boost/1.86.0', override=True) + + if self.options.with_wasmedge: + self.requires('wasmedge/0.11.2@xahaud/stable') if self.options.jemalloc: self.requires('jemalloc/5.2.1') if self.options.reporting: diff --git a/external/wasmedge/conanfile.py b/external/wasmedge/conanfile.py index 4b7c42050..cefda286d 100644 --- a/external/wasmedge/conanfile.py +++ b/external/wasmedge/conanfile.py @@ -38,8 +38,15 @@ class WasmedgeConan(ConanFile): raise ConanInvalidConfiguration("Binaries for this combination of version/os/arch/compiler are not available") def package_id(self): - del self.info.settings.compiler.version - self.info.settings.compiler = self._compiler_alias + # Make binary compatible across compiler versions (since we're downloading prebuilt) + self.info.settings.rm_safe("compiler.version") + # Group compilers by their binary compatibility + # Note: We must use self.info.settings here, not self.settings (forbidden in Conan 2) + compiler_name = str(self.info.settings.compiler) + if compiler_name in ["Visual Studio", "msvc"]: + self.info.settings.compiler = "Visual Studio" + else: + self.info.settings.compiler = "gcc" def build(self): # This is packaging binaries so the download needs to be in build diff --git a/release-builder.sh b/release-builder.sh index f2a64a673..672e68042 100755 --- a/release-builder.sh +++ b/release-builder.sh @@ -1,9 +1,11 @@ -#!/bin/bash +#!/bin/bash # We use set -e and bash with -u to bail on first non zero exit code of any # processes launched or upon any unbound variable. # We use set -x to print commands before running them to help with # debugging. +set -ex + echo "START BUILDING (HOST)" echo "Cleaning previously built binary" @@ -90,29 +92,37 @@ RUN /hbb_exe/activate-exec bash -c "dnf install -y epel-release && \ llvm14-static llvm14-devel && \ dnf clean all" -# Install Conan and CMake -RUN /hbb_exe/activate-exec pip3 install "conan==1.66.0" && \ +# Install Conan 2 and CMake +RUN /hbb_exe/activate-exec pip3 install "conan>=2.0,<3.0" && \ /hbb_exe/activate-exec wget -q https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz -O cmake.tar.gz && \ mkdir cmake && \ tar -xzf cmake.tar.gz --strip-components=1 -C cmake && \ rm cmake.tar.gz -# Install Boost 1.86.0 -RUN /hbb_exe/activate-exec bash -c "cd /tmp && \ +# Dual Boost configuration in HBB environment: +# - Manual Boost in /usr/local (minimal: for WasmEdge which is pre-built in Docker) +# - Conan Boost (full: for the application and all dependencies via toolchain) +# +# Install minimal Boost 1.86.0 for WasmEdge only (filesystem and its dependencies) +# The main application will use Conan-provided Boost for all other components +# IMPORTANT: Understanding Boost linking options: +# - link=static: Creates static Boost libraries (.a files) instead of shared (.so files) +# - runtime-link=shared: Links Boost libraries against shared libc (glibc) +# WasmEdge only needs boost::filesystem and boost::system +RUN /hbb_exe/activate-exec bash -c "echo 'Boost cache bust: v5-minimal' && \ + rm -rf /usr/local/lib/libboost* /usr/local/include/boost && \ + cd /tmp && \ wget -q https://archives.boost.io/release/1.86.0/source/boost_1_86_0.tar.gz -O boost.tar.gz && \ mkdir boost && \ tar -xzf boost.tar.gz --strip-components=1 -C boost && \ cd boost && \ ./bootstrap.sh && \ - ./b2 link=static -j${BUILD_CORES} && \ - ./b2 install && \ + ./b2 install \ + link=static runtime-link=shared -j${BUILD_CORES} \ + --with-filesystem --with-system && \ cd /tmp && \ rm -rf boost boost.tar.gz" -ENV BOOST_ROOT=/usr/local/src/boost_1_86_0 -ENV Boost_LIBRARY_DIRS=/usr/local/lib -ENV BOOST_INCLUDEDIR=/usr/local/src/boost_1_86_0 - ENV CMAKE_EXE_LINKER_FLAGS="-static-libstdc++" ENV LLVM_DIR=/usr/lib64/llvm14/lib/cmake/llvm @@ -155,6 +165,10 @@ RUN cd /tmp && \ cd build && \ /hbb_exe/activate-exec bash -c "source /opt/rh/gcc-toolset-11/enable && \ ln -sf /opt/rh/gcc-toolset-11/root/usr/bin/ar /usr/bin/ar && \ + ln -sf /opt/rh/gcc-toolset-11/root/usr/bin/ranlib /usr/bin/ranlib && \ + echo '=== Binutils version check ===' && \ + ar --version | head -1 && \ + ranlib --version | head -1 && \ cmake .. \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=/usr/local \ @@ -176,14 +190,28 @@ RUN cd /tmp && \ # Set environment variables ENV PATH=/usr/local/bin:$PATH -# Configure ccache and Conan +# Configure ccache and Conan 2 +# NOTE: Using echo commands instead of heredocs because heredocs in Docker RUN commands are finnicky RUN /hbb_exe/activate-exec bash -c "ccache -M 10G && \ ccache -o cache_dir=/cache/ccache && \ ccache -o compiler_check=content && \ - conan config set storage.path=/cache/conan && \ - (conan profile new default --detect || true) && \ - conan profile update settings.compiler.libcxx=libstdc++11 default && \ - conan profile update settings.compiler.cppstd=20 default" + mkdir -p ~/.conan2 /cache/conan2 /cache/conan2_download /cache/conan2_sources && \ + echo 'core.cache:storage_path=/cache/conan2' > ~/.conan2/global.conf && \ + echo 'core.download:download_cache=/cache/conan2_download' >> ~/.conan2/global.conf && \ + echo 'core.sources:download_cache=/cache/conan2_sources' >> ~/.conan2/global.conf && \ + conan profile detect --force && \ + echo '[settings]' > ~/.conan2/profiles/default && \ + echo 'arch=x86_64' >> ~/.conan2/profiles/default && \ + echo 'build_type=Release' >> ~/.conan2/profiles/default && \ + echo 'compiler=gcc' >> ~/.conan2/profiles/default && \ + echo 'compiler.cppstd=20' >> ~/.conan2/profiles/default && \ + echo 'compiler.libcxx=libstdc++11' >> ~/.conan2/profiles/default && \ + echo 'compiler.version=11' >> ~/.conan2/profiles/default && \ + echo 'os=Linux' >> ~/.conan2/profiles/default && \ + echo '' >> ~/.conan2/profiles/default && \ + echo '[conf]' >> ~/.conan2/profiles/default && \ + echo '# Force building from source for packages with binary compatibility issues' >> ~/.conan2/profiles/default && \ + echo '*:tools.system.package_manager:mode=build' >> ~/.conan2/profiles/default" DOCKERFILE_EOF )