From e4e0c95d3c25f549c3759bd2eb3afc56e81127d3 Mon Sep 17 00:00:00 2001 From: Nicholas Dudfield Date: Mon, 22 Sep 2025 18:15:11 +0700 Subject: [PATCH] build: migrate to Conan 2 Migrates the build system from Conan 1 to Conan 2: - Update all conan commands to v2 syntax (export, install, profile) - Convert profiles to Conan 2 INI format with [settings] and [conf] sections - Add tool_requires() for build tools (protobuf, grpc, b2) - Fix wasmedge/0.11.2 recipe for Conan 2 compatibility (package_id syntax) - Update CI workflows for all platforms (Linux, macOS, Docker/HBB) - Document dual Boost setup (manual for WasmEdge, Conan for app) - Force glibc-compatible builds via source compilation in HBB - Ensure dependency version consistency with override=True Maintains full backwards compatibility while preparing for Conan 1 EOL. --- .github/actions/xahau-ga-build/action.yml | 4 + .../actions/xahau-ga-dependencies/action.yml | 5 +- .github/workflows/xahau-ga-macos.yml | 52 ++++++-- .github/workflows/xahau-ga-nix.yml | 36 ++++-- build-core.sh | 24 +++- build-full.sh | 1 + conanfile.py | 118 +++++++++++------- external/wasmedge/conanfile.py | 11 +- release-builder.sh | 56 ++++++--- 9 files changed, 218 insertions(+), 89 deletions(-) diff --git a/.github/actions/xahau-ga-build/action.yml b/.github/actions/xahau-ga-build/action.yml index 3387f0391..66fe45d62 100644 --- a/.github/actions/xahau-ga-build/action.yml +++ b/.github/actions/xahau-ga-build/action.yml @@ -94,6 +94,10 @@ runs: fi # Run CMake configure + # Note: conanfile.py hardcodes 'build/generators' as the output path. + # If we're in a 'build' folder, Conan detects this and uses just 'generators/' + # If we're in '.build' (non-standard), Conan adds the full 'build/generators/' + # So we get: .build/build/generators/ with our non-standard folder name cmake .. \ -G "${{ inputs.generator }}" \ $CCACHE_ARGS \ diff --git a/.github/actions/xahau-ga-dependencies/action.yml b/.github/actions/xahau-ga-dependencies/action.yml index d295e20eb..cb14e4e57 100644 --- a/.github/actions/xahau-ga-dependencies/action.yml +++ b/.github/actions/xahau-ga-dependencies/action.yml @@ -78,8 +78,9 @@ runs: - name: Export custom recipes shell: bash run: | - conan export external/snappy snappy/1.1.10@xahaud/stable - conan export external/soci soci/4.0.3@xahaud/stable + conan export external/snappy --version 1.1.10 --user xahaud --channel stable + conan export external/soci --version 4.0.3 --user xahaud --channel stable + conan export external/wasmedge --version 0.11.2 --user xahaud --channel stable - name: Install dependencies shell: bash diff --git a/.github/workflows/xahau-ga-macos.yml b/.github/workflows/xahau-ga-macos.yml index efb1a2001..66c7e6877 100644 --- a/.github/workflows/xahau-ga-macos.yml +++ b/.github/workflows/xahau-ga-macos.yml @@ -32,9 +32,9 @@ jobs: - name: Install Conan run: | - brew install conan@1 - # Add Conan 1 to the PATH for this job - echo "$(brew --prefix conan@1)/bin" >> $GITHUB_PATH + brew install conan + # Verify Conan 2 is installed + conan --version - name: Install Coreutils run: | @@ -60,12 +60,20 @@ jobs: - name: Install CMake run: | - if which cmake > /dev/null 2>&1; then - echo "cmake executable exists" - cmake --version - else - brew install cmake - fi + # Install CMake 3.x to match local dev environments + # With Conan 2 and the policy args passed to CMake, newer versions + # can have issues with dependencies that require cmake_minimum_required < 3.5 + brew uninstall cmake --ignore-dependencies 2>/dev/null || true + + # Download and install CMake 3.31.7 directly + curl -L https://github.com/Kitware/CMake/releases/download/v3.31.7/cmake-3.31.7-macos-universal.tar.gz -o cmake.tar.gz + tar -xzf cmake.tar.gz + + # Move the entire CMake.app to /Applications + sudo mv cmake-3.31.7-macos-universal/CMake.app /Applications/ + + echo "/Applications/CMake.app/Contents/bin" >> $GITHUB_PATH + /Applications/CMake.app/Contents/bin/cmake --version - name: Install ccache run: brew install ccache @@ -92,8 +100,30 @@ jobs: - name: Configure Conan run: | - conan profile new default --detect || true # Ignore error if profile exists - conan profile update settings.compiler.cppstd=20 default + # Create the default profile directory if it doesn't exist + mkdir -p ~/.conan2/profiles + + # Detect compiler version + COMPILER_VERSION=$(clang --version | grep -oE 'version [0-9]+' | grep -oE '[0-9]+') + + # Create profile with our specific settings + cat > ~/.conan2/profiles/default <=2.0" - name: Configure ccache uses: ./.github/actions/xahau-configure-ccache @@ -54,18 +54,30 @@ jobs: - name: Configure Conan run: | - conan profile new default --detect || true # Ignore error if profile exists - conan profile update settings.compiler.cppstd=20 default - conan profile update settings.compiler=${{ matrix.compiler }} default - conan profile update settings.compiler.libcxx=libstdc++11 default - conan profile update env.CC=/usr/bin/${{ matrix.cc }} default - conan profile update env.CXX=/usr/bin/${{ matrix.cxx }} default - conan profile update conf.tools.build:compiler_executables='{"c": "/usr/bin/${{ matrix.cc }}", "cpp": "/usr/bin/${{ matrix.cxx }}"}' default + # Create the default profile directory if it doesn't exist + mkdir -p ~/.conan2/profiles + + # Create profile with our specific settings + cat > ~/.conan2/profiles/default < release.info && echo "Build date: `date`" >> release.info && diff --git a/build-full.sh b/build-full.sh index 9723da654..bbe614613 100644 --- a/build-full.sh +++ b/build-full.sh @@ -69,6 +69,7 @@ source /opt/rh/gcc-toolset-11/enable export PATH=/usr/local/bin:$PATH export CC='ccache gcc' && export CXX='ccache g++' && +export HBB_RELEASE_BUILD=1 && echo "-- Build Rippled --" && pwd && diff --git a/conanfile.py b/conanfile.py index 0a5bc0b4a..282c8bf90 100644 --- a/conanfile.py +++ b/conanfile.py @@ -23,22 +23,27 @@ class Xrpl(ConanFile): 'unity': [True, False], } + import os + + # Check if we're in HBB release build environment + IS_HBB_BUILD = os.getenv('HBB_RELEASE_BUILD') == '1' + requires = [ - 'boost/1.86.0', 'date/3.0.1', 'libarchive/3.6.0', - 'lz4/1.9.3', + 'lz4/1.9.4', 'grpc/1.50.1', 'nudb/2.0.8', 'openssl/1.1.1u', - 'protobuf/3.21.9', - 'snappy/1.1.10@xahaud/stable', + 'protobuf/3.21.12', 'soci/4.0.3@xahaud/stable', - 'sqlite3/3.42.0', - 'zlib/1.2.13', - 'wasmedge/0.11.2', + 'zlib/1.3.1', ] + # Only include deps that aren't manually installed in HBB + if not IS_HBB_BUILD: + requires.append('wasmedge/0.11.2@xahaud/stable') + default_options = { 'assertions': False, 'coverage': False, @@ -51,41 +56,41 @@ class Xrpl(ConanFile): 'tests': True, 'unity': False, - 'cassandra-cpp-driver:shared': False, - 'date:header_only': True, - 'grpc:shared': False, - 'grpc:secure': True, - 'libarchive:shared': False, - 'libarchive:with_acl': False, - 'libarchive:with_bzip2': False, - 'libarchive:with_cng': False, - 'libarchive:with_expat': False, - 'libarchive:with_iconv': False, - 'libarchive:with_libxml2': False, - 'libarchive:with_lz4': True, - 'libarchive:with_lzma': False, - 'libarchive:with_lzo': False, - 'libarchive:with_nettle': False, - 'libarchive:with_openssl': False, - 'libarchive:with_pcreposix': False, - 'libarchive:with_xattr': False, - 'libarchive:with_zlib': False, - 'libpq:shared': False, - 'lz4:shared': False, - 'openssl:shared': False, - 'protobuf:shared': False, - 'protobuf:with_zlib': True, - 'rocksdb:enable_sse': False, - 'rocksdb:lite': False, - 'rocksdb:shared': False, - 'rocksdb:use_rtti': True, - 'rocksdb:with_jemalloc': False, - 'rocksdb:with_lz4': True, - 'rocksdb:with_snappy': True, - 'snappy:shared': False, - 'soci:shared': False, - 'soci:with_sqlite3': True, - 'soci:with_boost': True, + 'cassandra-cpp-driver/*:shared': False, + 'date/*:header_only': True, + 'grpc/*:shared': False, + 'grpc/*:secure': True, + 'libarchive/*:shared': False, + 'libarchive/*:with_acl': False, + 'libarchive/*:with_bzip2': False, + 'libarchive/*:with_cng': False, + 'libarchive/*:with_expat': False, + 'libarchive/*:with_iconv': False, + 'libarchive/*:with_libxml2': False, + 'libarchive/*:with_lz4': True, + 'libarchive/*:with_lzma': False, + 'libarchive/*:with_lzo': False, + 'libarchive/*:with_nettle': False, + 'libarchive/*:with_openssl': False, + 'libarchive/*:with_pcreposix': False, + 'libarchive/*:with_xattr': False, + 'libarchive/*:with_zlib': False, + 'libpq/*:shared': False, + 'lz4/*:shared': False, + 'openssl/*:shared': False, + 'protobuf/*:shared': False, + 'protobuf/*:with_zlib': True, + 'rocksdb/*:enable_sse': False, + 'rocksdb/*:lite': False, + 'rocksdb/*:shared': False, + 'rocksdb/*:use_rtti': True, + 'rocksdb/*:with_jemalloc': False, + 'rocksdb/*:with_lz4': True, + 'rocksdb/*:with_snappy': True, + 'snappy/*:shared': False, + 'soci/*:shared': False, + 'soci/*:with_sqlite3': True, + 'soci/*:with_boost': True, } def set_version(self): @@ -96,11 +101,38 @@ class Xrpl(ConanFile): match = next(m for m in matches if m) self.version = match.group(1) + def build_requirements(self): + # These provide build tools (protoc, grpc plugins) that run during build + self.tool_requires('protobuf/3.21.12') + self.tool_requires('grpc/1.50.1') + # Force b2 to build from source for glibc compatibility in HBB builds + if self.IS_HBB_BUILD: + self.tool_requires('b2/5.3.2') + def configure(self): if self.settings.compiler == 'apple-clang': - self.options['boost'].visibility = 'global' + self.options['boost/*'].visibility = 'global' + + # HBB builds have both manual boost (for WasmEdge) and Conan boost (for the app and deps like nudb/soci) + # The linker will deduplicate symbols from both versions + def requirements(self): + # Force sqlite3 version to avoid conflicts with soci + self.requires('sqlite3/3.42.0', override=True) + # Force our custom snappy build for all dependencies + self.requires('snappy/1.1.10@xahaud/stable', override=True) + + # Force boost version for all dependencies to avoid conflicts + # NOTE: HBB builds have both manual boost (for WasmEdge) and Conan boost because: + # - WasmEdge is manually built and linked against minimal system boost (filesystem + system) + # - The application itself requires boost (chrono, coroutine, thread, etc.) + # - Dependencies like nudb/soci also require boost components + # - The application and deps use Conan's boost (via CMake toolchain) + # - Both are boost 1.86.0, so symbols are identical (no version conflicts) + # - Static linking means symbols are resolved at link time, not runtime + self.requires('boost/1.86.0', override=True) + if self.options.jemalloc: self.requires('jemalloc/5.2.1') if self.options.reporting: diff --git a/external/wasmedge/conanfile.py b/external/wasmedge/conanfile.py index 4b7c42050..cefda286d 100644 --- a/external/wasmedge/conanfile.py +++ b/external/wasmedge/conanfile.py @@ -38,8 +38,15 @@ class WasmedgeConan(ConanFile): raise ConanInvalidConfiguration("Binaries for this combination of version/os/arch/compiler are not available") def package_id(self): - del self.info.settings.compiler.version - self.info.settings.compiler = self._compiler_alias + # Make binary compatible across compiler versions (since we're downloading prebuilt) + self.info.settings.rm_safe("compiler.version") + # Group compilers by their binary compatibility + # Note: We must use self.info.settings here, not self.settings (forbidden in Conan 2) + compiler_name = str(self.info.settings.compiler) + if compiler_name in ["Visual Studio", "msvc"]: + self.info.settings.compiler = "Visual Studio" + else: + self.info.settings.compiler = "gcc" def build(self): # This is packaging binaries so the download needs to be in build diff --git a/release-builder.sh b/release-builder.sh index f2a64a673..9918a07b9 100755 --- a/release-builder.sh +++ b/release-builder.sh @@ -90,29 +90,37 @@ RUN /hbb_exe/activate-exec bash -c "dnf install -y epel-release && \ llvm14-static llvm14-devel && \ dnf clean all" -# Install Conan and CMake -RUN /hbb_exe/activate-exec pip3 install "conan==1.66.0" && \ +# Install Conan 2 and CMake +RUN /hbb_exe/activate-exec pip3 install "conan>=2.0" && \ /hbb_exe/activate-exec wget -q https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz -O cmake.tar.gz && \ mkdir cmake && \ tar -xzf cmake.tar.gz --strip-components=1 -C cmake && \ rm cmake.tar.gz -# Install Boost 1.86.0 -RUN /hbb_exe/activate-exec bash -c "cd /tmp && \ +# Dual Boost configuration in HBB environment: +# - Manual Boost in /usr/local (minimal: for WasmEdge which is pre-built in Docker) +# - Conan Boost (full: for the application and all dependencies via toolchain) +# +# Install minimal Boost 1.86.0 for WasmEdge only (filesystem and its dependencies) +# The main application will use Conan-provided Boost for all other components +# IMPORTANT: Understanding Boost linking options: +# - link=static: Creates static Boost libraries (.a files) instead of shared (.so files) +# - runtime-link=shared: Links Boost libraries against shared libc (glibc) +# WasmEdge only needs boost::filesystem and boost::system +RUN /hbb_exe/activate-exec bash -c "echo 'Boost cache bust: v5-minimal' && \ + rm -rf /usr/local/lib/libboost* /usr/local/include/boost && \ + cd /tmp && \ wget -q https://archives.boost.io/release/1.86.0/source/boost_1_86_0.tar.gz -O boost.tar.gz && \ mkdir boost && \ tar -xzf boost.tar.gz --strip-components=1 -C boost && \ cd boost && \ ./bootstrap.sh && \ - ./b2 link=static -j${BUILD_CORES} && \ - ./b2 install && \ + ./b2 install \ + link=static runtime-link=shared -j${BUILD_CORES} \ + --with-filesystem --with-system && \ cd /tmp && \ rm -rf boost boost.tar.gz" -ENV BOOST_ROOT=/usr/local/src/boost_1_86_0 -ENV Boost_LIBRARY_DIRS=/usr/local/lib -ENV BOOST_INCLUDEDIR=/usr/local/src/boost_1_86_0 - ENV CMAKE_EXE_LINKER_FLAGS="-static-libstdc++" ENV LLVM_DIR=/usr/lib64/llvm14/lib/cmake/llvm @@ -155,6 +163,10 @@ RUN cd /tmp && \ cd build && \ /hbb_exe/activate-exec bash -c "source /opt/rh/gcc-toolset-11/enable && \ ln -sf /opt/rh/gcc-toolset-11/root/usr/bin/ar /usr/bin/ar && \ + ln -sf /opt/rh/gcc-toolset-11/root/usr/bin/ranlib /usr/bin/ranlib && \ + echo '=== Binutils version check ===' && \ + ar --version | head -1 && \ + ranlib --version | head -1 && \ cmake .. \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=/usr/local \ @@ -176,14 +188,28 @@ RUN cd /tmp && \ # Set environment variables ENV PATH=/usr/local/bin:$PATH -# Configure ccache and Conan +# Configure ccache and Conan 2 +# NOTE: Using echo commands instead of heredocs because heredocs in Docker RUN commands are finnicky RUN /hbb_exe/activate-exec bash -c "ccache -M 10G && \ ccache -o cache_dir=/cache/ccache && \ ccache -o compiler_check=content && \ - conan config set storage.path=/cache/conan && \ - (conan profile new default --detect || true) && \ - conan profile update settings.compiler.libcxx=libstdc++11 default && \ - conan profile update settings.compiler.cppstd=20 default" + mkdir -p ~/.conan2 /cache/conan2 /cache/conan2_download /cache/conan2_sources && \ + echo 'core.cache:storage_path=/cache/conan2' > ~/.conan2/global.conf && \ + echo 'core.download:download_cache=/cache/conan2_download' >> ~/.conan2/global.conf && \ + echo 'core.sources:download_cache=/cache/conan2_sources' >> ~/.conan2/global.conf && \ + conan profile detect --force && \ + echo '[settings]' > ~/.conan2/profiles/default && \ + echo 'arch=x86_64' >> ~/.conan2/profiles/default && \ + echo 'build_type=Release' >> ~/.conan2/profiles/default && \ + echo 'compiler=gcc' >> ~/.conan2/profiles/default && \ + echo 'compiler.cppstd=20' >> ~/.conan2/profiles/default && \ + echo 'compiler.libcxx=libstdc++11' >> ~/.conan2/profiles/default && \ + echo 'compiler.version=11' >> ~/.conan2/profiles/default && \ + echo 'os=Linux' >> ~/.conan2/profiles/default && \ + echo '' >> ~/.conan2/profiles/default && \ + echo '[conf]' >> ~/.conan2/profiles/default && \ + echo '# Force building from source for packages with binary compatibility issues' >> ~/.conan2/profiles/default && \ + echo '*:tools.system.package_manager:mode=build' >> ~/.conan2/profiles/default" DOCKERFILE_EOF )