From a18b30b765c50f318f1a3928f19bd3ecb32b328f Mon Sep 17 00:00:00 2001 From: Bart Date: Wed, 26 Nov 2025 12:28:06 -0500 Subject: [PATCH 1/4] docs: Update instructions how to (re)generate conan.lock file (#6070) This change updates the instructions for how to generate a Conan lockfile that is compatible with Linux, macOS, and Windows. Whenever Conan dependencies change, the Conan lock file needs to be regenerated. However, different OSes have slightly different requirements, and thus require slightly different dependencies. Luckily a single `conan.lock` file can be used, as long as the union of dependencies is included. The current instructions in the `BUILD.md` file are insufficient to regenerate the lock file such that it is compatible with all OSes, which this change addresses. The three profiles contain the bare minimum needed to generate the lockfile; it does not particularly matter whether the build type is Debug or Release, for instance. --- BUILD.md | 34 ++++++--------------------- conan/lockfile/README.md | 42 ++++++++++++++++++++++++++++++++++ conan/lockfile/linux.profile | 8 +++++++ conan/lockfile/macos.profile | 8 +++++++ conan/lockfile/windows.profile | 9 ++++++++ 5 files changed, 74 insertions(+), 27 deletions(-) create mode 100644 conan/lockfile/README.md create mode 100644 conan/lockfile/linux.profile create mode 100644 conan/lockfile/macos.profile create mode 100644 conan/lockfile/windows.profile diff --git a/BUILD.md b/BUILD.md index 39570edbd3..c4b9b93467 100644 --- a/BUILD.md +++ b/BUILD.md @@ -88,6 +88,13 @@ These instructions assume a basic familiarity with Conan and CMake. If you are unfamiliar with Conan, then please read [this crash course](./docs/build/conan.md) or the official [Getting Started][3] walkthrough. +#### Conan lockfile + +To achieve reproducible dependencies, we use a [Conan lockfile](https://docs.conan.io/2/tutorial/versioning/lockfiles.html), +which has to be updated every time dependencies change. + +Please see the [instructions on how to regenerate the lockfile](conan/lockfile/README.md). + #### Default profile We recommend that you import the provided `conan/profiles/default` profile: @@ -450,33 +457,6 @@ tools.build:cxxflags=['-DBOOST_ASIO_DISABLE_CONCEPTS'] The location of `rippled` binary in your build directory depends on your CMake generator. Pass `--help` to see the rest of the command line options. -#### Conan lockfile - -To achieve reproducible dependencies, we use [Conan lockfile](https://docs.conan.io/2/tutorial/versioning/lockfiles.html). - -The `conan.lock` file in the repository contains a "snapshot" of the current dependencies. -It is implicitly used when running `conan` commands, you don't need to specify it. - -You have to update this file every time you add a new dependency or change a revision or version of an existing dependency. - -> [!NOTE] -> Conan uses local cache by default when creating a lockfile. -> -> To ensure, that lockfile creation works the same way on all developer machines, you should clear the local cache before creating a new lockfile. - -To create a new lockfile, run the following commands in the repository root: - -```bash -conan remove '*' --confirm -rm conan.lock -# This ensure that xrplf remote is the first to be consulted -conan remote add --force --index 0 xrplf https://conan.ripplex.io -conan lock create . -o '&:jemalloc=True' -o '&:rocksdb=True' -``` - -> [!NOTE] -> If some dependencies are exclusive for some OS, you may need to run the last command for them adding `--profile:all `. - ## Coverage report The coverage report is intended for developers using compilers GCC diff --git a/conan/lockfile/README.md b/conan/lockfile/README.md new file mode 100644 index 0000000000..d7ebba1b30 --- /dev/null +++ b/conan/lockfile/README.md @@ -0,0 +1,42 @@ +# Conan lockfile + +To achieve reproducible dependencies, we use a [Conan lockfile](https://docs.conan.io/2/tutorial/versioning/lockfiles.html). + +The `conan.lock` file in the repository contains a "snapshot" of the current +dependencies. It is implicitly used when running `conan` commands, so you don't +need to specify it. + +You have to update this file every time you add a new dependency or change a +revision or version of an existing dependency. + +## Updating the lockfile + +To update a lockfile, run the following commands from the repository root: + +```bash +# Ensure that the xrplf remote is the first to be consulted, so any recipes we +# patched are used. We also add it there to not created huge diff when the +# official Conan Center Index is updated. +conan remote add --force --index 0 xrplf https://conan.ripplex.io + +# Remove all local packages to prevent the local cache from influencing the +# lockfile. +conan remove '*' --confirm + +# Create a new lockfile that is compatible with Linux, macOS, and Windows. The +# first create command will create a new lockfile, while the subsequent create +# commands will merge any additional dependencies into the created lockfile. +rm conan.lock +conan lock create . \ + --options '&:jemalloc=True' \ + --options '&:rocksdb=True' \ + --profile:all=conan/lockfile/linux.profile +conan lock create . \ + --options '&:jemalloc=True' \ + --options '&:rocksdb=True' \ + --profile:all=conan/lockfile/macos.profile +conan lock create . \ + --options '&:jemalloc=True' \ + --options '&:rocksdb=True' \ + --profile:all=conan/lockfile/windows.profile +``` diff --git a/conan/lockfile/linux.profile b/conan/lockfile/linux.profile new file mode 100644 index 0000000000..25ad5988c5 --- /dev/null +++ b/conan/lockfile/linux.profile @@ -0,0 +1,8 @@ +[settings] +arch=x86_64 +build_type=Release +compiler=gcc +compiler.cppstd=20 +compiler.libcxx=libstdc++11 +compiler.version=13 +os=Linux diff --git a/conan/lockfile/macos.profile b/conan/lockfile/macos.profile new file mode 100644 index 0000000000..332a0c143d --- /dev/null +++ b/conan/lockfile/macos.profile @@ -0,0 +1,8 @@ +[settings] +arch=armv8 +build_type=Release +compiler=apple-clang +compiler.cppstd=20 +compiler.libcxx=libc++ +compiler.version=17.0 +os=Macos diff --git a/conan/lockfile/windows.profile b/conan/lockfile/windows.profile new file mode 100644 index 0000000000..4bb266a62e --- /dev/null +++ b/conan/lockfile/windows.profile @@ -0,0 +1,9 @@ +[settings] +arch=x86_64 +build_type=Release +compiler=msvc +compiler.cppstd=20 +compiler.runtime=dynamic +compiler.runtime_type=Release +compiler.version=194 +os=Windows From 4e25dc9291ff4a0433e17a6d7a63782960a35f1c Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Wed, 26 Nov 2025 17:54:36 +0000 Subject: [PATCH 2/4] chore: Update lockfile (#6083) This change update recipes, to make sure we have the best compatibility with latest compilers and so on. --- conan.lock | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/conan.lock b/conan.lock index 67c4dd8a10..93e91a307a 100644 --- a/conan.lock +++ b/conan.lock @@ -7,31 +7,32 @@ "soci/4.0.3#a9f8d773cd33e356b5879a4b0564f287%1756234262.318", "snappy/1.1.10#968fef506ff261592ec30c574d4a7809%1756234314.246", "rocksdb/10.5.1#4a197eca381a3e5ae8adf8cffa5aacd0%1762797952.535", - "re2/20230301#dfd6e2bf050eb90ddd8729cfb4c844a4%1756234257.976", - "protobuf/3.21.12#d927114e28de9f4691a6bbcdd9a529d1%1756234251.614", + "re2/20230301#ca3b241baec15bd31ea9187150e0b333%1764175362.029", + "protobuf/3.21.12#44ee56c0a6eea0c19aeeaca680370b88%1764175361.456", "openssl/3.5.4#a1d5835cc6ed5c5b8f3cd5b9b5d24205%1760106486.594", "nudb/2.0.9#fb8dfd1a5557f5e0528114c2da17721e%1763150366.909", "lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504%1756234228.999", "libiconv/1.17#1e65319e945f2d31941a9d28cc13c058%1756223727.64", "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1756230911.03", - "libarchive/3.8.1#5cf685686322e906cb42706ab7e099a8%1756234256.696", + "libarchive/3.8.1#ffee18995c706e02bf96e7a2f7042e0d%1764175360.142", "jemalloc/5.3.0#e951da9cf599e956cebc117880d2d9f8%1729241615.244", "grpc/1.50.1#02291451d1e17200293a409410d1c4e1%1756234248.958", "doctest/2.4.12#eb9fb352fb2fdfc8abb17ec270945165%1762797941.757", - "date/3.0.4#f74bbba5a08fa388256688743136cb6f%1756234217.493", - "c-ares/1.34.5#b78b91e7cfb1f11ce777a285bbf169c6%1756234217.915", - "bzip2/1.0.8#00b4a4658791c1f06914e087f0e792f5%1756234261.716", + "date/3.0.4#862e11e80030356b53c2c38599ceb32b%1763584497.32", + "c-ares/1.34.5#5581c2b62a608b40bb85d965ab3ec7c8%1764175359.429", + "bzip2/1.0.8#c470882369c2d95c5c77e970c0c7e321%1764175359.429", "boost/1.88.0#8852c0b72ce8271fb8ff7c53456d4983%1756223752.326", - "abseil/20230802.1#f0f91485b111dc9837a68972cb19ca7b%1756234220.907" + "abseil/20230802.1#90ba607d4ee8fb5fb157c3db540671fc%1764175359.429" ], "build_requires": [ "zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1756234269.497", "strawberryperl/5.32.1.1#707032463aa0620fa17ec0d887f5fe41%1756234281.733", - "protobuf/3.21.12#d927114e28de9f4691a6bbcdd9a529d1%1756234251.614", + "protobuf/3.21.12#44ee56c0a6eea0c19aeeaca680370b88%1764175361.456", "nasm/2.16.01#31e26f2ee3c4346ecd347911bd126904%1756234232.901", "msys2/cci.latest#5b73b10144f73cc5bfe0572ed9be39e1%1751977009.857", - "m4/1.4.19#f119296e5c4772b3bb7ab060ae8f417b%1760707875.678", - "cmake/3.31.8#dde3bde00bb843687e55aea5afa0e220%1756234232.89", + "m4/1.4.19#70dc8bbb33e981d119d2acc0175cf381%1763158052.846", + "cmake/4.2.0#ae0a44f44a1ef9ab68fd4b3e9a1f8671%1764175359.44", + "cmake/3.31.10#313d16a1aa16bbdb2ca0792467214b76%1764175359.429", "b2/5.3.3#107c15377719889654eb9a162a673975%1756234226.28", "automake/1.16.5#b91b7c384c3deaa9d535be02da14d04f%1755524470.56", "autoconf/2.71#51077f068e61700d65bb05541ea1e4b0%1731054366.86" @@ -48,14 +49,11 @@ "boost/1.83.0": [ "boost/1.88.0" ], - "sqlite3/[>=3.44 <4]": [ + "sqlite3/3.44.2": [ "sqlite3/3.49.1" ], "lz4/[>=1.9.4 <2]": [ "lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504" - ], - "sqlite3/3.44.2": [ - "sqlite3/3.49.1" ] }, "config_requires": [] From 2ed4047840bc89052aad6ab3bc5077c9d84c2a38 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Wed, 26 Nov 2025 18:43:14 +0000 Subject: [PATCH 3/4] chore: Make conan lockfile generation commands into a script (#6085) This change moves the lockfile instructions into a script, and instead of removing packages it sets the Conan home directory to a temporary directory. There are several advantages, such as: * Not affecting the user's Conan home directory, so there is no need to remove packages. * Only the script needs to be run, rather than several commands. --- conan/lockfile/README.md | 32 +------------------------------- conan/lockfile/regenerate.sh | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 31 deletions(-) create mode 100755 conan/lockfile/regenerate.sh diff --git a/conan/lockfile/README.md b/conan/lockfile/README.md index d7ebba1b30..5e70de102d 100644 --- a/conan/lockfile/README.md +++ b/conan/lockfile/README.md @@ -9,34 +9,4 @@ need to specify it. You have to update this file every time you add a new dependency or change a revision or version of an existing dependency. -## Updating the lockfile - -To update a lockfile, run the following commands from the repository root: - -```bash -# Ensure that the xrplf remote is the first to be consulted, so any recipes we -# patched are used. We also add it there to not created huge diff when the -# official Conan Center Index is updated. -conan remote add --force --index 0 xrplf https://conan.ripplex.io - -# Remove all local packages to prevent the local cache from influencing the -# lockfile. -conan remove '*' --confirm - -# Create a new lockfile that is compatible with Linux, macOS, and Windows. The -# first create command will create a new lockfile, while the subsequent create -# commands will merge any additional dependencies into the created lockfile. -rm conan.lock -conan lock create . \ - --options '&:jemalloc=True' \ - --options '&:rocksdb=True' \ - --profile:all=conan/lockfile/linux.profile -conan lock create . \ - --options '&:jemalloc=True' \ - --options '&:rocksdb=True' \ - --profile:all=conan/lockfile/macos.profile -conan lock create . \ - --options '&:jemalloc=True' \ - --options '&:rocksdb=True' \ - --profile:all=conan/lockfile/windows.profile -``` +To update a lockfile, run from the repository root: `./conan/lockfile/regenerate.sh` diff --git a/conan/lockfile/regenerate.sh b/conan/lockfile/regenerate.sh new file mode 100755 index 0000000000..30d38dc2d7 --- /dev/null +++ b/conan/lockfile/regenerate.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -ex + +TEMP_DIR=$(mktemp -d) +trap "rm -rf $TEMP_DIR" EXIT + +echo "Using temporary CONAN_HOME: $TEMP_DIR" + +# We use a temporary Conan home to avoid polluting the user's existing Conan +# configuration and to not use local cache (which leads to non-reproducible lockfiles). +export CONAN_HOME="$TEMP_DIR" + +# Ensure that the xrplf remote is the first to be consulted, so any recipes we +# patched are used. We also add it there to not created huge diff when the +# official Conan Center Index is updated. +conan remote add --force --index 0 xrplf https://conan.ripplex.io + +# Delete any existing lockfile. +rm -f conan.lock + +# Create a new lockfile that is compatible with Linux, macOS, and Windows. The +# first create command will create a new lockfile, while the subsequent create +# commands will merge any additional dependencies into the created lockfile. +conan lock create . \ + --options '&:jemalloc=True' \ + --options '&:rocksdb=True' \ + --profile:all=conan/lockfile/linux.profile +conan lock create . \ + --options '&:jemalloc=True' \ + --options '&:rocksdb=True' \ + --profile:all=conan/lockfile/macos.profile +conan lock create . \ + --options '&:jemalloc=True' \ + --options '&:rocksdb=True' \ + --profile:all=conan/lockfile/windows.profile From f399749ee2612200d166a53fbd863cfcf8f6fa61 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Wed, 26 Nov 2025 19:14:17 +0000 Subject: [PATCH 4/4] chore: Add black pre-commit hook (#6086) This change adds Black as a pre-commit hook to format Python files. --- .github/scripts/strategy-matrix/generate.py | 246 ++++++++++++----- .pre-commit-config.yaml | 5 + conanfile.py | 276 ++++++++++---------- tests/conan/conanfile.py | 19 +- 4 files changed, 332 insertions(+), 214 deletions(-) diff --git a/.github/scripts/strategy-matrix/generate.py b/.github/scripts/strategy-matrix/generate.py index 230c18c2df..79530a1d75 100755 --- a/.github/scripts/strategy-matrix/generate.py +++ b/.github/scripts/strategy-matrix/generate.py @@ -2,11 +2,12 @@ import argparse import itertools import json -from pathlib import Path from dataclasses import dataclass +from pathlib import Path THIS_DIR = Path(__file__).parent.resolve() + @dataclass class Config: architecture: list[dict] @@ -14,7 +15,8 @@ class Config: build_type: list[str] cmake_args: list[str] -''' + +""" Generate a strategy matrix for GitHub Actions CI. On each PR commit we will build a selection of Debian, RHEL, Ubuntu, MacOS, and @@ -27,19 +29,23 @@ We will further set additional CMake arguments as follows: - All release builds will have the `assert` option. - Certain Debian Bookworm configurations will change the reference fee, enable codecov, and enable voidstar in PRs. -''' +""" + + def generate_strategy_matrix(all: bool, config: Config) -> list: configurations = [] - for architecture, os, build_type, cmake_args in itertools.product(config.architecture, config.os, config.build_type, config.cmake_args): + for architecture, os, build_type, cmake_args in itertools.product( + config.architecture, config.os, config.build_type, config.cmake_args + ): # The default CMake target is 'all' for Linux and MacOS and 'install' # for Windows, but it can get overridden for certain configurations. - cmake_target = 'install' if os["distro_name"] == 'windows' else 'all' + cmake_target = "install" if os["distro_name"] == "windows" else "all" # We build and test all configurations by default, except for Windows in # Debug, because it is too slow, as well as when code coverage is # enabled as that mode already runs the tests. build_only = False - if os['distro_name'] == 'windows' and build_type == 'Debug': + if os["distro_name"] == "windows" and build_type == "Debug": build_only = True # Only generate a subset of configurations in PRs. @@ -54,21 +60,46 @@ def generate_strategy_matrix(all: bool, config: Config) -> list: # - Bookworm using Clang 17: Release and no Unity on linux/amd64, # set the reference fee to 1000. # - Bookworm using Clang 20: Debug and Unity on linux/amd64. - if os['distro_name'] == 'debian': + if os["distro_name"] == "debian": skip = True - if os['distro_version'] == 'bookworm': - if f'{os['compiler_name']}-{os['compiler_version']}' == 'gcc-13' and build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64': - cmake_args = f'-DUNIT_TEST_REFERENCE_FEE=500 {cmake_args}' + if os["distro_version"] == "bookworm": + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-13" + and build_type == "Release" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "linux/amd64" + ): + cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=500 {cmake_args}" skip = False - if f'{os['compiler_name']}-{os['compiler_version']}' == 'gcc-15' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/amd64': + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15" + and build_type == "Debug" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "linux/amd64" + ): skip = False - if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-16' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/arm64': - cmake_args = f'-Dvoidstar=ON {cmake_args}' + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "clang-16" + and build_type == "Debug" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "linux/arm64" + ): + cmake_args = f"-Dvoidstar=ON {cmake_args}" skip = False - if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-17' and build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64': - cmake_args = f'-DUNIT_TEST_REFERENCE_FEE=1000 {cmake_args}' + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "clang-17" + and build_type == "Release" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "linux/amd64" + ): + cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=1000 {cmake_args}" skip = False - if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-20' and build_type == 'Debug' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64': + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "clang-20" + and build_type == "Debug" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "linux/amd64" + ): skip = False if skip: continue @@ -76,13 +107,23 @@ def generate_strategy_matrix(all: bool, config: Config) -> list: # RHEL: # - 9 using GCC 12: Debug and Unity on linux/amd64. # - 10 using Clang: Release and no Unity on linux/amd64. - if os['distro_name'] == 'rhel': + if os["distro_name"] == "rhel": skip = True - if os['distro_version'] == '9': - if f'{os['compiler_name']}-{os['compiler_version']}' == 'gcc-12' and build_type == 'Debug' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64': + if os["distro_version"] == "9": + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12" + and build_type == "Debug" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "linux/amd64" + ): skip = False - elif os['distro_version'] == '10': - if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-any' and build_type == 'Release' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/amd64': + elif os["distro_version"] == "10": + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "clang-any" + and build_type == "Release" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "linux/amd64" + ): skip = False if skip: continue @@ -92,104 +133,169 @@ def generate_strategy_matrix(all: bool, config: Config) -> list: # - Noble using GCC 14: Release and Unity on linux/amd64. # - Noble using Clang 18: Debug and no Unity on linux/amd64. # - Noble using Clang 19: Release and Unity on linux/arm64. - if os['distro_name'] == 'ubuntu': + if os["distro_name"] == "ubuntu": skip = True - if os['distro_version'] == 'jammy': - if f'{os['compiler_name']}-{os['compiler_version']}' == 'gcc-12' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/arm64': + if os["distro_version"] == "jammy": + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12" + and build_type == "Debug" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "linux/arm64" + ): skip = False - elif os['distro_version'] == 'noble': - if f'{os['compiler_name']}-{os['compiler_version']}' == 'gcc-14' and build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64': + elif os["distro_version"] == "noble": + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-14" + and build_type == "Release" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "linux/amd64" + ): skip = False - if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-18' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/amd64': + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "clang-18" + and build_type == "Debug" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "linux/amd64" + ): skip = False - if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-19' and build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/arm64': + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "clang-19" + and build_type == "Release" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "linux/arm64" + ): skip = False if skip: continue # MacOS: # - Debug and no Unity on macos/arm64. - if os['distro_name'] == 'macos' and not (build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'macos/arm64'): + if os["distro_name"] == "macos" and not ( + build_type == "Debug" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "macos/arm64" + ): continue # Windows: # - Release and Unity on windows/amd64. - if os['distro_name'] == 'windows' and not (build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'windows/amd64'): + if os["distro_name"] == "windows" and not ( + build_type == "Release" + and "-Dunity=ON" in cmake_args + and architecture["platform"] == "windows/amd64" + ): continue - # Additional CMake arguments. - cmake_args = f'{cmake_args} -Dtests=ON -Dwerr=ON -Dxrpld=ON' - if not f'{os['compiler_name']}-{os['compiler_version']}' in ['gcc-12', 'clang-16']: - cmake_args = f'{cmake_args} -Dwextra=ON' - if build_type == 'Release': - cmake_args = f'{cmake_args} -Dassert=ON' + cmake_args = f"{cmake_args} -Dtests=ON -Dwerr=ON -Dxrpld=ON" + if not f"{os['compiler_name']}-{os['compiler_version']}" in [ + "gcc-12", + "clang-16", + ]: + cmake_args = f"{cmake_args} -Dwextra=ON" + if build_type == "Release": + cmake_args = f"{cmake_args} -Dassert=ON" # We skip all RHEL on arm64 due to a build failure that needs further # investigation. - if os['distro_name'] == 'rhel' and architecture['platform'] == 'linux/arm64': + if os["distro_name"] == "rhel" and architecture["platform"] == "linux/arm64": continue # We skip all clang 20+ on arm64 due to Boost build error. - if f'{os['compiler_name']}-{os['compiler_version']}' in ['clang-20', 'clang-21'] and architecture['platform'] == 'linux/arm64': + if ( + f"{os['compiler_name']}-{os['compiler_version']}" + in ["clang-20", "clang-21"] + and architecture["platform"] == "linux/arm64" + ): continue # Enable code coverage for Debian Bookworm using GCC 15 in Debug and no # Unity on linux/amd64 - if f'{os['compiler_name']}-{os['compiler_version']}' == 'gcc-15' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/amd64': - cmake_args = f'-Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0 {cmake_args}' + if ( + f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15" + and build_type == "Debug" + and "-Dunity=OFF" in cmake_args + and architecture["platform"] == "linux/amd64" + ): + cmake_args = f"-Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0 {cmake_args}" # Generate a unique name for the configuration, e.g. macos-arm64-debug # or debian-bookworm-gcc-12-amd64-release-unity. - config_name = os['distro_name'] - if (n := os['distro_version']) != '': - config_name += f'-{n}' - if (n := os['compiler_name']) != '': - config_name += f'-{n}' - if (n := os['compiler_version']) != '': - config_name += f'-{n}' - config_name += f'-{architecture['platform'][architecture['platform'].find('/')+1:]}' - config_name += f'-{build_type.lower()}' - if '-Dunity=ON' in cmake_args: - config_name += '-unity' + config_name = os["distro_name"] + if (n := os["distro_version"]) != "": + config_name += f"-{n}" + if (n := os["compiler_name"]) != "": + config_name += f"-{n}" + if (n := os["compiler_version"]) != "": + config_name += f"-{n}" + config_name += ( + f"-{architecture['platform'][architecture['platform'].find('/') + 1 :]}" + ) + config_name += f"-{build_type.lower()}" + if "-Dunity=ON" in cmake_args: + config_name += "-unity" # Add the configuration to the list, with the most unique fields first, # so that they are easier to identify in the GitHub Actions UI, as long # names get truncated. - configurations.append({ - 'config_name': config_name, - 'cmake_args': cmake_args, - 'cmake_target': cmake_target, - 'build_only': build_only, - 'build_type': build_type, - 'os': os, - 'architecture': architecture, - }) + configurations.append( + { + "config_name": config_name, + "cmake_args": cmake_args, + "cmake_target": cmake_target, + "build_only": build_only, + "build_type": build_type, + "os": os, + "architecture": architecture, + } + ) return configurations def read_config(file: Path) -> Config: config = json.loads(file.read_text()) - if config['architecture'] is None or config['os'] is None or config['build_type'] is None or config['cmake_args'] is None: - raise Exception('Invalid configuration file.') + if ( + config["architecture"] is None + or config["os"] is None + or config["build_type"] is None + or config["cmake_args"] is None + ): + raise Exception("Invalid configuration file.") return Config(**config) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-a', '--all', help='Set to generate all configurations (generally used when merging a PR) or leave unset to generate a subset of configurations (generally used when committing to a PR).', action="store_true") - parser.add_argument('-c', '--config', help='Path to the JSON file containing the strategy matrix configurations.', required=False, type=Path) + parser.add_argument( + "-a", + "--all", + help="Set to generate all configurations (generally used when merging a PR) or leave unset to generate a subset of configurations (generally used when committing to a PR).", + action="store_true", + ) + parser.add_argument( + "-c", + "--config", + help="Path to the JSON file containing the strategy matrix configurations.", + required=False, + type=Path, + ) args = parser.parse_args() matrix = [] - if args.config is None or args.config == '': - matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "linux.json")) - matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "macos.json")) - matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "windows.json")) + if args.config is None or args.config == "": + matrix += generate_strategy_matrix( + args.all, read_config(THIS_DIR / "linux.json") + ) + matrix += generate_strategy_matrix( + args.all, read_config(THIS_DIR / "macos.json") + ) + matrix += generate_strategy_matrix( + args.all, read_config(THIS_DIR / "windows.json") + ) else: matrix += generate_strategy_matrix(args.all, read_config(args.config)) # Generate the strategy matrix. - print(f'matrix={json.dumps({"include": matrix})}') + print(f"matrix={json.dumps({'include': matrix})}") diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a032fee75e..c85e0798f7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,6 +31,11 @@ repos: hooks: - id: prettier + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 25.11.0 + hooks: + - id: black + exclude: | (?x)^( external/.*| diff --git a/conanfile.py b/conanfile.py index 41ec5d35f3..5a0a50a0e7 100644 --- a/conanfile.py +++ b/conanfile.py @@ -1,149 +1,155 @@ -from conan import ConanFile, __version__ as conan_version -from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout import re -class Xrpl(ConanFile): - name = 'xrpl' +from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout - license = 'ISC' - author = 'John Freeman ' - url = 'https://github.com/xrplf/rippled' - description = 'The XRP Ledger' - settings = 'os', 'compiler', 'build_type', 'arch' +from conan import ConanFile +from conan import __version__ as conan_version + + +class Xrpl(ConanFile): + name = "xrpl" + + license = "ISC" + author = "John Freeman " + url = "https://github.com/xrplf/rippled" + description = "The XRP Ledger" + settings = "os", "compiler", "build_type", "arch" options = { - 'assertions': [True, False], - 'coverage': [True, False], - 'fPIC': [True, False], - 'jemalloc': [True, False], - 'rocksdb': [True, False], - 'shared': [True, False], - 'static': [True, False], - 'tests': [True, False], - 'unity': [True, False], - 'xrpld': [True, False], + "assertions": [True, False], + "coverage": [True, False], + "fPIC": [True, False], + "jemalloc": [True, False], + "rocksdb": [True, False], + "shared": [True, False], + "static": [True, False], + "tests": [True, False], + "unity": [True, False], + "xrpld": [True, False], } requires = [ - 'grpc/1.50.1', - 'libarchive/3.8.1', - 'nudb/2.0.9', - 'openssl/3.5.4', - 'soci/4.0.3', - 'zlib/1.3.1', + "grpc/1.50.1", + "libarchive/3.8.1", + "nudb/2.0.9", + "openssl/3.5.4", + "soci/4.0.3", + "zlib/1.3.1", ] test_requires = [ - 'doctest/2.4.12', + "doctest/2.4.12", ] tool_requires = [ - 'protobuf/3.21.12', + "protobuf/3.21.12", ] default_options = { - 'assertions': False, - 'coverage': False, - 'fPIC': True, - 'jemalloc': False, - 'rocksdb': True, - 'shared': False, - 'static': True, - 'tests': False, - 'unity': False, - 'xrpld': False, - - 'date/*:header_only': True, - 'grpc/*:shared': False, - 'grpc/*:secure': True, - 'libarchive/*:shared': False, - 'libarchive/*:with_acl': False, - 'libarchive/*:with_bzip2': False, - 'libarchive/*:with_cng': False, - 'libarchive/*:with_expat': False, - 'libarchive/*:with_iconv': False, - 'libarchive/*:with_libxml2': False, - 'libarchive/*:with_lz4': True, - 'libarchive/*:with_lzma': False, - 'libarchive/*:with_lzo': False, - 'libarchive/*:with_nettle': False, - 'libarchive/*:with_openssl': False, - 'libarchive/*:with_pcreposix': False, - 'libarchive/*:with_xattr': False, - 'libarchive/*:with_zlib': False, - 'lz4/*:shared': False, - 'openssl/*:shared': False, - 'protobuf/*:shared': False, - 'protobuf/*:with_zlib': True, - 'rocksdb/*:enable_sse': False, - 'rocksdb/*:lite': False, - 'rocksdb/*:shared': False, - 'rocksdb/*:use_rtti': True, - 'rocksdb/*:with_jemalloc': False, - 'rocksdb/*:with_lz4': True, - 'rocksdb/*:with_snappy': True, - 'snappy/*:shared': False, - 'soci/*:shared': False, - 'soci/*:with_sqlite3': True, - 'soci/*:with_boost': True, - 'xxhash/*:shared': False, + "assertions": False, + "coverage": False, + "fPIC": True, + "jemalloc": False, + "rocksdb": True, + "shared": False, + "static": True, + "tests": False, + "unity": False, + "xrpld": False, + "date/*:header_only": True, + "grpc/*:shared": False, + "grpc/*:secure": True, + "libarchive/*:shared": False, + "libarchive/*:with_acl": False, + "libarchive/*:with_bzip2": False, + "libarchive/*:with_cng": False, + "libarchive/*:with_expat": False, + "libarchive/*:with_iconv": False, + "libarchive/*:with_libxml2": False, + "libarchive/*:with_lz4": True, + "libarchive/*:with_lzma": False, + "libarchive/*:with_lzo": False, + "libarchive/*:with_nettle": False, + "libarchive/*:with_openssl": False, + "libarchive/*:with_pcreposix": False, + "libarchive/*:with_xattr": False, + "libarchive/*:with_zlib": False, + "lz4/*:shared": False, + "openssl/*:shared": False, + "protobuf/*:shared": False, + "protobuf/*:with_zlib": True, + "rocksdb/*:enable_sse": False, + "rocksdb/*:lite": False, + "rocksdb/*:shared": False, + "rocksdb/*:use_rtti": True, + "rocksdb/*:with_jemalloc": False, + "rocksdb/*:with_lz4": True, + "rocksdb/*:with_snappy": True, + "snappy/*:shared": False, + "soci/*:shared": False, + "soci/*:with_sqlite3": True, + "soci/*:with_boost": True, + "xxhash/*:shared": False, } def set_version(self): if self.version is None: - path = f'{self.recipe_folder}/src/libxrpl/protocol/BuildInfo.cpp' - regex = r'versionString\s?=\s?\"(.*)\"' - with open(path, encoding='utf-8') as file: + path = f"{self.recipe_folder}/src/libxrpl/protocol/BuildInfo.cpp" + regex = r"versionString\s?=\s?\"(.*)\"" + with open(path, encoding="utf-8") as file: matches = (re.search(regex, line) for line in file) match = next(m for m in matches if m) self.version = match.group(1) def configure(self): - if self.settings.compiler == 'apple-clang': - self.options['boost'].visibility = 'global' - if self.settings.compiler in ['clang', 'gcc']: - self.options['boost'].without_cobalt = True + if self.settings.compiler == "apple-clang": + self.options["boost"].visibility = "global" + if self.settings.compiler in ["clang", "gcc"]: + self.options["boost"].without_cobalt = True def requirements(self): # Conan 2 requires transitive headers to be specified - transitive_headers_opt = {'transitive_headers': True} if conan_version.split('.')[0] == '2' else {} - self.requires('boost/1.88.0', force=True, **transitive_headers_opt) - self.requires('date/3.0.4', **transitive_headers_opt) - self.requires('lz4/1.10.0', force=True) - self.requires('protobuf/3.21.12', force=True) - self.requires('sqlite3/3.49.1', force=True) + transitive_headers_opt = ( + {"transitive_headers": True} if conan_version.split(".")[0] == "2" else {} + ) + self.requires("boost/1.88.0", force=True, **transitive_headers_opt) + self.requires("date/3.0.4", **transitive_headers_opt) + self.requires("lz4/1.10.0", force=True) + self.requires("protobuf/3.21.12", force=True) + self.requires("sqlite3/3.49.1", force=True) if self.options.jemalloc: - self.requires('jemalloc/5.3.0') + self.requires("jemalloc/5.3.0") if self.options.rocksdb: - self.requires('rocksdb/10.5.1') - self.requires('xxhash/0.8.3', **transitive_headers_opt) + self.requires("rocksdb/10.5.1") + self.requires("xxhash/0.8.3", **transitive_headers_opt) exports_sources = ( - 'CMakeLists.txt', - 'cfg/*', - 'cmake/*', - 'external/*', - 'include/*', - 'src/*', + "CMakeLists.txt", + "cfg/*", + "cmake/*", + "external/*", + "include/*", + "src/*", ) def layout(self): cmake_layout(self) # Fix this setting to follow the default introduced in Conan 1.48 # to align with our build instructions. - self.folders.generators = 'build/generators' + self.folders.generators = "build/generators" + + generators = "CMakeDeps" - generators = 'CMakeDeps' def generate(self): tc = CMakeToolchain(self) - tc.variables['tests'] = self.options.tests - tc.variables['assert'] = self.options.assertions - tc.variables['coverage'] = self.options.coverage - tc.variables['jemalloc'] = self.options.jemalloc - tc.variables['rocksdb'] = self.options.rocksdb - tc.variables['BUILD_SHARED_LIBS'] = self.options.shared - tc.variables['static'] = self.options.static - tc.variables['unity'] = self.options.unity - tc.variables['xrpld'] = self.options.xrpld + tc.variables["tests"] = self.options.tests + tc.variables["assert"] = self.options.assertions + tc.variables["coverage"] = self.options.coverage + tc.variables["jemalloc"] = self.options.jemalloc + tc.variables["rocksdb"] = self.options.rocksdb + tc.variables["BUILD_SHARED_LIBS"] = self.options.shared + tc.variables["static"] = self.options.static + tc.variables["unity"] = self.options.unity + tc.variables["xrpld"] = self.options.xrpld tc.generate() def build(self): @@ -158,40 +164,40 @@ class Xrpl(ConanFile): cmake.install() def package_info(self): - libxrpl = self.cpp_info.components['libxrpl'] + libxrpl = self.cpp_info.components["libxrpl"] libxrpl.libs = [ - 'xrpl', - 'xrpl.libpb', - 'ed25519', - 'secp256k1', + "xrpl", + "xrpl.libpb", + "ed25519", + "secp256k1", ] # TODO: Fix the protobufs to include each other relative to # `include/`, not `include/ripple/proto/`. - libxrpl.includedirs = ['include', 'include/ripple/proto'] + libxrpl.includedirs = ["include", "include/ripple/proto"] libxrpl.requires = [ - 'boost::headers', - 'boost::chrono', - 'boost::container', - 'boost::coroutine', - 'boost::date_time', - 'boost::filesystem', - 'boost::json', - 'boost::program_options', - 'boost::process', - 'boost::regex', - 'boost::system', - 'boost::thread', - 'date::date', - 'grpc::grpc++', - 'libarchive::libarchive', - 'lz4::lz4', - 'nudb::nudb', - 'openssl::crypto', - 'protobuf::libprotobuf', - 'soci::soci', - 'sqlite3::sqlite', - 'xxhash::xxhash', - 'zlib::zlib', + "boost::headers", + "boost::chrono", + "boost::container", + "boost::coroutine", + "boost::date_time", + "boost::filesystem", + "boost::json", + "boost::program_options", + "boost::process", + "boost::regex", + "boost::system", + "boost::thread", + "date::date", + "grpc::grpc++", + "libarchive::libarchive", + "lz4::lz4", + "nudb::nudb", + "openssl::crypto", + "protobuf::libprotobuf", + "soci::soci", + "sqlite3::sqlite", + "xxhash::xxhash", + "zlib::zlib", ] if self.options.rocksdb: - libxrpl.requires.append('rocksdb::librocksdb') + libxrpl.requires.append("rocksdb::librocksdb") diff --git a/tests/conan/conanfile.py b/tests/conan/conanfile.py index 1ea1b333fc..71e43ef25a 100644 --- a/tests/conan/conanfile.py +++ b/tests/conan/conanfile.py @@ -1,28 +1,29 @@ from pathlib import Path -from conan import ConanFile from conan.tools.build import can_run from conan.tools.cmake import CMake, cmake_layout +from conan import ConanFile + + class Example(ConanFile): + name = "example" + license = "ISC" + author = "John Freeman , Michael Legleux