Merge branch 'develop' into ximinez/fix/validator-cache

This change adds Black as a pre-commit hook to format Python files.

.github/scripts/strategy-matrix/generate.py

@@ -2,11 +2,12 @@
 import argparse
 import itertools
 import json
-from pathlib import Path
 from dataclasses import dataclass
+from pathlib import Path
 
 THIS_DIR = Path(__file__).parent.resolve()
 
+
 @dataclass
 class Config:
     architecture: list[dict]
@@ -14,7 +15,8 @@ class Config:
     build_type: list[str]
     cmake_args: list[str]
 
-'''
+
+"""
 Generate a strategy matrix for GitHub Actions CI.
 
 On each PR commit we will build a selection of Debian, RHEL, Ubuntu, MacOS, and
@@ -27,19 +29,23 @@ We will further set additional CMake arguments as follows:
 - All release builds will have the `assert` option.
 - Certain Debian Bookworm configurations will change the reference fee, enable
   codecov, and enable voidstar in PRs.
-'''
+"""
+
+
 def generate_strategy_matrix(all: bool, config: Config) -> list:
     configurations = []
-    for architecture, os, build_type, cmake_args in itertools.product(config.architecture, config.os, config.build_type, config.cmake_args):
+    for architecture, os, build_type, cmake_args in itertools.product(
+        config.architecture, config.os, config.build_type, config.cmake_args
+    ):
         # The default CMake target is 'all' for Linux and MacOS and 'install'
         # for Windows, but it can get overridden for certain configurations.
-        cmake_target = 'install' if os["distro_name"] == 'windows' else 'all'
+        cmake_target = "install" if os["distro_name"] == "windows" else "all"
 
         # We build and test all configurations by default, except for Windows in
         # Debug, because it is too slow, as well as when code coverage is
         # enabled as that mode already runs the tests.
         build_only = False
-        if os['distro_name'] == 'windows' and build_type == 'Debug':
+        if os["distro_name"] == "windows" and build_type == "Debug":
             build_only = True
 
         # Only generate a subset of configurations in PRs.
@@ -54,21 +60,46 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
             # - Bookworm using Clang 17: Release and no Unity on linux/amd64,
             #   set the reference fee to 1000.
             # - Bookworm using Clang 20: Debug and Unity on linux/amd64.
-            if os['distro_name'] == 'debian':
+            if os["distro_name"] == "debian":
                 skip = True
-                if os['distro_version'] == 'bookworm':
-                    if f'{os['compiler_name']}-{os['compiler_version']}' == 'gcc-13' and build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64':
-                        cmake_args = f'-DUNIT_TEST_REFERENCE_FEE=500 {cmake_args}'
+                if os["distro_version"] == "bookworm":
+                    if (
+                        f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-13"
+                        and build_type == "Release"
+                        and "-Dunity=ON" in cmake_args
+                        and architecture["platform"] == "linux/amd64"
+                    ):
+                        cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=500 {cmake_args}"
                         skip = False
-                    if f'{os['compiler_name']}-{os['compiler_version']}' == 'gcc-15' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/amd64':
+                    if (
+                        f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15"
+                        and build_type == "Debug"
+                        and "-Dunity=OFF" in cmake_args
+                        and architecture["platform"] == "linux/amd64"
+                    ):
                         skip = False
-                    if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-16' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/arm64':
-                        cmake_args = f'-Dvoidstar=ON {cmake_args}'
+                    if (
+                        f"{os['compiler_name']}-{os['compiler_version']}" == "clang-16"
+                        and build_type == "Debug"
+                        and "-Dunity=OFF" in cmake_args
+                        and architecture["platform"] == "linux/arm64"
+                    ):
+                        cmake_args = f"-Dvoidstar=ON {cmake_args}"
                         skip = False
-                    if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-17' and build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64':
-                        cmake_args = f'-DUNIT_TEST_REFERENCE_FEE=1000 {cmake_args}'
+                    if (
+                        f"{os['compiler_name']}-{os['compiler_version']}" == "clang-17"
+                        and build_type == "Release"
+                        and "-Dunity=ON" in cmake_args
+                        and architecture["platform"] == "linux/amd64"
+                    ):
+                        cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=1000 {cmake_args}"
                         skip = False
-                    if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-20' and build_type == 'Debug' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64':
+                    if (
+                        f"{os['compiler_name']}-{os['compiler_version']}" == "clang-20"
+                        and build_type == "Debug"
+                        and "-Dunity=ON" in cmake_args
+                        and architecture["platform"] == "linux/amd64"
+                    ):
                         skip = False
                 if skip:
                     continue
@@ -76,13 +107,23 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
             # RHEL:
             # - 9 using GCC 12: Debug and Unity on linux/amd64.
             # - 10 using Clang: Release and no Unity on linux/amd64.
-            if os['distro_name'] == 'rhel':
+            if os["distro_name"] == "rhel":
                 skip = True
-                if os['distro_version'] == '9':
-                    if f'{os['compiler_name']}-{os['compiler_version']}' == 'gcc-12' and build_type == 'Debug' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64':
+                if os["distro_version"] == "9":
+                    if (
+                        f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12"
+                        and build_type == "Debug"
+                        and "-Dunity=ON" in cmake_args
+                        and architecture["platform"] == "linux/amd64"
+                    ):
                         skip = False
-                elif os['distro_version'] == '10':
-                    if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-any' and build_type == 'Release' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/amd64':
+                elif os["distro_version"] == "10":
+                    if (
+                        f"{os['compiler_name']}-{os['compiler_version']}" == "clang-any"
+                        and build_type == "Release"
+                        and "-Dunity=OFF" in cmake_args
+                        and architecture["platform"] == "linux/amd64"
+                    ):
                         skip = False
                 if skip:
                     continue
@@ -92,104 +133,169 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
             # - Noble using GCC 14: Release and Unity on linux/amd64.
             # - Noble using Clang 18: Debug and no Unity on linux/amd64.
             # - Noble using Clang 19: Release and Unity on linux/arm64.
-            if os['distro_name'] == 'ubuntu':
+            if os["distro_name"] == "ubuntu":
                 skip = True
-                if os['distro_version'] == 'jammy':
-                    if f'{os['compiler_name']}-{os['compiler_version']}' == 'gcc-12' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/arm64':
+                if os["distro_version"] == "jammy":
+                    if (
+                        f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12"
+                        and build_type == "Debug"
+                        and "-Dunity=OFF" in cmake_args
+                        and architecture["platform"] == "linux/arm64"
+                    ):
                         skip = False
-                elif os['distro_version'] == 'noble':
-                    if f'{os['compiler_name']}-{os['compiler_version']}' == 'gcc-14' and build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/amd64':
+                elif os["distro_version"] == "noble":
+                    if (
+                        f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-14"
+                        and build_type == "Release"
+                        and "-Dunity=ON" in cmake_args
+                        and architecture["platform"] == "linux/amd64"
+                    ):
                         skip = False
-                    if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-18' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/amd64':
+                    if (
+                        f"{os['compiler_name']}-{os['compiler_version']}" == "clang-18"
+                        and build_type == "Debug"
+                        and "-Dunity=OFF" in cmake_args
+                        and architecture["platform"] == "linux/amd64"
+                    ):
                         skip = False
-                    if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-19' and build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'linux/arm64':
+                    if (
+                        f"{os['compiler_name']}-{os['compiler_version']}" == "clang-19"
+                        and build_type == "Release"
+                        and "-Dunity=ON" in cmake_args
+                        and architecture["platform"] == "linux/arm64"
+                    ):
                         skip = False
                 if skip:
                     continue
 
             # MacOS:
             # - Debug and no Unity on macos/arm64.
-            if os['distro_name'] == 'macos' and not (build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'macos/arm64'):
+            if os["distro_name"] == "macos" and not (
+                build_type == "Debug"
+                and "-Dunity=OFF" in cmake_args
+                and architecture["platform"] == "macos/arm64"
+            ):
                 continue
 
             # Windows:
             # - Release and Unity on windows/amd64.
-            if os['distro_name'] == 'windows' and not (build_type == 'Release' and '-Dunity=ON' in cmake_args and architecture['platform'] == 'windows/amd64'):
+            if os["distro_name"] == "windows" and not (
+                build_type == "Release"
+                and "-Dunity=ON" in cmake_args
+                and architecture["platform"] == "windows/amd64"
+            ):
                 continue
 
-
         # Additional CMake arguments.
-        cmake_args = f'{cmake_args} -Dtests=ON -Dwerr=ON -Dxrpld=ON'
-        if not f'{os['compiler_name']}-{os['compiler_version']}' in ['gcc-12', 'clang-16']:
-            cmake_args = f'{cmake_args} -Dwextra=ON'
-        if build_type == 'Release':
-            cmake_args = f'{cmake_args} -Dassert=ON'
+        cmake_args = f"{cmake_args} -Dtests=ON -Dwerr=ON -Dxrpld=ON"
+        if not f"{os['compiler_name']}-{os['compiler_version']}" in [
+            "gcc-12",
+            "clang-16",
+        ]:
+            cmake_args = f"{cmake_args} -Dwextra=ON"
+        if build_type == "Release":
+            cmake_args = f"{cmake_args} -Dassert=ON"
 
         # We skip all RHEL on arm64 due to a build failure that needs further
         # investigation.
-        if os['distro_name'] == 'rhel' and architecture['platform'] == 'linux/arm64':
+        if os["distro_name"] == "rhel" and architecture["platform"] == "linux/arm64":
             continue
 
         # We skip all clang 20+ on arm64 due to Boost build error.
-        if f'{os['compiler_name']}-{os['compiler_version']}' in ['clang-20', 'clang-21'] and architecture['platform'] == 'linux/arm64':
+        if (
+            f"{os['compiler_name']}-{os['compiler_version']}"
+            in ["clang-20", "clang-21"]
+            and architecture["platform"] == "linux/arm64"
+        ):
             continue
 
         # Enable code coverage for Debian Bookworm using GCC 15 in Debug and no
         # Unity on linux/amd64
-        if f'{os['compiler_name']}-{os['compiler_version']}' == 'gcc-15' and build_type == 'Debug' and '-Dunity=OFF' in cmake_args and architecture['platform'] == 'linux/amd64':
-            cmake_args = f'-Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0 {cmake_args}'
+        if (
+            f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15"
+            and build_type == "Debug"
+            and "-Dunity=OFF" in cmake_args
+            and architecture["platform"] == "linux/amd64"
+        ):
+            cmake_args = f"-Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0 {cmake_args}"
 
         # Generate a unique name for the configuration, e.g. macos-arm64-debug
         # or debian-bookworm-gcc-12-amd64-release-unity.
-        config_name = os['distro_name']
-        if (n := os['distro_version']) != '':
-            config_name += f'-{n}'
-        if (n := os['compiler_name']) != '':
-            config_name += f'-{n}'
-        if (n := os['compiler_version']) != '':
-            config_name += f'-{n}'
-        config_name += f'-{architecture['platform'][architecture['platform'].find('/')+1:]}'
-        config_name += f'-{build_type.lower()}'
-        if '-Dunity=ON' in cmake_args:
-            config_name += '-unity'
+        config_name = os["distro_name"]
+        if (n := os["distro_version"]) != "":
+            config_name += f"-{n}"
+        if (n := os["compiler_name"]) != "":
+            config_name += f"-{n}"
+        if (n := os["compiler_version"]) != "":
+            config_name += f"-{n}"
+        config_name += (
+            f"-{architecture['platform'][architecture['platform'].find('/') + 1 :]}"
+        )
+        config_name += f"-{build_type.lower()}"
+        if "-Dunity=ON" in cmake_args:
+            config_name += "-unity"
 
         # Add the configuration to the list, with the most unique fields first,
         # so that they are easier to identify in the GitHub Actions UI, as long
         # names get truncated.
-        configurations.append({
-            'config_name': config_name,
-            'cmake_args': cmake_args,
-            'cmake_target': cmake_target,
-            'build_only': build_only,
-            'build_type': build_type,
-            'os': os,
-            'architecture': architecture,
-        })
+        configurations.append(
+            {
+                "config_name": config_name,
+                "cmake_args": cmake_args,
+                "cmake_target": cmake_target,
+                "build_only": build_only,
+                "build_type": build_type,
+                "os": os,
+                "architecture": architecture,
+            }
+        )
 
     return configurations
 
 
 def read_config(file: Path) -> Config:
     config = json.loads(file.read_text())
-    if config['architecture'] is None or config['os'] is None or config['build_type'] is None or config['cmake_args'] is None:
-        raise Exception('Invalid configuration file.')
+    if (
+        config["architecture"] is None
+        or config["os"] is None
+        or config["build_type"] is None
+        or config["cmake_args"] is None
+    ):
+        raise Exception("Invalid configuration file.")
 
     return Config(**config)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument('-a', '--all', help='Set to generate all configurations (generally used when merging a PR) or leave unset to generate a subset of configurations (generally used when committing to a PR).', action="store_true")
-    parser.add_argument('-c', '--config', help='Path to the JSON file containing the strategy matrix configurations.', required=False, type=Path)
+    parser.add_argument(
+        "-a",
+        "--all",
+        help="Set to generate all configurations (generally used when merging a PR) or leave unset to generate a subset of configurations (generally used when committing to a PR).",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-c",
+        "--config",
+        help="Path to the JSON file containing the strategy matrix configurations.",
+        required=False,
+        type=Path,
+    )
     args = parser.parse_args()
 
     matrix = []
-    if args.config is None or args.config == '':
-        matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "linux.json"))
-        matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "macos.json"))
-        matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "windows.json"))
+    if args.config is None or args.config == "":
+        matrix += generate_strategy_matrix(
+            args.all, read_config(THIS_DIR / "linux.json")
+        )
+        matrix += generate_strategy_matrix(
+            args.all, read_config(THIS_DIR / "macos.json")
+        )
+        matrix += generate_strategy_matrix(
+            args.all, read_config(THIS_DIR / "windows.json")
+        )
     else:
         matrix += generate_strategy_matrix(args.all, read_config(args.config))
 
     # Generate the strategy matrix.
-    print(f'matrix={json.dumps({"include": matrix})}')
+    print(f"matrix={json.dumps({'include': matrix})}")

.pre-commit-config.yaml

@@ -31,6 +31,11 @@ repos:
     hooks:
       - id: prettier
 
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 25.11.0
+    hooks:
+      - id: black
+
 exclude: |
   (?x)^(
       external/.*|

BUILD.md

@@ -88,6 +88,13 @@ These instructions assume a basic familiarity with Conan and CMake. If you are
 unfamiliar with Conan, then please read [this crash course](./docs/build/conan.md) or the official
 [Getting Started][3] walkthrough.
 
+#### Conan lockfile
+
+To achieve reproducible dependencies, we use a [Conan lockfile](https://docs.conan.io/2/tutorial/versioning/lockfiles.html),
+which has to be updated every time dependencies change.
+
+Please see the [instructions on how to regenerate the lockfile](conan/lockfile/README.md).
+
 #### Default profile
 
 We recommend that you import the provided `conan/profiles/default` profile:
@@ -450,33 +457,6 @@ tools.build:cxxflags=['-DBOOST_ASIO_DISABLE_CONCEPTS']
    The location of `rippled` binary in your build directory depends on your
    CMake generator. Pass `--help` to see the rest of the command line options.
 
-#### Conan lockfile
-
-To achieve reproducible dependencies, we use [Conan lockfile](https://docs.conan.io/2/tutorial/versioning/lockfiles.html).
-
-The `conan.lock` file in the repository contains a "snapshot" of the current dependencies.
-It is implicitly used when running `conan` commands, you don't need to specify it.
-
-You have to update this file every time you add a new dependency or change a revision or version of an existing dependency.
-
-> [!NOTE]
-> Conan uses local cache by default when creating a lockfile.
->
-> To ensure, that lockfile creation works the same way on all developer machines, you should clear the local cache before creating a new lockfile.
-
-To create a new lockfile, run the following commands in the repository root:
-
-```bash
-conan remove '*' --confirm
-rm conan.lock
-# This ensure that xrplf remote is the first to be consulted
-conan remote add --force --index 0 xrplf https://conan.ripplex.io
-conan lock create . -o '&:jemalloc=True' -o '&:rocksdb=True'
-```
-
-> [!NOTE]
-> If some dependencies are exclusive for some OS, you may need to run the last command for them adding `--profile:all <PROFILE>`.
-
 ## Coverage report
 
 The coverage report is intended for developers using compilers GCC

conan.lock

@@ -7,31 +7,32 @@
         "soci/4.0.3#a9f8d773cd33e356b5879a4b0564f287%1756234262.318",
         "snappy/1.1.10#968fef506ff261592ec30c574d4a7809%1756234314.246",
         "rocksdb/10.5.1#4a197eca381a3e5ae8adf8cffa5aacd0%1762797952.535",
-        "re2/20230301#dfd6e2bf050eb90ddd8729cfb4c844a4%1756234257.976",
-        "protobuf/3.21.12#d927114e28de9f4691a6bbcdd9a529d1%1756234251.614",
+        "re2/20230301#ca3b241baec15bd31ea9187150e0b333%1764175362.029",
+        "protobuf/3.21.12#44ee56c0a6eea0c19aeeaca680370b88%1764175361.456",
         "openssl/3.5.4#a1d5835cc6ed5c5b8f3cd5b9b5d24205%1760106486.594",
         "nudb/2.0.9#fb8dfd1a5557f5e0528114c2da17721e%1763150366.909",
         "lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504%1756234228.999",
         "libiconv/1.17#1e65319e945f2d31941a9d28cc13c058%1756223727.64",
         "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1756230911.03",
-        "libarchive/3.8.1#5cf685686322e906cb42706ab7e099a8%1756234256.696",
+        "libarchive/3.8.1#ffee18995c706e02bf96e7a2f7042e0d%1764175360.142",
         "jemalloc/5.3.0#e951da9cf599e956cebc117880d2d9f8%1729241615.244",
         "grpc/1.50.1#02291451d1e17200293a409410d1c4e1%1756234248.958",
         "doctest/2.4.12#eb9fb352fb2fdfc8abb17ec270945165%1762797941.757",
-        "date/3.0.4#f74bbba5a08fa388256688743136cb6f%1756234217.493",
-        "c-ares/1.34.5#b78b91e7cfb1f11ce777a285bbf169c6%1756234217.915",
-        "bzip2/1.0.8#00b4a4658791c1f06914e087f0e792f5%1756234261.716",
+        "date/3.0.4#862e11e80030356b53c2c38599ceb32b%1763584497.32",
+        "c-ares/1.34.5#5581c2b62a608b40bb85d965ab3ec7c8%1764175359.429",
+        "bzip2/1.0.8#c470882369c2d95c5c77e970c0c7e321%1764175359.429",
         "boost/1.88.0#8852c0b72ce8271fb8ff7c53456d4983%1756223752.326",
-        "abseil/20230802.1#f0f91485b111dc9837a68972cb19ca7b%1756234220.907"
+        "abseil/20230802.1#90ba607d4ee8fb5fb157c3db540671fc%1764175359.429"
     ],
     "build_requires": [
         "zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1756234269.497",
         "strawberryperl/5.32.1.1#707032463aa0620fa17ec0d887f5fe41%1756234281.733",
-        "protobuf/3.21.12#d927114e28de9f4691a6bbcdd9a529d1%1756234251.614",
+        "protobuf/3.21.12#44ee56c0a6eea0c19aeeaca680370b88%1764175361.456",
         "nasm/2.16.01#31e26f2ee3c4346ecd347911bd126904%1756234232.901",
         "msys2/cci.latest#5b73b10144f73cc5bfe0572ed9be39e1%1751977009.857",
-        "m4/1.4.19#f119296e5c4772b3bb7ab060ae8f417b%1760707875.678",
-        "cmake/3.31.8#dde3bde00bb843687e55aea5afa0e220%1756234232.89",
+        "m4/1.4.19#70dc8bbb33e981d119d2acc0175cf381%1763158052.846",
+        "cmake/4.2.0#ae0a44f44a1ef9ab68fd4b3e9a1f8671%1764175359.44",
+        "cmake/3.31.10#313d16a1aa16bbdb2ca0792467214b76%1764175359.429",
         "b2/5.3.3#107c15377719889654eb9a162a673975%1756234226.28",
         "automake/1.16.5#b91b7c384c3deaa9d535be02da14d04f%1755524470.56",
         "autoconf/2.71#51077f068e61700d65bb05541ea1e4b0%1731054366.86"
@@ -48,14 +49,11 @@
         "boost/1.83.0": [
             "boost/1.88.0"
         ],
-        "sqlite3/[>=3.44 <4]": [
+        "sqlite3/3.44.2": [
             "sqlite3/3.49.1"
         ],
         "lz4/[>=1.9.4 <2]": [
             "lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504"
-        ],
-        "sqlite3/3.44.2": [
-            "sqlite3/3.49.1"
         ]
     },
     "config_requires": []

conan/lockfile/README.md

@@ -0,0 +1,12 @@
+# Conan lockfile
+
+To achieve reproducible dependencies, we use a [Conan lockfile](https://docs.conan.io/2/tutorial/versioning/lockfiles.html).
+
+The `conan.lock` file in the repository contains a "snapshot" of the current
+dependencies. It is implicitly used when running `conan` commands, so you don't
+need to specify it.
+
+You have to update this file every time you add a new dependency or change a
+revision or version of an existing dependency.
+
+To update a lockfile, run from the repository root: `./conan/lockfile/regenerate.sh`

conan/lockfile/linux.profile

@@ -0,0 +1,8 @@
+[settings]
+arch=x86_64
+build_type=Release
+compiler=gcc
+compiler.cppstd=20
+compiler.libcxx=libstdc++11
+compiler.version=13
+os=Linux

conan/lockfile/macos.profile

@@ -0,0 +1,8 @@
+[settings]
+arch=armv8
+build_type=Release
+compiler=apple-clang
+compiler.cppstd=20
+compiler.libcxx=libc++
+compiler.version=17.0
+os=Macos

conan/lockfile/regenerate.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -ex
+
+TEMP_DIR=$(mktemp -d)
+trap "rm -rf $TEMP_DIR" EXIT
+
+echo "Using temporary CONAN_HOME: $TEMP_DIR"
+
+# We use a temporary Conan home to avoid polluting the user's existing Conan
+# configuration and to not use local cache (which leads to non-reproducible lockfiles).
+export CONAN_HOME="$TEMP_DIR"
+
+# Ensure that the xrplf remote is the first to be consulted, so any recipes we
+# patched are used. We also add it there to not created huge diff when the
+# official Conan Center Index is updated.
+conan remote add --force --index 0 xrplf https://conan.ripplex.io
+
+# Delete any existing lockfile.
+rm -f conan.lock
+
+# Create a new lockfile that is compatible with Linux, macOS, and Windows. The
+# first create command will create a new lockfile, while the subsequent create
+# commands will merge any additional dependencies into the created lockfile.
+conan lock create . \
+  --options '&:jemalloc=True' \
+  --options '&:rocksdb=True' \
+  --profile:all=conan/lockfile/linux.profile
+conan lock create . \
+  --options '&:jemalloc=True' \
+  --options '&:rocksdb=True' \
+  --profile:all=conan/lockfile/macos.profile
+conan lock create . \
+  --options '&:jemalloc=True' \
+  --options '&:rocksdb=True' \
+  --profile:all=conan/lockfile/windows.profile

conan/lockfile/windows.profile

@@ -0,0 +1,9 @@
+[settings]
+arch=x86_64
+build_type=Release
+compiler=msvc
+compiler.cppstd=20
+compiler.runtime=dynamic
+compiler.runtime_type=Release
+compiler.version=194
+os=Windows

conanfile.py

@@ -1,149 +1,155 @@
-from conan import ConanFile, __version__ as conan_version
-from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
 import re
 
-class Xrpl(ConanFile):
-    name = 'xrpl'
+from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
 
-    license = 'ISC'
-    author = 'John Freeman <jfreeman@ripple.com>'
-    url = 'https://github.com/xrplf/rippled'
-    description = 'The XRP Ledger'
-    settings = 'os', 'compiler', 'build_type', 'arch'
+from conan import ConanFile
+from conan import __version__ as conan_version
+
+
+class Xrpl(ConanFile):
+    name = "xrpl"
+
+    license = "ISC"
+    author = "John Freeman <jfreeman@ripple.com>"
+    url = "https://github.com/xrplf/rippled"
+    description = "The XRP Ledger"
+    settings = "os", "compiler", "build_type", "arch"
     options = {
-        'assertions': [True, False],
-        'coverage': [True, False],
-        'fPIC': [True, False],
-        'jemalloc': [True, False],
-        'rocksdb': [True, False],
-        'shared': [True, False],
-        'static': [True, False],
-        'tests': [True, False],
-        'unity': [True, False],
-        'xrpld': [True, False],
+        "assertions": [True, False],
+        "coverage": [True, False],
+        "fPIC": [True, False],
+        "jemalloc": [True, False],
+        "rocksdb": [True, False],
+        "shared": [True, False],
+        "static": [True, False],
+        "tests": [True, False],
+        "unity": [True, False],
+        "xrpld": [True, False],
     }
 
     requires = [
-        'grpc/1.50.1',
-        'libarchive/3.8.1',
-        'nudb/2.0.9',
-        'openssl/3.5.4',
-        'soci/4.0.3',
-        'zlib/1.3.1',
+        "grpc/1.50.1",
+        "libarchive/3.8.1",
+        "nudb/2.0.9",
+        "openssl/3.5.4",
+        "soci/4.0.3",
+        "zlib/1.3.1",
     ]
 
     test_requires = [
-        'doctest/2.4.12',
+        "doctest/2.4.12",
     ]
 
     tool_requires = [
-        'protobuf/3.21.12',
+        "protobuf/3.21.12",
     ]
 
     default_options = {
-        'assertions': False,
-        'coverage': False,
-        'fPIC': True,
-        'jemalloc': False,
-        'rocksdb': True,
-        'shared': False,
-        'static': True,
-        'tests': False,
-        'unity': False,
-        'xrpld': False,
-
-        'date/*:header_only': True,
-        'grpc/*:shared': False,
-        'grpc/*:secure': True,
-        'libarchive/*:shared': False,
-        'libarchive/*:with_acl': False,
-        'libarchive/*:with_bzip2': False,
-        'libarchive/*:with_cng': False,
-        'libarchive/*:with_expat': False,
-        'libarchive/*:with_iconv': False,
-        'libarchive/*:with_libxml2': False,
-        'libarchive/*:with_lz4': True,
-        'libarchive/*:with_lzma': False,
-        'libarchive/*:with_lzo': False,
-        'libarchive/*:with_nettle': False,
-        'libarchive/*:with_openssl': False,
-        'libarchive/*:with_pcreposix': False,
-        'libarchive/*:with_xattr': False,
-        'libarchive/*:with_zlib': False,
-        'lz4/*:shared': False,
-        'openssl/*:shared': False,
-        'protobuf/*:shared': False,
-        'protobuf/*:with_zlib': True,
-        'rocksdb/*:enable_sse': False,
-        'rocksdb/*:lite': False,
-        'rocksdb/*:shared': False,
-        'rocksdb/*:use_rtti': True,
-        'rocksdb/*:with_jemalloc': False,
-        'rocksdb/*:with_lz4': True,
-        'rocksdb/*:with_snappy': True,
-        'snappy/*:shared': False,
-        'soci/*:shared': False,
-        'soci/*:with_sqlite3': True,
-        'soci/*:with_boost': True,
-        'xxhash/*:shared': False,
+        "assertions": False,
+        "coverage": False,
+        "fPIC": True,
+        "jemalloc": False,
+        "rocksdb": True,
+        "shared": False,
+        "static": True,
+        "tests": False,
+        "unity": False,
+        "xrpld": False,
+        "date/*:header_only": True,
+        "grpc/*:shared": False,
+        "grpc/*:secure": True,
+        "libarchive/*:shared": False,
+        "libarchive/*:with_acl": False,
+        "libarchive/*:with_bzip2": False,
+        "libarchive/*:with_cng": False,
+        "libarchive/*:with_expat": False,
+        "libarchive/*:with_iconv": False,
+        "libarchive/*:with_libxml2": False,
+        "libarchive/*:with_lz4": True,
+        "libarchive/*:with_lzma": False,
+        "libarchive/*:with_lzo": False,
+        "libarchive/*:with_nettle": False,
+        "libarchive/*:with_openssl": False,
+        "libarchive/*:with_pcreposix": False,
+        "libarchive/*:with_xattr": False,
+        "libarchive/*:with_zlib": False,
+        "lz4/*:shared": False,
+        "openssl/*:shared": False,
+        "protobuf/*:shared": False,
+        "protobuf/*:with_zlib": True,
+        "rocksdb/*:enable_sse": False,
+        "rocksdb/*:lite": False,
+        "rocksdb/*:shared": False,
+        "rocksdb/*:use_rtti": True,
+        "rocksdb/*:with_jemalloc": False,
+        "rocksdb/*:with_lz4": True,
+        "rocksdb/*:with_snappy": True,
+        "snappy/*:shared": False,
+        "soci/*:shared": False,
+        "soci/*:with_sqlite3": True,
+        "soci/*:with_boost": True,
+        "xxhash/*:shared": False,
     }
 
     def set_version(self):
         if self.version is None:
-            path = f'{self.recipe_folder}/src/libxrpl/protocol/BuildInfo.cpp'
-            regex = r'versionString\s?=\s?\"(.*)\"'
-            with open(path, encoding='utf-8') as file:
+            path = f"{self.recipe_folder}/src/libxrpl/protocol/BuildInfo.cpp"
+            regex = r"versionString\s?=\s?\"(.*)\""
+            with open(path, encoding="utf-8") as file:
                 matches = (re.search(regex, line) for line in file)
                 match = next(m for m in matches if m)
                 self.version = match.group(1)
 
     def configure(self):
-        if self.settings.compiler == 'apple-clang':
-            self.options['boost'].visibility = 'global'
-        if self.settings.compiler in ['clang', 'gcc']:
-            self.options['boost'].without_cobalt = True
+        if self.settings.compiler == "apple-clang":
+            self.options["boost"].visibility = "global"
+        if self.settings.compiler in ["clang", "gcc"]:
+            self.options["boost"].without_cobalt = True
 
     def requirements(self):
         # Conan 2 requires transitive headers to be specified
-        transitive_headers_opt = {'transitive_headers': True} if conan_version.split('.')[0] == '2' else {}
-        self.requires('boost/1.88.0', force=True, **transitive_headers_opt)
-        self.requires('date/3.0.4', **transitive_headers_opt)
-        self.requires('lz4/1.10.0', force=True)
-        self.requires('protobuf/3.21.12', force=True)
-        self.requires('sqlite3/3.49.1', force=True)
+        transitive_headers_opt = (
+            {"transitive_headers": True} if conan_version.split(".")[0] == "2" else {}
+        )
+        self.requires("boost/1.88.0", force=True, **transitive_headers_opt)
+        self.requires("date/3.0.4", **transitive_headers_opt)
+        self.requires("lz4/1.10.0", force=True)
+        self.requires("protobuf/3.21.12", force=True)
+        self.requires("sqlite3/3.49.1", force=True)
         if self.options.jemalloc:
-            self.requires('jemalloc/5.3.0')
+            self.requires("jemalloc/5.3.0")
         if self.options.rocksdb:
-            self.requires('rocksdb/10.5.1')
-        self.requires('xxhash/0.8.3', **transitive_headers_opt)
+            self.requires("rocksdb/10.5.1")
+        self.requires("xxhash/0.8.3", **transitive_headers_opt)
 
     exports_sources = (
-        'CMakeLists.txt',
-        'cfg/*',
-        'cmake/*',
-        'external/*',
-        'include/*',
-        'src/*',
+        "CMakeLists.txt",
+        "cfg/*",
+        "cmake/*",
+        "external/*",
+        "include/*",
+        "src/*",
     )
 
     def layout(self):
         cmake_layout(self)
         # Fix this setting to follow the default introduced in Conan 1.48
         # to align with our build instructions.
-        self.folders.generators = 'build/generators'
+        self.folders.generators = "build/generators"
+
+    generators = "CMakeDeps"
 
-    generators = 'CMakeDeps'
     def generate(self):
         tc = CMakeToolchain(self)
-        tc.variables['tests'] = self.options.tests
-        tc.variables['assert'] = self.options.assertions
-        tc.variables['coverage'] = self.options.coverage
-        tc.variables['jemalloc'] = self.options.jemalloc
-        tc.variables['rocksdb'] = self.options.rocksdb
-        tc.variables['BUILD_SHARED_LIBS'] = self.options.shared
-        tc.variables['static'] = self.options.static
-        tc.variables['unity'] = self.options.unity
-        tc.variables['xrpld'] = self.options.xrpld
+        tc.variables["tests"] = self.options.tests
+        tc.variables["assert"] = self.options.assertions
+        tc.variables["coverage"] = self.options.coverage
+        tc.variables["jemalloc"] = self.options.jemalloc
+        tc.variables["rocksdb"] = self.options.rocksdb
+        tc.variables["BUILD_SHARED_LIBS"] = self.options.shared
+        tc.variables["static"] = self.options.static
+        tc.variables["unity"] = self.options.unity
+        tc.variables["xrpld"] = self.options.xrpld
         tc.generate()
 
     def build(self):
@@ -158,40 +164,40 @@ class Xrpl(ConanFile):
         cmake.install()
 
     def package_info(self):
-        libxrpl = self.cpp_info.components['libxrpl']
+        libxrpl = self.cpp_info.components["libxrpl"]
         libxrpl.libs = [
-            'xrpl',
-            'xrpl.libpb',
-            'ed25519',
-            'secp256k1',
+            "xrpl",
+            "xrpl.libpb",
+            "ed25519",
+            "secp256k1",
         ]
         # TODO: Fix the protobufs to include each other relative to
         # `include/`, not `include/ripple/proto/`.
-        libxrpl.includedirs = ['include', 'include/ripple/proto']
+        libxrpl.includedirs = ["include", "include/ripple/proto"]
         libxrpl.requires = [
-            'boost::headers',
-            'boost::chrono',
-            'boost::container',
-            'boost::coroutine',
-            'boost::date_time',
-            'boost::filesystem',
-            'boost::json',
-            'boost::program_options',
-            'boost::process',
-            'boost::regex',
-            'boost::system',
-            'boost::thread',
-            'date::date',
-            'grpc::grpc++',
-            'libarchive::libarchive',
-            'lz4::lz4',
-            'nudb::nudb',
-            'openssl::crypto',
-            'protobuf::libprotobuf',
-            'soci::soci',
-            'sqlite3::sqlite',
-            'xxhash::xxhash',
-            'zlib::zlib',
+            "boost::headers",
+            "boost::chrono",
+            "boost::container",
+            "boost::coroutine",
+            "boost::date_time",
+            "boost::filesystem",
+            "boost::json",
+            "boost::program_options",
+            "boost::process",
+            "boost::regex",
+            "boost::system",
+            "boost::thread",
+            "date::date",
+            "grpc::grpc++",
+            "libarchive::libarchive",
+            "lz4::lz4",
+            "nudb::nudb",
+            "openssl::crypto",
+            "protobuf::libprotobuf",
+            "soci::soci",
+            "sqlite3::sqlite",
+            "xxhash::xxhash",
+            "zlib::zlib",
         ]
         if self.options.rocksdb:
-            libxrpl.requires.append('rocksdb::librocksdb')
+            libxrpl.requires.append("rocksdb::librocksdb")

src/tests/libxrpl/net/HTTPClient.cpp

@@ -10,7 +10,6 @@
 #include <doctest/doctest.h>
 
 #include <atomic>
-#include <iostream>
 #include <map>
 #include <thread>
 
@@ -18,40 +17,6 @@ using namespace ripple;
 
 namespace {
 
-struct logger
-{
-    std::string name;
-    logger const* const parent = nullptr;
-    static std::size_t depth;
-    logger(std::string n) : name(n)
-    {
-        std::clog << indent() << name << " begin\n";
-        ++depth;
-    }
-
-    logger(logger const& p, std::string n) : parent(&p), name(n)
-    {
-        std::clog << indent() << parent->name << " : " << name << " begin\n";
-        ++depth;
-    }
-
-    ~logger()
-    {
-        --depth;
-        if (parent)
-            std::clog << indent() << parent->name << " : " << name << " end\n";
-        else
-            std::clog << indent() << name << " end\n";
-    }
-
-    std::string
-    indent()
-    {
-        return std::string(depth, ' ');
-    }
-};
-std::size_t logger::depth = 0;
-
 // Simple HTTP server using Beast for testing
 class TestHTTPServer
 {
@@ -70,7 +35,6 @@ private:
 public:
     TestHTTPServer() : acceptor_(ioc_), port_(0)
     {
-        logger l("TestHTTPServer()");
         // Bind to any available port
         endpoint_ = {boost::asio::ip::tcp::v4(), 0};
         acceptor_.open(endpoint_.protocol());
@@ -86,7 +50,6 @@ public:
 
     ~TestHTTPServer()
     {
-        logger l("~TestHTTPServer()");
         stop();
     }
 
@@ -124,7 +87,6 @@ private:
     void
     stop()
     {
-        logger l("TestHTTPServer::stop");
         running_ = false;
         acceptor_.close();
     }
@@ -132,7 +94,6 @@ private:
     void
     accept()
     {
-        logger l("TestHTTPServer::accept");
         if (!running_)
             return;
 
@@ -154,37 +115,31 @@ private:
     void
     handleConnection(boost::asio::ip::tcp::socket socket)
     {
-        logger l("TestHTTPServer::handleConnection");
         try
         {
-            std::optional<logger> r(std::in_place, l, "read the http request");
             // Read the HTTP request
             boost::beast::flat_buffer buffer;
             boost::beast::http::request<boost::beast::http::string_body> req;
             boost::beast::http::read(socket, buffer, req);
 
             // Create response
-            r.emplace(l, "create response");
             boost::beast::http::response<boost::beast::http::string_body> res;
             res.version(req.version());
             res.result(status_code_);
             res.set(boost::beast::http::field::server, "TestServer");
 
             // Add custom headers
-            r.emplace(l, "add custom headers");
             for (auto const& [name, value] : custom_headers_)
             {
                 res.set(name, value);
             }
 
             // Set body and prepare payload first
-            r.emplace(l, "set body and prepare payload");
             res.body() = response_body_;
             res.prepare_payload();
 
             // Override Content-Length with custom headers after prepare_payload
             // This allows us to test case-insensitive header parsing
-            r.emplace(l, "override content-length");
             for (auto const& [name, value] : custom_headers_)
             {
                 if (boost::iequals(name, "Content-Length"))
@@ -195,25 +150,19 @@ private:
             }
 
             // Send response
-            r.emplace(l, "send response");
             boost::beast::http::write(socket, res);
 
             // Shutdown socket gracefully
-            r.emplace(l, "shutdown socket");
             boost::system::error_code ec;
             socket.shutdown(boost::asio::ip::tcp::socket::shutdown_send, ec);
         }
         catch (std::exception const&)
         {
             // Connection handling errors are expected
-            logger c(l, "catch");
         }
 
         if (running_)
-        {
-            logger r(l, "accept");
             accept();
-        }
     }
 };
 
@@ -227,16 +176,12 @@ runHTTPTest(
     std::string& result_data,
     boost::system::error_code& result_error)
 {
-    logger l("runHTTPTest");
     // Create a null journal for testing
     beast::Journal j{beast::Journal::getNullSink()};
 
-    std::optional<logger> r(std::in_place, l, "initializeSSLContext");
-
     // Initialize HTTPClient SSL context
     HTTPClient::initializeSSLContext("", "", false, j);
 
-    r.emplace(l, "HTTPClient::get");
     HTTPClient::get(
         false,  // no SSL
         server.ioc(),
@@ -261,7 +206,6 @@ runHTTPTest(
     while (!completed &&
            std::chrono::steady_clock::now() - start < std::chrono::seconds(10))
     {
-        r.emplace(l, "ioc.run_one");
         if (server.ioc().run_one() == 0)
         {
             break;
@@ -275,8 +219,6 @@ runHTTPTest(
 
 TEST_CASE("HTTPClient case insensitive Content-Length")
 {
-    logger l("HTTPClient case insensitive Content-Length");
-
     // Test different cases of Content-Length header
     std::vector<std::string> header_cases = {
         "Content-Length",  // Standard case
@@ -288,7 +230,6 @@ TEST_CASE("HTTPClient case insensitive Content-Length")
 
     for (auto const& header_name : header_cases)
     {
-        logger h(l, header_name);
         TestHTTPServer server;
         std::string test_body = "Hello World!";
         server.setResponseBody(test_body);
@@ -317,7 +258,6 @@ TEST_CASE("HTTPClient case insensitive Content-Length")
 
 TEST_CASE("HTTPClient basic HTTP request")
 {
-    logger l("HTTPClient basic HTTP request");
     TestHTTPServer server;
     std::string test_body = "Test response body";
     server.setResponseBody(test_body);
@@ -339,7 +279,6 @@ TEST_CASE("HTTPClient basic HTTP request")
 
 TEST_CASE("HTTPClient empty response")
 {
-    logger l("HTTPClient empty response");
     TestHTTPServer server;
     server.setResponseBody("");  // Empty body
     server.setHeader("Content-Length", "0");
@@ -360,7 +299,6 @@ TEST_CASE("HTTPClient empty response")
 
 TEST_CASE("HTTPClient different status codes")
 {
-    logger l("HTTPClient different status codes");
     std::vector<unsigned int> status_codes = {200, 404, 500};
 
     for (auto status : status_codes)

src/xrpld/app/misc/detail/ValidatorSite.cpp

@@ -129,7 +129,12 @@ ValidatorSite::load(
     {
         try
         {
-            sites_.emplace_back(uri);
+            // This is not super efficient, but it doesn't happen often.
+            bool found = std::ranges::any_of(sites_, [&uri](auto const& site) {
+                return site.loadedResource->uri == uri;
+            });
+            if (!found)
+                sites_.emplace_back(uri);
         }
         catch (std::exception const& e)
         {
@@ -191,6 +196,17 @@ ValidatorSite::setTimer(
     std::lock_guard<std::mutex> const& site_lock,
     std::lock_guard<std::mutex> const& state_lock)
 {
+    if (!sites_.empty() &&  //
+        std::ranges::all_of(sites_, [](auto const& site) {
+            return site.lastRefreshStatus.has_value();
+        }))
+    {
+        // If all of the sites have been handled at least once (including
+        // errors and timeouts), call missingSite, which will load the cache
+        // files for any lists that are still unavailable.
+        missingSite(site_lock);
+    }
+
     auto next = std::min_element(
         sites_.begin(), sites_.end(), [](Site const& a, Site const& b) {
             return a.nextRefresh < b.nextRefresh;
@@ -303,13 +319,16 @@ ValidatorSite::onRequestTimeout(std::size_t siteIdx, error_code const& ec)
         // processes a network error. Usually, this function runs first,
         // but on extremely rare occasions, the response handler can run
         // first, which will leave activeResource empty.
-        auto const& site = sites_[siteIdx];
+        auto& site = sites_[siteIdx];
         if (site.activeResource)
             JLOG(j_.warn()) << "Request for " << site.activeResource->uri
                             << " took too long";
         else
             JLOG(j_.error()) << "Request took too long, but a response has "
                                 "already been processed";
+        if (!site.lastRefreshStatus)
+            site.lastRefreshStatus.emplace(Site::Status{
+                clock_type::now(), ListDisposition::invalid, "timeout"});
     }
 
     std::lock_guard lock_state{state_mutex_};

tests/conan/conanfile.py

@@ -1,28 +1,29 @@
 from pathlib import Path
 
-from conan import ConanFile
 from conan.tools.build import can_run
 from conan.tools.cmake import CMake, cmake_layout
 
+from conan import ConanFile
+
+
 class Example(ConanFile):
+    name = "example"
+    license = "ISC"
+    author = "John Freeman <jfreeman08@gmail.com>, Michael Legleux <mlegleux@ripple.com"
 
-    name = 'example'
-    license = 'ISC'
-    author = 'John Freeman <jfreeman08@gmail.com>, Michael Legleux <mlegleux@ripple.com'
-
-    settings = 'os', 'compiler', 'build_type', 'arch'
+    settings = "os", "compiler", "build_type", "arch"
 
     requires = ["xrpl/head"]
 
     default_options = {
-        'xrpl/*:xrpld': False,
+        "xrpl/*:xrpld": False,
     }
 
-    generators = ['CMakeDeps', 'CMakeToolchain']
+    generators = ["CMakeDeps", "CMakeToolchain"]
 
     def set_version(self):
         if self.version is None:
-            self.version = '0.1.0'
+            self.version = "0.1.0"
 
     def layout(self):
         cmake_layout(self)