Merge branch 'develop' into ximinez/fix/validator-cache

- If any [validator_list_keys] are not available after all
  [validator_list_sites] have had a chance to be queried, then fall
  back to loading cache files. Currently, cache files are only used if
  no sites are defined, or the request to one of them has an error. It
  does not include cases where not enough sites are defined, or if a
  site returns an invalid VL (or something else entirely).
- Resolves #5320

.github/scripts/strategy-matrix/generate.py

@@ -51,20 +51,22 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
         # Only generate a subset of configurations in PRs.
         if not all:
             # Debian:
-            # - Bookworm using GCC 13: Release on linux/amd64, set the reference
-            #   fee to 500.
-            # - Bookworm using GCC 15: Debug on linux/amd64, enable code
-            #   coverage (which will be done below).
-            # - Bookworm using Clang 16: Debug on linux/arm64, enable voidstar.
-            # - Bookworm using Clang 17: Release on linux/amd64, set the
-            #   reference fee to 1000.
-            # - Bookworm using Clang 20: Debug on linux/amd64.
+            # - Bookworm using GCC 13: Release and Unity on linux/amd64, set
+            #   the reference fee to 500.
+            # - Bookworm using GCC 15: Debug and no Unity on linux/amd64, enable
+            #   code coverage (which will be done below).
+            # - Bookworm using Clang 16: Debug and no Unity on linux/arm64,
+            #   enable voidstar.
+            # - Bookworm using Clang 17: Release and no Unity on linux/amd64,
+            #   set the reference fee to 1000.
+            # - Bookworm using Clang 20: Debug and Unity on linux/amd64.
             if os["distro_name"] == "debian":
                 skip = True
                 if os["distro_version"] == "bookworm":
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-13"
                         and build_type == "Release"
+                        and "-Dunity=ON" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=500 {cmake_args}"
@@ -72,12 +74,14 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15"
                         and build_type == "Debug"
+                        and "-Dunity=OFF" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         skip = False
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-16"
                         and build_type == "Debug"
+                        and "-Dunity=OFF" in cmake_args
                         and architecture["platform"] == "linux/arm64"
                     ):
                         cmake_args = f"-Dvoidstar=ON {cmake_args}"
@@ -85,6 +89,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-17"
                         and build_type == "Release"
+                        and "-Dunity=ON" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=1000 {cmake_args}"
@@ -92,6 +97,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-20"
                         and build_type == "Debug"
+                        and "-Dunity=ON" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         skip = False
@@ -99,14 +105,15 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     continue
 
             # RHEL:
-            # - 9 using GCC 12: Debug on linux/amd64.
-            # - 10 using Clang: Release on linux/amd64.
+            # - 9 using GCC 12: Debug and Unity on linux/amd64.
+            # - 10 using Clang: Release and no Unity on linux/amd64.
             if os["distro_name"] == "rhel":
                 skip = True
                 if os["distro_version"] == "9":
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12"
                         and build_type == "Debug"
+                        and "-Dunity=ON" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         skip = False
@@ -114,6 +121,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-any"
                         and build_type == "Release"
+                        and "-Dunity=OFF" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         skip = False
@@ -121,16 +129,17 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     continue
 
             # Ubuntu:
-            # - Jammy using GCC 12: Debug on linux/arm64.
-            # - Noble using GCC 14: Release on linux/amd64.
-            # - Noble using Clang 18: Debug on linux/amd64.
-            # - Noble using Clang 19: Release on linux/arm64.
+            # - Jammy using GCC 12: Debug and no Unity on linux/arm64.
+            # - Noble using GCC 14: Release and Unity on linux/amd64.
+            # - Noble using Clang 18: Debug and no Unity on linux/amd64.
+            # - Noble using Clang 19: Release and Unity on linux/arm64.
             if os["distro_name"] == "ubuntu":
                 skip = True
                 if os["distro_version"] == "jammy":
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12"
                         and build_type == "Debug"
+                        and "-Dunity=OFF" in cmake_args
                         and architecture["platform"] == "linux/arm64"
                     ):
                         skip = False
@@ -138,18 +147,21 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-14"
                         and build_type == "Release"
+                        and "-Dunity=ON" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         skip = False
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-18"
                         and build_type == "Debug"
+                        and "-Dunity=OFF" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         skip = False
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-19"
                         and build_type == "Release"
+                        and "-Dunity=ON" in cmake_args
                         and architecture["platform"] == "linux/arm64"
                     ):
                         skip = False
@@ -157,16 +169,20 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     continue
 
             # MacOS:
-            # - Debug on macos/arm64.
+            # - Debug and no Unity on macos/arm64.
             if os["distro_name"] == "macos" and not (
-                build_type == "Debug" and architecture["platform"] == "macos/arm64"
+                build_type == "Debug"
+                and "-Dunity=OFF" in cmake_args
+                and architecture["platform"] == "macos/arm64"
             ):
                 continue
 
             # Windows:
-            # - Release on windows/amd64.
+            # - Release and Unity on windows/amd64.
             if os["distro_name"] == "windows" and not (
-                build_type == "Release" and architecture["platform"] == "windows/amd64"
+                build_type == "Release"
+                and "-Dunity=ON" in cmake_args
+                and architecture["platform"] == "windows/amd64"
             ):
                 continue
 
@@ -193,17 +209,18 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
         ):
             continue
 
-        # Enable code coverage for Debian Bookworm using GCC 15 in Debug on
-        # linux/amd64
+        # Enable code coverage for Debian Bookworm using GCC 15 in Debug and no
+        # Unity on linux/amd64
         if (
             f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15"
             and build_type == "Debug"
+            and "-Dunity=OFF" in cmake_args
             and architecture["platform"] == "linux/amd64"
         ):
             cmake_args = f"-Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0 {cmake_args}"
 
         # Generate a unique name for the configuration, e.g. macos-arm64-debug
-        # or debian-bookworm-gcc-12-amd64-release.
+        # or debian-bookworm-gcc-12-amd64-release-unity.
         config_name = os["distro_name"]
         if (n := os["distro_version"]) != "":
             config_name += f"-{n}"
@@ -217,6 +234,8 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
         config_name += f"-{build_type.lower()}"
         if "-Dcoverage=ON" in cmake_args:
             config_name += "-coverage"
+        if "-Dunity=ON" in cmake_args:
+            config_name += "-unity"
 
         # Add the configuration to the list, with the most unique fields first,
         # so that they are easier to identify in the GitHub Actions UI, as long

.github/scripts/strategy-matrix/linux.json

@@ -208,5 +208,5 @@
     }
   ],
   "build_type": ["Debug", "Release"],
-  "cmake_args": [""]
+  "cmake_args": ["-Dunity=OFF", "-Dunity=ON"]
 }

.github/scripts/strategy-matrix/macos.json

@@ -15,5 +15,8 @@
     }
   ],
   "build_type": ["Debug", "Release"],
-  "cmake_args": ["-DCMAKE_POLICY_VERSION_MINIMUM=3.5"]
+  "cmake_args": [
+    "-Dunity=OFF -DCMAKE_POLICY_VERSION_MINIMUM=3.5",
+    "-Dunity=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5"
+  ]
 }

.github/scripts/strategy-matrix/windows.json

@@ -15,5 +15,5 @@
     }
   ],
   "build_type": ["Debug", "Release"],
-  "cmake_args": [""]
+  "cmake_args": ["-Dunity=OFF", "-Dunity=ON"]
 }

BUILD.md

@@ -368,36 +368,6 @@ The workaround for this error is to add two lines to your profile:
 tools.build:cxxflags=['-DBOOST_ASIO_DISABLE_CONCEPTS']
 ```
 
-### Set Up Ccache
-
-To speed up repeated compilations, we recommend that you install
-[ccache](https://ccache.dev), a tool that wraps your compiler so that it can
-cache build objects locally.
-
-#### Linux
-
-You can install it using the package manager, e.g. `sudo apt install ccache`
-(Ubuntu) or `sudo dnf install ccache` (RHEL).
-
-#### macOS
-
-You can install it using Homebrew, i.e. `brew install ccache`.
-
-#### Windows
-
-You can install it using Chocolatey, i.e. `choco install ccache`. If you already
-have Ccache installed, then `choco upgrade ccache` will update it to the latest
-version. However, if you see an error such as:
-
-```
-terminate called after throwing an instance of 'std::bad_alloc'
-      what():  std::bad_alloc
-C:\Program Files\Microsoft Visual Studio\2022\Community\MSBuild\Microsoft\VC\v170\Microsoft.CppCommon.targets(617,5): error MSB6006: "cl.exe" exited with code 3.
-```
-
-then please install a specific version of Ccache that we know works, via: `choco
-install ccache --version 4.11.3 --allow-downgrade`.
-
 ### Build and Test
 
 1. Create a build directory and move into it.
@@ -575,10 +545,16 @@ See [Sanitizers docs](./docs/build/sanitizers.md) for more details.
 | `assert`   | OFF           | Enable assertions.                                             |
 | `coverage` | OFF           | Prepare the coverage report.                                   |
 | `tests`    | OFF           | Build tests.                                                   |
+| `unity`    | OFF           | Configure a unity build.                                       |
 | `xrpld`    | OFF           | Build the xrpld application, and not just the libxrpl library. |
 | `werr`     | OFF           | Treat compilation warnings as errors                           |
 | `wextra`   | OFF           | Enable additional compilation warnings                         |
 
+[Unity builds][5] may be faster for the first build
+(at the cost of much more memory) since they concatenate sources into fewer
+translation units. Non-unity builds may be faster for incremental builds,
+and can be helpful for detecting `#include` omissions.
+
 ## Troubleshooting
 
 ### Conan
@@ -645,6 +621,7 @@ If you want to experiment with a new package, follow these steps:
 [1]: https://github.com/conan-io/conan-center-index/issues/13168
 [2]: https://en.cppreference.com/w/cpp/compiler_support/20
 [3]: https://docs.conan.io/en/latest/getting_started.html
+[5]: https://en.wikipedia.org/wiki/Unity_build
 [6]: https://github.com/boostorg/beast/issues/2648
 [7]: https://github.com/boostorg/beast/issues/2661
 [gcovr]: https://gcovr.com/en/stable/getting-started.html

cmake/XrplAddTest.cmake

@@ -9,5 +9,8 @@ function (xrpl_add_test name)
 
     isolate_headers(${target} "${CMAKE_SOURCE_DIR}" "${CMAKE_SOURCE_DIR}/tests/${name}" PRIVATE)
 
+    # Make sure the test isn't optimized away in unity builds
+    set_target_properties(${target} PROPERTIES UNITY_BUILD_MODE GROUP UNITY_BUILD_BATCH_SIZE 0) # Adjust as needed
+
     add_test(NAME ${target} COMMAND ${target})
 endfunction ()

cmake/XrplCore.cmake

@@ -4,7 +4,12 @@
 
 include(target_protobuf_sources)
 
+# Protocol buffers cannot participate in a unity build,
+# because all the generated sources
+# define a bunch of `static const` variables with the same names,
+# so we just build them as a separate library.
 add_library(xrpl.libpb)
+set_target_properties(xrpl.libpb PROPERTIES UNITY_BUILD OFF)
 target_protobuf_sources(xrpl.libpb xrpl/proto LANGUAGE cpp IMPORT_DIRS include/xrpl/proto
                         PROTOS include/xrpl/proto/xrpl.proto)
 
@@ -155,4 +160,12 @@ if (xrpld)
         # antithesis_instrumentation.h, which is not exported as INTERFACE
         target_include_directories(xrpld PRIVATE ${CMAKE_SOURCE_DIR}/external/antithesis-sdk)
     endif ()
+
+    # any files that don't play well with unity should be added here
+    if (tests)
+        set_source_files_properties(
+            # these two seem to produce conflicts in beast teardown template methods
+            src/test/rpc/ValidatorRPC_test.cpp src/test/ledger/Invariants_test.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION
+                                                                                              TRUE)
+    endif ()
 endif ()

cmake/XrplSettings.cmake

@@ -30,6 +30,14 @@ if (tests)
     endif ()
 endif ()
 
+option(unity "Creates a build using UNITY support in cmake." OFF)
+if (unity)
+    if (NOT is_ci)
+        set(CMAKE_UNITY_BUILD_BATCH_SIZE 15 CACHE STRING "")
+    endif ()
+    set(CMAKE_UNITY_BUILD ON CACHE BOOL "Do a unity build")
+endif ()
+
 if (is_clang AND is_linux)
     option(voidstar "Enable Antithesis instrumentation." OFF)
 endif ()

conanfile.py

@@ -23,6 +23,7 @@ class Xrpl(ConanFile):
         "shared": [True, False],
         "static": [True, False],
         "tests": [True, False],
+        "unity": [True, False],
         "xrpld": [True, False],
     }
 
@@ -54,6 +55,7 @@ class Xrpl(ConanFile):
         "shared": False,
         "static": True,
         "tests": False,
+        "unity": False,
         "xrpld": False,
         "date/*:header_only": True,
         "ed25519/*:shared": False,
@@ -166,6 +168,7 @@ class Xrpl(ConanFile):
         tc.variables["rocksdb"] = self.options.rocksdb
         tc.variables["BUILD_SHARED_LIBS"] = self.options.shared
         tc.variables["static"] = self.options.static
+        tc.variables["unity"] = self.options.unity
         tc.variables["xrpld"] = self.options.xrpld
         tc.generate()
 

include/xrpl/basics/rocksdb.h

@@ -1,5 +1,5 @@
-#ifndef XRPL_BASICS_ROCKSDB_H_INCLUDED
-#define XRPL_BASICS_ROCKSDB_H_INCLUDED
+#ifndef XRPL_UNITY_ROCKSDB_H_INCLUDED
+#define XRPL_UNITY_ROCKSDB_H_INCLUDED
 
 #if XRPL_ROCKSDB_AVAILABLE
 // #include <rocksdb2/port/port_posix.h>

include/xrpl/protocol/detail/features.macro

@@ -15,7 +15,7 @@
 
 // Add new amendments to the top of this list.
 // Keep it sorted in reverse chronological order.
-XRPL_FIX    (LendingProtocolV1_1,        Supported::yes, VoteBehavior::DefaultNo)
+
 XRPL_FIX    (ExpiredNFTokenOfferRemoval, Supported::yes, VoteBehavior::DefaultNo)
 XRPL_FIX    (BatchInnerSigs,             Supported::yes, VoteBehavior::DefaultNo)
 XRPL_FEATURE(LendingProtocol,            Supported::yes, VoteBehavior::DefaultNo)

src/libxrpl/net/RegisterSSLCerts.cpp

@@ -85,8 +85,7 @@ registerSSLCerts(boost::asio::ssl::context& ctx, boost::system::error_code& ec,
 // There is a very unpleasant interaction between <wincrypt> and
 // openssl x509 types (namely the former has macros that stomp
 // on the latter), these undefs allow this TU to be safely used in
-// unity builds without messing up subsequent TUs. Although we
-// no longer use unity builds, leaving the undefs here does no harm.
+// unity builds without messing up subsequent TUs.
 #if BOOST_OS_WINDOWS
 #undef X509_NAME
 #undef X509_EXTENSIONS

src/test/app/LoanBroker_test.cpp

@@ -1701,21 +1701,10 @@ class LoanBroker_test : public beast::unit_test::suite
         testRIPD4274MPT();
     }
 
-    void
-    testFixAmendmentEnabled()
-    {
-        using namespace jtx;
-        testcase("testFixAmendmentEnabled");
-        Env env{*this};
-
-        BEAST_EXPECT(env.enabled(fixLendingProtocolV1_1));
-    }
-
 public:
     void
     run() override
     {
-        testFixAmendmentEnabled();
         testLoanBrokerSetDebtMaximum();
         testLoanBrokerCoverDepositNullVault();
 

src/xrpld/app/misc/detail/ValidatorSite.cpp

@@ -124,7 +124,11 @@ ValidatorSite::load(std::vector<std::string> const& siteURIs, std::lock_guard<st
     {
         try
         {
-            sites_.emplace_back(uri);
+            // This is not super efficient, but it doesn't happen often.
+            bool found =
+                std::ranges::any_of(sites_, [&uri](auto const& site) { return site.loadedResource->uri == uri; });
+            if (!found)
+                sites_.emplace_back(uri);
         }
         catch (std::exception const& e)
         {
@@ -183,6 +187,15 @@ ValidatorSite::stop()
 void
 ValidatorSite::setTimer(std::lock_guard<std::mutex> const& site_lock, std::lock_guard<std::mutex> const& state_lock)
 {
+    if (!sites_.empty() &&  //
+        std::ranges::all_of(sites_, [](auto const& site) { return site.lastRefreshStatus.has_value(); }))
+    {
+        // If all of the sites have been handled at least once (including
+        // errors and timeouts), call missingSite, which will load the cache
+        // files for any lists that are still unavailable.
+        missingSite(site_lock);
+    }
+
     auto next = std::min_element(
         sites_.begin(), sites_.end(), [](Site const& a, Site const& b) { return a.nextRefresh < b.nextRefresh; });
 
@@ -285,12 +298,14 @@ ValidatorSite::onRequestTimeout(std::size_t siteIdx, error_code const& ec)
         // processes a network error. Usually, this function runs first,
         // but on extremely rare occasions, the response handler can run
         // first, which will leave activeResource empty.
-        auto const& site = sites_[siteIdx];
+        auto& site = sites_[siteIdx];
         if (site.activeResource)
             JLOG(j_.warn()) << "Request for " << site.activeResource->uri << " took too long";
         else
             JLOG(j_.error()) << "Request took too long, but a response has "
                                 "already been processed";
+        if (!site.lastRefreshStatus)
+            site.lastRefreshStatus.emplace(Site::Status{clock_type::now(), ListDisposition::invalid, "timeout"});
     }
 
     std::lock_guard lock_state{state_mutex_};