Merge branch 'develop' into ripple/wasmi

* Use conan repo for wasmi lib
* Generate lockfile

.config/cspell.config.yaml

@@ -272,6 +272,7 @@ words:
   - venv
   - vfalco
   - vinnie
+  - wasmi
   - wextra
   - wptr
   - writeme

.github/scripts/strategy-matrix/generate.py

@@ -51,20 +51,22 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
         # Only generate a subset of configurations in PRs.
         if not all:
             # Debian:
-            # - Bookworm using GCC 13: Release on linux/amd64, set the reference
-            #   fee to 500.
-            # - Bookworm using GCC 15: Debug on linux/amd64, enable code
-            #   coverage (which will be done below).
-            # - Bookworm using Clang 16: Debug on linux/arm64, enable voidstar.
-            # - Bookworm using Clang 17: Release on linux/amd64, set the
-            #   reference fee to 1000.
-            # - Bookworm using Clang 20: Debug on linux/amd64.
+            # - Bookworm using GCC 13: Release and Unity on linux/amd64, set
+            #   the reference fee to 500.
+            # - Bookworm using GCC 15: Debug and no Unity on linux/amd64, enable
+            #   code coverage (which will be done below).
+            # - Bookworm using Clang 16: Debug and no Unity on linux/arm64,
+            #   enable voidstar.
+            # - Bookworm using Clang 17: Release and no Unity on linux/amd64,
+            #   set the reference fee to 1000.
+            # - Bookworm using Clang 20: Debug and Unity on linux/amd64.
             if os["distro_name"] == "debian":
                 skip = True
                 if os["distro_version"] == "bookworm":
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-13"
                         and build_type == "Release"
+                        and "-Dunity=ON" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=500 {cmake_args}"
@@ -72,12 +74,14 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15"
                         and build_type == "Debug"
+                        and "-Dunity=OFF" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         skip = False
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-16"
                         and build_type == "Debug"
+                        and "-Dunity=OFF" in cmake_args
                         and architecture["platform"] == "linux/arm64"
                     ):
                         cmake_args = f"-Dvoidstar=ON {cmake_args}"
@@ -85,6 +89,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-17"
                         and build_type == "Release"
+                        and "-Dunity=ON" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=1000 {cmake_args}"
@@ -92,6 +97,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-20"
                         and build_type == "Debug"
+                        and "-Dunity=ON" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         skip = False
@@ -99,14 +105,15 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     continue
 
             # RHEL:
-            # - 9 using GCC 12: Debug on linux/amd64.
-            # - 10 using Clang: Release on linux/amd64.
+            # - 9 using GCC 12: Debug and Unity on linux/amd64.
+            # - 10 using Clang: Release and no Unity on linux/amd64.
             if os["distro_name"] == "rhel":
                 skip = True
                 if os["distro_version"] == "9":
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12"
                         and build_type == "Debug"
+                        and "-Dunity=ON" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         skip = False
@@ -114,6 +121,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-any"
                         and build_type == "Release"
+                        and "-Dunity=OFF" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         skip = False
@@ -121,16 +129,17 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     continue
 
             # Ubuntu:
-            # - Jammy using GCC 12: Debug on linux/arm64.
-            # - Noble using GCC 14: Release on linux/amd64.
-            # - Noble using Clang 18: Debug on linux/amd64.
-            # - Noble using Clang 19: Release on linux/arm64.
+            # - Jammy using GCC 12: Debug and no Unity on linux/arm64.
+            # - Noble using GCC 14: Release and Unity on linux/amd64.
+            # - Noble using Clang 18: Debug and no Unity on linux/amd64.
+            # - Noble using Clang 19: Release and Unity on linux/arm64.
             if os["distro_name"] == "ubuntu":
                 skip = True
                 if os["distro_version"] == "jammy":
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12"
                         and build_type == "Debug"
+                        and "-Dunity=OFF" in cmake_args
                         and architecture["platform"] == "linux/arm64"
                     ):
                         skip = False
@@ -138,18 +147,21 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-14"
                         and build_type == "Release"
+                        and "-Dunity=ON" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         skip = False
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-18"
                         and build_type == "Debug"
+                        and "-Dunity=OFF" in cmake_args
                         and architecture["platform"] == "linux/amd64"
                     ):
                         skip = False
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-19"
                         and build_type == "Release"
+                        and "-Dunity=ON" in cmake_args
                         and architecture["platform"] == "linux/arm64"
                     ):
                         skip = False
@@ -157,16 +169,20 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     continue
 
             # MacOS:
-            # - Debug on macos/arm64.
+            # - Debug and no Unity on macos/arm64.
             if os["distro_name"] == "macos" and not (
-                build_type == "Debug" and architecture["platform"] == "macos/arm64"
+                build_type == "Debug"
+                and "-Dunity=OFF" in cmake_args
+                and architecture["platform"] == "macos/arm64"
             ):
                 continue
 
             # Windows:
-            # - Release on windows/amd64.
+            # - Release and Unity on windows/amd64.
             if os["distro_name"] == "windows" and not (
-                build_type == "Release" and architecture["platform"] == "windows/amd64"
+                build_type == "Release"
+                and "-Dunity=ON" in cmake_args
+                and architecture["platform"] == "windows/amd64"
             ):
                 continue
 
@@ -193,17 +209,18 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
         ):
             continue
 
-        # Enable code coverage for Debian Bookworm using GCC 15 in Debug on
-        # linux/amd64
+        # Enable code coverage for Debian Bookworm using GCC 15 in Debug and no
+        # Unity on linux/amd64
         if (
             f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15"
             and build_type == "Debug"
+            and "-Dunity=OFF" in cmake_args
             and architecture["platform"] == "linux/amd64"
         ):
             cmake_args = f"-Dcoverage=ON -Dcoverage_format=xml -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_C_FLAGS=-O0 -DCMAKE_CXX_FLAGS=-O0 {cmake_args}"
 
         # Generate a unique name for the configuration, e.g. macos-arm64-debug
-        # or debian-bookworm-gcc-12-amd64-release.
+        # or debian-bookworm-gcc-12-amd64-release-unity.
         config_name = os["distro_name"]
         if (n := os["distro_version"]) != "":
             config_name += f"-{n}"
@@ -217,6 +234,8 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
         config_name += f"-{build_type.lower()}"
         if "-Dcoverage=ON" in cmake_args:
             config_name += "-coverage"
+        if "-Dunity=ON" in cmake_args:
+            config_name += "-unity"
 
         # Add the configuration to the list, with the most unique fields first,
         # so that they are easier to identify in the GitHub Actions UI, as long

.github/scripts/strategy-matrix/linux.json

@@ -208,5 +208,5 @@
     }
   ],
   "build_type": ["Debug", "Release"],
-  "cmake_args": [""]
+  "cmake_args": ["-Dunity=OFF", "-Dunity=ON"]
 }

.github/scripts/strategy-matrix/macos.json

@@ -15,5 +15,8 @@
     }
   ],
   "build_type": ["Debug", "Release"],
-  "cmake_args": ["-DCMAKE_POLICY_VERSION_MINIMUM=3.5"]
+  "cmake_args": [
+    "-Dunity=OFF -DCMAKE_POLICY_VERSION_MINIMUM=3.5",
+    "-Dunity=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5"
+  ]
 }

.github/scripts/strategy-matrix/windows.json

@@ -15,5 +15,5 @@
     }
   ],
   "build_type": ["Debug", "Release"],
-  "cmake_args": [""]
+  "cmake_args": ["-Dunity=OFF", "-Dunity=ON"]
 }

BUILD.md

@@ -545,10 +545,16 @@ See [Sanitizers docs](./docs/build/sanitizers.md) for more details.
 | `assert`   | OFF           | Enable assertions.                                             |
 | `coverage` | OFF           | Prepare the coverage report.                                   |
 | `tests`    | OFF           | Build tests.                                                   |
+| `unity`    | OFF           | Configure a unity build.                                       |
 | `xrpld`    | OFF           | Build the xrpld application, and not just the libxrpl library. |
 | `werr`     | OFF           | Treat compilation warnings as errors                           |
 | `wextra`   | OFF           | Enable additional compilation warnings                         |
 
+[Unity builds][5] may be faster for the first build
+(at the cost of much more memory) since they concatenate sources into fewer
+translation units. Non-unity builds may be faster for incremental builds,
+and can be helpful for detecting `#include` omissions.
+
 ## Troubleshooting
 
 ### Conan
@@ -615,6 +621,7 @@ If you want to experiment with a new package, follow these steps:
 [1]: https://github.com/conan-io/conan-center-index/issues/13168
 [2]: https://en.cppreference.com/w/cpp/compiler_support/20
 [3]: https://docs.conan.io/en/latest/getting_started.html
+[5]: https://en.wikipedia.org/wiki/Unity_build
 [6]: https://github.com/boostorg/beast/issues/2648
 [7]: https://github.com/boostorg/beast/issues/2661
 [gcovr]: https://gcovr.com/en/stable/getting-started.html

CMakeLists.txt

@@ -103,6 +103,7 @@ find_package(OpenSSL REQUIRED)
 find_package(secp256k1 REQUIRED)
 find_package(SOCI REQUIRED)
 find_package(SQLite3 REQUIRED)
+find_package(wasmi REQUIRED)
 find_package(xxHash REQUIRED)
 
 target_link_libraries(

cmake/XrplAddTest.cmake

@@ -9,5 +9,8 @@ function (xrpl_add_test name)
 
     isolate_headers(${target} "${CMAKE_SOURCE_DIR}" "${CMAKE_SOURCE_DIR}/tests/${name}" PRIVATE)
 
+    # Make sure the test isn't optimized away in unity builds
+    set_target_properties(${target} PROPERTIES UNITY_BUILD_MODE GROUP UNITY_BUILD_BATCH_SIZE 0) # Adjust as needed
+
     add_test(NAME ${target} COMMAND ${target})
 endfunction ()

cmake/XrplCore.cmake

@@ -4,7 +4,12 @@
 
 include(target_protobuf_sources)
 
+# Protocol buffers cannot participate in a unity build,
+# because all the generated sources
+# define a bunch of `static const` variables with the same names,
+# so we just build them as a separate library.
 add_library(xrpl.libpb)
+set_target_properties(xrpl.libpb PROPERTIES UNITY_BUILD OFF)
 target_protobuf_sources(xrpl.libpb xrpl/proto LANGUAGE cpp IMPORT_DIRS include/xrpl/proto
                         PROTOS include/xrpl/proto/xrpl.proto)
 
@@ -40,6 +45,7 @@ target_link_libraries(
               Xrpl::opts
               Xrpl::syslibs
               secp256k1::secp256k1
+              wasmi::wasmi
               xrpl.libpb
               xxHash::xxhash
               $<$<BOOL:${voidstar}>:antithesis-sdk-cpp>)
@@ -155,4 +161,12 @@ if (xrpld)
         # antithesis_instrumentation.h, which is not exported as INTERFACE
         target_include_directories(xrpld PRIVATE ${CMAKE_SOURCE_DIR}/external/antithesis-sdk)
     endif ()
+
+    # any files that don't play well with unity should be added here
+    if (tests)
+        set_source_files_properties(
+            # these two seem to produce conflicts in beast teardown template methods
+            src/test/rpc/ValidatorRPC_test.cpp src/test/ledger/Invariants_test.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION
+                                                                                              TRUE)
+    endif ()
 endif ()

cmake/XrplSettings.cmake

@@ -30,6 +30,14 @@ if (tests)
     endif ()
 endif ()
 
+option(unity "Creates a build using UNITY support in cmake." OFF)
+if (unity)
+    if (NOT is_ci)
+        set(CMAKE_UNITY_BUILD_BATCH_SIZE 15 CACHE STRING "")
+    endif ()
+    set(CMAKE_UNITY_BUILD ON CACHE BOOL "Do a unity build")
+endif ()
+
 if (is_clang AND is_linux)
     option(voidstar "Enable Antithesis instrumentation." OFF)
 endif ()

conan.lock

@@ -3,6 +3,7 @@
     "requires": [
         "zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1765850150.075",
         "xxhash/0.8.3#681d36a0a6111fc56e5e45ea182c19cc%1765850149.987",
+        "wasmi/1.0.6#407c9db14601a8af1c7dd3b388f3e4cd%1768164779.349",
         "sqlite3/3.49.1#8631739a4c9b93bd3d6b753bac548a63%1765850149.926",
         "soci/4.0.3#a9f8d773cd33e356b5879a4b0564f287%1765850149.46",
         "snappy/1.1.10#968fef506ff261592ec30c574d4a7809%1765850147.878",

conanfile.py

@@ -23,6 +23,7 @@ class Xrpl(ConanFile):
         "shared": [True, False],
         "static": [True, False],
         "tests": [True, False],
+        "unity": [True, False],
         "xrpld": [True, False],
     }
 
@@ -34,6 +35,7 @@ class Xrpl(ConanFile):
         "openssl/3.5.4",
         "secp256k1/0.7.0",
         "soci/4.0.3",
+        "wasmi/1.0.6",
         "zlib/1.3.1",
     ]
 
@@ -54,6 +56,7 @@ class Xrpl(ConanFile):
         "shared": False,
         "static": True,
         "tests": False,
+        "unity": False,
         "xrpld": False,
         "date/*:header_only": True,
         "ed25519/*:shared": False,
@@ -166,6 +169,7 @@ class Xrpl(ConanFile):
         tc.variables["rocksdb"] = self.options.rocksdb
         tc.variables["BUILD_SHARED_LIBS"] = self.options.shared
         tc.variables["static"] = self.options.static
+        tc.variables["unity"] = self.options.unity
         tc.variables["xrpld"] = self.options.xrpld
         tc.generate()
 
@@ -212,6 +216,7 @@ class Xrpl(ConanFile):
             "soci::soci",
             "secp256k1::secp256k1",
             "sqlite3::sqlite",
+            "wasmi::wasmi",
             "xxhash::xxhash",
             "zlib::zlib",
         ]

include/xrpl/basics/rocksdb.h

@@ -1,5 +1,5 @@
-#ifndef XRPL_BASICS_ROCKSDB_H_INCLUDED
-#define XRPL_BASICS_ROCKSDB_H_INCLUDED
+#ifndef XRPL_UNITY_ROCKSDB_H_INCLUDED
+#define XRPL_UNITY_ROCKSDB_H_INCLUDED
 
 #if XRPL_ROCKSDB_AVAILABLE
 // #include <rocksdb2/port/port_posix.h>

src/libxrpl/net/RegisterSSLCerts.cpp

@@ -85,8 +85,7 @@ registerSSLCerts(boost::asio::ssl::context& ctx, boost::system::error_code& ec,
 // There is a very unpleasant interaction between <wincrypt> and
 // openssl x509 types (namely the former has macros that stomp
 // on the latter), these undefs allow this TU to be safely used in
-// unity builds without messing up subsequent TUs. Although we
-// no longer use unity builds, leaving the undefs here does no harm.
+// unity builds without messing up subsequent TUs.
 #if BOOST_OS_WINDOWS
 #undef X509_NAME
 #undef X509_EXTENSIONS