Merge branch 'develop' into ripple/wasmi

.github/actions/build-deps/action.yml

@@ -35,6 +35,7 @@ runs:
         LOG_VERBOSITY: ${{ inputs.log_verbosity }}
       run: |
         echo 'Installing dependencies.'
+        conan export --version 0.42.1 external/wasmi  #TODO Switch to conan package
         mkdir -p "${BUILD_DIR}"
         cd "${BUILD_DIR}"
         conan install \

.github/scripts/strategy-matrix/linux.json

@@ -15,196 +15,196 @@
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "12",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "13",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "15",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "16",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "17",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "18",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "19",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "20",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "debian",
       "distro_version": "trixie",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "debian",
       "distro_version": "trixie",
       "compiler_name": "gcc",
       "compiler_version": "15",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "debian",
       "distro_version": "trixie",
       "compiler_name": "clang",
       "compiler_version": "20",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "debian",
       "distro_version": "trixie",
       "compiler_name": "clang",
       "compiler_version": "21",
-      "image_sha": "0525eae"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "rhel",
       "distro_version": "8",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "rhel",
       "distro_version": "8",
       "compiler_name": "clang",
       "compiler_version": "any",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "rhel",
       "distro_version": "9",
       "compiler_name": "gcc",
       "compiler_version": "12",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "rhel",
       "distro_version": "9",
       "compiler_name": "gcc",
       "compiler_version": "13",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "rhel",
       "distro_version": "9",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "rhel",
       "distro_version": "9",
       "compiler_name": "clang",
       "compiler_version": "any",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "rhel",
       "distro_version": "10",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "rhel",
       "distro_version": "10",
       "compiler_name": "clang",
       "compiler_version": "any",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "jammy",
       "compiler_name": "gcc",
       "compiler_version": "12",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "gcc",
       "compiler_version": "13",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "clang",
       "compiler_version": "16",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "clang",
       "compiler_version": "17",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "clang",
       "compiler_version": "18",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "clang",
       "compiler_version": "19",
-      "image_sha": "e1782cd"
+      "image_sha": "ca4517d"
     }
   ],
   "build_type": ["Debug", "Release"],

.github/workflows/reusable-build-test-config.yml

@@ -77,7 +77,7 @@ jobs:
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
 
       - name: Prepare runner
-        uses: XRPLF/actions/.github/actions/prepare-runner@99685816bb60a95a66852f212f382580e180df3a
+        uses: XRPLF/actions/.github/actions/prepare-runner@ff9f8f649df5855ffe1a1ae715df43e51807f2e0
         with:
           disable_ccache: false
 

.github/workflows/upload-conan-deps.yml

@@ -70,7 +70,7 @@ jobs:
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
 
       - name: Prepare runner
-        uses: XRPLF/actions/.github/actions/prepare-runner@99685816bb60a95a66852f212f382580e180df3a
+        uses: XRPLF/actions/.github/actions/prepare-runner@ff9f8f649df5855ffe1a1ae715df43e51807f2e0
         with:
           disable_ccache: false
 

CMakeLists.txt

@@ -113,6 +113,7 @@ find_package(date REQUIRED)
 find_package(ed25519 REQUIRED)
 find_package(nudb REQUIRED)
 find_package(secp256k1 REQUIRED)
+find_package(wasmi REQUIRED)
 find_package(xxHash REQUIRED)
 
 target_link_libraries(xrpl_libs INTERFACE

cfg/rippled-example.cfg

@@ -1051,11 +1051,10 @@
 #                           The online delete process checks periodically
 #                           that rippled is still in sync with the network,
 #                           and that the validated ledger is less than
-#                           'age_threshold_seconds' old, and that all
+#                           'age_threshold_seconds' old. If not, then continue
-#                           recent ledgers are available. If not, then continue
 #                           sleeping for this number of seconds and
 #                           checking until healthy.
-#                           Default is 1.
+#                           Default is 5.
 #
 #   Notes:
 #       The 'node_db' entry configures the primary, persistent storage.

cmake/XrplCore.cmake

@@ -63,6 +63,7 @@ target_link_libraries(xrpl.imports.main
     Xrpl::opts
     Xrpl::syslibs
     secp256k1::secp256k1
+    wasmi::wasmi
     xrpl.libpb
     xxHash::xxhash
     $<$<BOOL:${voidstar}>:antithesis-sdk-cpp>

conan.lock

@@ -1,62 +0,0 @@
-{
-    "version": "0.5",
-    "requires": [
-        "zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1756234269.497",
-        "xxhash/0.8.3#681d36a0a6111fc56e5e45ea182c19cc%1756234289.683",
-        "sqlite3/3.49.1#8631739a4c9b93bd3d6b753bac548a63%1756234266.869",
-        "soci/4.0.3#a9f8d773cd33e356b5879a4b0564f287%1756234262.318",
-        "snappy/1.1.10#968fef506ff261592ec30c574d4a7809%1756234314.246",
-        "secp256k1/0.7.0#9210e4c655d0a665400378a663960d17%1764261276.764",
-        "rocksdb/10.5.1#4a197eca381a3e5ae8adf8cffa5aacd0%1762797952.535",
-        "re2/20230301#ca3b241baec15bd31ea9187150e0b333%1764175362.029",
-        "protobuf/6.32.1#f481fd276fc23a33b85a3ed1e898b693%1759134873.45",
-        "openssl/3.5.4#a1d5835cc6ed5c5b8f3cd5b9b5d24205%1760106486.594",
-        "nudb/2.0.9#fb8dfd1a5557f5e0528114c2da17721e%1763150366.909",
-        "lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504%1756234228.999",
-        "libiconv/1.17#1e65319e945f2d31941a9d28cc13c058%1756223727.64",
-        "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1756230911.03",
-        "libarchive/3.8.1#ffee18995c706e02bf96e7a2f7042e0d%1764175360.142",
-        "jemalloc/5.3.0#e951da9cf599e956cebc117880d2d9f8%1729241615.244",
-        "grpc/1.72.0#f244a57bff01e708c55a1100b12e1589%1763158050.628",
-        "ed25519/2015.03#ae761bdc52730a843f0809bdf6c1b1f6%1764270189.893",
-        "doctest/2.4.12#eb9fb352fb2fdfc8abb17ec270945165%1762797941.757",
-        "date/3.0.4#862e11e80030356b53c2c38599ceb32b%1763584497.32",
-        "c-ares/1.34.5#5581c2b62a608b40bb85d965ab3ec7c8%1764175359.429",
-        "bzip2/1.0.8#c470882369c2d95c5c77e970c0c7e321%1764175359.429",
-        "boost/1.88.0#8852c0b72ce8271fb8ff7c53456d4983%1756223752.326",
-        "abseil/20250127.0#9e8e8cfc89a1324139fc0ee3bd4d8c8c%1753819045.301"
-    ],
-    "build_requires": [
-        "zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1756234269.497",
-        "strawberryperl/5.32.1.1#707032463aa0620fa17ec0d887f5fe41%1756234281.733",
-        "protobuf/6.32.1#f481fd276fc23a33b85a3ed1e898b693%1759134873.45",
-        "nasm/2.16.01#31e26f2ee3c4346ecd347911bd126904%1756234232.901",
-        "msys2/cci.latest#5b73b10144f73cc5bfe0572ed9be39e1%1751977009.857",
-        "m4/1.4.19#70dc8bbb33e981d119d2acc0175cf381%1763158052.846",
-        "cmake/4.2.0#ae0a44f44a1ef9ab68fd4b3e9a1f8671%1764175359.44",
-        "cmake/3.31.10#313d16a1aa16bbdb2ca0792467214b76%1764175359.429",
-        "b2/5.3.3#107c15377719889654eb9a162a673975%1756234226.28",
-        "automake/1.16.5#b91b7c384c3deaa9d535be02da14d04f%1755524470.56",
-        "autoconf/2.71#51077f068e61700d65bb05541ea1e4b0%1731054366.86",
-        "abseil/20250127.0#9e8e8cfc89a1324139fc0ee3bd4d8c8c%1753819045.301"
-    ],
-    "python_requires": [],
-    "overrides": {
-        "protobuf/5.27.0": [
-            "protobuf/6.32.1"
-        ],
-        "lz4/1.9.4": [
-            "lz4/1.10.0"
-        ],
-        "boost/1.83.0": [
-            "boost/1.88.0"
-        ],
-        "sqlite3/3.44.2": [
-            "sqlite3/3.49.1"
-        ],
-        "lz4/[>=1.9.4 <2]": [
-            "lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504"
-        ]
-    },
-    "config_requires": []
-}

conanfile.py

@@ -35,6 +35,7 @@ class Xrpl(ConanFile):
         "openssl/3.5.4",
         "secp256k1/0.7.0",
         "soci/4.0.3",
+        "wasmi/0.42.1",
         "zlib/1.3.1",
     ]
 
@@ -212,6 +213,7 @@ class Xrpl(ConanFile):
             "soci::soci",
             "secp256k1::secp256k1",
             "sqlite3::sqlite",
+            "wasmi::wasmi",
             "xxhash::xxhash",
             "zlib::zlib",
         ]

external/wasmi/conandata.yml

@@ -0,0 +1,11 @@
+# Do not update. Maintained by programmability team.
+
+sources:
+  "0.42.1":
+    sha256: 2a5697be33c7afce8f671af4a5a3621d9e93ce55d253d31bd8201458e465fbb8
+    url: https://github.com/wasmi-labs/wasmi/archive/refs/tags/v0.42.1.tar.gz
+patches:
+  "0.42.1":
+    - patch_file: "patches/0001-xrplf-0.42.1.patch"
+      patch_type: conan
+      patch_description: Integration patch. Make wasmi lib to be able to build with existing smart-escrow branch.

external/wasmi/conanfile.py

@@ -0,0 +1,48 @@
+from conan import ConanFile, tools
+from conan.tools.cmake import CMake, CMakeToolchain, CMakeDeps, cmake_layout
+from conan.tools.files import apply_conandata_patches, export_conandata_patches, get
+import os
+
+required_conan_version = ">=2.0.0"
+
+class WasmiConan(ConanFile):
+    name = "wasmi"
+    license = "Apache License v2.0"
+    url = "https://github.com/wasmi-labs/wasmi.git"
+    description = "WebAssembly (Wasm) interpreter"
+    package_type = "library"
+    settings = "os", "arch", "compiler", "build_type"
+    options = {"shared": [False]}
+    default_options = {"shared": False}
+
+    def export_sources(self):
+        export_conandata_patches(self)
+
+    def layout(self):
+        cmake_layout(self, src_folder="src")
+
+    def source(self):
+        get(self, **self.conan_data["sources"][self.version], strip_root=True)
+        apply_conandata_patches(self)
+
+    def generate(self):
+        tc = CMakeToolchain(self)
+        tc.variables["BUILD_SHARED_LIBS"] = self.options.shared
+        tc.generate()
+
+        deps = CMakeDeps(self)
+        deps.generate()
+
+    def build(self):
+        cmake = CMake(self)
+        cmake.configure(build_script_folder=os.path.join(self.source_folder, "crates", "c_api"))
+        cmake.build()
+
+    def package(self):
+        cmake = CMake(self)
+        cmake.install()
+
+    def package_info(self):
+        self.cpp_info.libs = ["wasmi"]
+        self.cpp_info.names["cmake_find_package"] = "wasmi"
+        self.cpp_info.names["cmake_find_package_multi"] = "wasmi"

external/wasmi/patches/0001-xrplf-0.42.1.patch

@@ -0,0 +1,180 @@
+diff --git a/crates/c_api/CMakeLists.txt b/crates/c_api/CMakeLists.txt
+index b15c787a..54eaed2d 100644
+--- a/crates/c_api/CMakeLists.txt
++++ b/crates/c_api/CMakeLists.txt
+@@ -6,6 +6,8 @@ option(BUILD_SHARED_LIBS "Build using shared libraries" OFF)
+ option(WASMI_ALWAYS_BUILD "If cmake should always invoke cargo to build Wasmi" ON)
+ set(WASMI_TARGET "" CACHE STRING "Rust target to build for")
+ 
++add_compile_definitions(COMPILING_WASM_RUNTIME_API=1)
++
+ if(NOT WASMI_TARGET)
+     execute_process(
+         COMMAND rustc -vV
+@@ -43,6 +45,10 @@ endif()
+ list(TRANSFORM WASMI_SHARED_FILES PREPEND ${WASMI_TARGET_DIR}/)
+ list(TRANSFORM WASMI_STATIC_FILES PREPEND ${WASMI_TARGET_DIR}/)
+ 
++if(NOT BUILD_SHARED_LIBS)
++  set(WASMI_SHARED_FILES)
++endif()
++
+ # Instructions on how to build and install the Wasmi Rust crate.
+ find_program(WASMI_CARGO_BINARY cargo REQUIRED)
+ include(ExternalProject)
+@@ -79,7 +85,6 @@ else()
+     target_link_libraries(wasmi INTERFACE ${WASMI_STATIC_FILES})
+ 
+     if(WASMI_TARGET MATCHES "windows")
+-        target_compile_options(wasmi INTERFACE -DWASM_API_EXTERN= -DWASI_API_EXTERN=)
+         target_link_libraries(wasmi INTERFACE ws2_32 advapi32 userenv ntdll shell32 ole32 bcrypt)
+     elseif(NOT WASMI_TARGET MATCHES "darwin")
+         target_link_libraries(wasmi INTERFACE pthread dl m)
+@@ -112,6 +117,7 @@ install(
+     DESTINATION ${CMAKE_INSTALL_LIBDIR}
+ )
+ 
++if(BUILD_SHARED_LIBS)
+ if(WASMI_TARGET MATCHES "darwin")
+     set(INSTALLED_LIB "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libwasmi.dylib")
+     install(
+@@ -131,6 +137,7 @@ if(WASMI_TARGET MATCHES "darwin")
+         install(CODE "execute_process(COMMAND ${install_name_tool_cmd})")
+     endif()
+ endif()
++endif()
+ 
+ # Documentation Generation via Doxygen:
+ set(DOXYGEN_CONF_IN ${CMAKE_CURRENT_SOURCE_DIR}/doxygen.conf.in)
+@@ -141,19 +148,3 @@ add_custom_target(doc
+     DEPENDS ${WASMI_GENERATED_CONF_H} ${DOXYGEN_CONF_OUT}
+     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+ )
+-
+-# C-Header Formatting via clang-format:
+-find_program(CLANG_FORMAT clang-format REQUIRED)
+-file(GLOB_RECURSE HEADER_FILES
+-    ${CMAKE_CURRENT_SOURCE_DIR}/include/wasmi.h
+-    ${CMAKE_CURRENT_SOURCE_DIR}/include/wasmi/*.h
+-    ${CMAKE_CURRENT_SOURCE_DIR}/include/wasmi/*.hh
+-)
+-add_custom_target(check-format
+-    COMMAND ${CLANG_FORMAT} -style=llvm -Werror --dry-run ${HEADER_FILES}
+-    COMMENT "clang-format: Check formatting for Wasmi C-API header files"
+-)
+-add_custom_target(format
+-    COMMAND ${CLANG_FORMAT} -style=llvm -i ${HEADER_FILES}
+-    COMMENT "clang-format: Apply formatting rules for Wasmi C-API header files"
+-)
+diff --git a/crates/c_api/include/wasm.h b/crates/c_api/include/wasm.h
+index 5ee617ff..37809a6f 100644
+--- a/crates/c_api/include/wasm.h
++++ b/crates/c_api/include/wasm.h
+@@ -13,11 +13,17 @@
+ #include <assert.h>
+ 
+ #ifndef WASM_API_EXTERN
+-#if defined(_WIN32) && !defined(__MINGW32__) && !defined(LIBWASM_STATIC)
++#if defined(_MSC_BUILD)
++#if defined(COMPILING_WASM_RUNTIME_API)
++#define WASM_API_EXTERN __declspec(dllexport)
++#elif defined(_DLL)
+ #define WASM_API_EXTERN __declspec(dllimport)
+ #else
+ #define WASM_API_EXTERN
+ #endif
++#else
++#define WASM_API_EXTERN
++#endif
+ #endif
+ 
+ #ifdef __cplusplus
+@@ -146,6 +152,13 @@ WASM_DECLARE_OWN(store)
+ 
+ WASM_API_EXTERN own wasm_store_t* wasm_store_new(wasm_engine_t*);
+ 
++// Store fuel functions (forward declarations)
++struct wasmi_error;
++
++WASM_API_EXTERN struct wasmi_error* wasm_store_get_fuel(const wasm_store_t*, uint64_t* fuel);
++WASM_API_EXTERN struct wasmi_error* wasm_store_set_fuel(wasm_store_t*, uint64_t fuel);
++//WASM_API_EXTERN void *wasm_store_get_data(const wasm_store_t*);
++//WASM_API_EXTERN void wasm_store_set_data(wasm_store_t*, void *data);
+ 
+ ///////////////////////////////////////////////////////////////////////////////
+ // Type Representations
+diff --git a/crates/c_api/include/wasmi.h b/crates/c_api/include/wasmi.h
+index 2caffa37..0c0584ec 100644
+--- a/crates/c_api/include/wasmi.h
++++ b/crates/c_api/include/wasmi.h
+@@ -10,7 +10,7 @@
+ /**
+  * \brief Wasmi version string.
+  */
+-#define WASMI_VERSION "0.35.0"
++#define WASMI_VERSION "0.42.1"
+ /**
+  * \brief Wasmi major version number.
+  */
+@@ -18,10 +18,10 @@
+ /**
+  * \brief Wasmi minor version number.
+  */
+-#define WASMI_VERSION_MINOR 35
++#define WASMI_VERSION_MINOR 42
+ /**
+  * \brief Wasmi patch version number.
+  */
+-#define WASMI_VERSION_PATCH 0
++#define WASMI_VERSION_PATCH 1
+ 
+ #endif // WASMI_H
+diff --git a/crates/c_api/src/store.rs b/crates/c_api/src/store.rs
+index 56d4898f..543dbff8 100644
+--- a/crates/c_api/src/store.rs
++++ b/crates/c_api/src/store.rs
+@@ -175,3 +175,44 @@ pub extern "C" fn wasmi_context_set_fuel(
+ ) -> Option<Box<wasmi_error_t>> {
+     crate::handle_result(store.set_fuel(fuel), |()| {})
+ }
++
++////////////////////////////////////////////////////////////////////////////////////////
++////////////////////////////////////////////////////////////////////////////////////////
++////////////////////////////////////////////////////////////////////////////////////////
++////////////////////////////////////////////////////////////////////////////////////////
++
++/// Returns the current fuel of the wasm store context in `fuel`.
++///
++/// Wraps [`Store::get_fuel`].
++///
++/// # Errors
++///
++/// If [`Store::get_fuel`] errors.
++#[no_mangle]
++pub extern "C" fn wasm_store_get_fuel(
++    store: &wasm_store_t,
++    fuel: &mut u64,
++) -> Option<Box<wasmi_error_t>> {
++    let context = unsafe { store.inner.context() };
++    crate::handle_result(context.get_fuel(), |amt| {
++        *fuel = amt;
++    })
++}
++
++/// Sets the current fuel of the wasm store context to `fuel`.
++///
++/// Wraps [`Store::set_fuel`].
++///
++/// # Errors
++///
++/// If [`Store::set_fuel`] errors.
++#[no_mangle]
++pub extern "C" fn wasm_store_set_fuel(
++    store: &mut wasm_store_t,
++    fuel: u64,
++) -> Option<Box<wasmi_error_t>> {
++
++    let mut context = unsafe { store.inner.context_mut() };
++    crate::handle_result(context.set_fuel(fuel), |()| {})
++}
++

src/test/app/LedgerMaster_test.cpp

@@ -2,7 +2,6 @@
 #include <test/jtx/Env.h>
 
 #include <xrpld/app/ledger/LedgerMaster.h>
-#include <xrpld/app/misc/SHAMapStore.h>
 
 namespace ripple {
 namespace test {
@@ -101,88 +100,6 @@ class LedgerMaster_test : public beast::unit_test::suite
         }
     }
 
-    void
-    testCompleteLedgerRange(FeatureBitset features)
-    {
-        // Note that this test is intentionally very similar to
-        // SHAMapStore_test::testLedgerGaps, but has a different
-        // focus.
-
-        testcase("Complete Ledger operations");
-
-        using namespace test::jtx;
-
-        auto const deleteInterval = 8;
-
-        Env env{*this, envconfig([](auto cfg) {
-                    return online_delete(std::move(cfg), deleteInterval);
-                })};
-
-        auto const alice = Account("alice");
-        env.fund(XRP(1000), alice);
-        env.close();
-
-        auto& lm = env.app().getLedgerMaster();
-        LedgerIndex minSeq = 2;
-        LedgerIndex maxSeq = env.closed()->info().seq;
-        auto& store = env.app().getSHAMapStore();
-        store.rendezvous();
-        LedgerIndex lastRotated = store.getLastRotated();
-        BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
-        BEAST_EXPECTS(
-            lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
-        BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
-        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
-        BEAST_EXPECT(
-            lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
-        BEAST_EXPECT(
-            lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
-        BEAST_EXPECT(
-            lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
-        BEAST_EXPECT(
-            lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
-
-        // Close enough ledgers to rotate a few times
-        for (int i = 0; i < 24; ++i)
-        {
-            for (int t = 0; t < 3; ++t)
-            {
-                env(noop(alice));
-            }
-            env.close();
-            store.rendezvous();
-
-            ++maxSeq;
-
-            if (maxSeq == lastRotated + deleteInterval)
-            {
-                minSeq = lastRotated;
-                lastRotated = maxSeq;
-            }
-            BEAST_EXPECTS(
-                env.closed()->info().seq == maxSeq,
-                to_string(env.closed()->info().seq));
-            BEAST_EXPECTS(
-                store.getLastRotated() == lastRotated,
-                to_string(store.getLastRotated()));
-            std::stringstream expectedRange;
-            expectedRange << minSeq << "-" << maxSeq;
-            BEAST_EXPECTS(
-                lm.getCompleteLedgers() == expectedRange.str(),
-                lm.getCompleteLedgers());
-            BEAST_EXPECT(
-                lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
-            BEAST_EXPECT(
-                lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
-            BEAST_EXPECT(
-                lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
-            BEAST_EXPECT(
-                lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
-            BEAST_EXPECT(
-                lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
-        }
-    }
-
 public:
     void
     run() override
@@ -196,7 +113,6 @@ public:
     testWithFeats(FeatureBitset features)
     {
         testTxnIdFromIndex(features);
-        testCompleteLedgerRange(features);
     }
 };
 

src/test/app/SHAMapStore_test.cpp

@@ -1,7 +1,6 @@
 #include <test/jtx.h>
 #include <test/jtx/envconfig.h>
 
-#include <xrpld/app/ledger/LedgerMaster.h>
 #include <xrpld/app/main/Application.h>
 #include <xrpld/app/main/NodeStoreScheduler.h>
 #include <xrpld/app/misc/SHAMapStore.h>
@@ -11,8 +10,6 @@
 #include <xrpl/nodestore/detail/DatabaseRotatingImp.h>
 #include <xrpl/protocol/jss.h>
 
-#include <thread>
-
 namespace ripple {
 namespace test {
 
@@ -23,8 +20,10 @@ class SHAMapStore_test : public beast::unit_test::suite
     static auto
     onlineDelete(std::unique_ptr<Config> cfg)
     {
-        using namespace jtx;
+        cfg->LEDGER_HISTORY = deleteInterval;
-        return online_delete(std::move(cfg), deleteInterval);
+        auto& section = cfg->section(ConfigSection::nodeDatabase());
+        section.set("online_delete", std::to_string(deleteInterval));
+        return cfg;
     }
 
     static auto
@@ -627,184 +626,6 @@ public:
         BEAST_EXPECT(dbr->getName() == "3");
     }
 
-    void
-    testLedgerGaps()
-    {
-        // Note that this test is intentionally very similar to
-        // LedgerMaster_test::testCompleteLedgerRange, but has a different
-        // focus.
-
-        testcase("Wait for ledger gaps to fill in");
-
-        using namespace test::jtx;
-
-        Env env{*this, envconfig(onlineDelete)};
-
-        std::map<LedgerIndex, uint256> hashes;
-
-        auto failureMessage = [&](char const* label,
-                                  auto expected,
-                                  auto actual) {
-            std::stringstream ss;
-            ss << label << ": Expected: " << expected << ", Got: " << actual;
-            return ss.str();
-        };
-
-        auto const alice = Account("alice");
-        env.fund(XRP(1000), alice);
-        env.close();
-
-        auto& lm = env.app().getLedgerMaster();
-        LedgerIndex minSeq = 2;
-        LedgerIndex maxSeq = env.closed()->info().seq;
-        auto& store = env.app().getSHAMapStore();
-        store.rendezvous();
-        LedgerIndex lastRotated = store.getLastRotated();
-        BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
-        BEAST_EXPECTS(
-            lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
-        BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
-        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
-        BEAST_EXPECT(
-            lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
-        BEAST_EXPECT(
-            lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
-        BEAST_EXPECT(
-            lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
-        BEAST_EXPECT(
-            lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
-
-        // Close enough ledgers to rotate a few times
-        while (maxSeq < 20)
-        {
-            for (int t = 0; t < 3; ++t)
-            {
-                env(noop(alice));
-            }
-            env.close();
-            store.rendezvous();
-
-            ++maxSeq;
-
-            if (maxSeq + 1 == lastRotated + deleteInterval)
-            {
-                using namespace std::chrono_literals;
-
-                // The next ledger will trigger a rotation. Delete the
-                // current ledger from LedgerMaster.
-                std::this_thread::sleep_for(100ms);
-                LedgerIndex const deleteSeq = maxSeq;
-                while (!lm.haveLedger(deleteSeq))
-                {
-                    std::this_thread::sleep_for(100ms);
-                }
-                lm.clearLedger(deleteSeq);
-
-                auto expectedRange =
-                    [](auto minSeq, auto deleteSeq, auto maxSeq) {
-                        std::stringstream expectedRange;
-                        expectedRange << minSeq << "-" << (deleteSeq - 1);
-                        if (deleteSeq + 1 == maxSeq)
-                            expectedRange << "," << maxSeq;
-                        else if (deleteSeq < maxSeq)
-                            expectedRange << "," << (deleteSeq + 1) << "-"
-                                          << maxSeq;
-                        return expectedRange.str();
-                    };
-                BEAST_EXPECTS(
-                    lm.getCompleteLedgers() ==
-                        expectedRange(minSeq, deleteSeq, maxSeq),
-                    failureMessage(
-                        "Complete ledgers",
-                        expectedRange(minSeq, deleteSeq, maxSeq),
-                        lm.getCompleteLedgers()));
-                BEAST_EXPECT(
-                    lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 1);
-
-                // Close another ledger, which will trigger a rotation, but the
-                // rotation will be stuck until the missing ledger is filled in.
-                env.close();
-                // DO NOT CALL rendezvous()! You'll end up with a deadlock.
-                ++maxSeq;
-
-                // Nothing has changed
-                BEAST_EXPECTS(
-                    store.getLastRotated() == lastRotated,
-                    failureMessage(
-                        "lastRotated", lastRotated, store.getLastRotated()));
-                BEAST_EXPECTS(
-                    lm.getCompleteLedgers() ==
-                        expectedRange(minSeq, deleteSeq, maxSeq),
-                    failureMessage(
-                        "Complete ledgers",
-                        expectedRange(minSeq, deleteSeq, maxSeq),
-                        lm.getCompleteLedgers()));
-
-                // Close 5 more ledgers, waiting one second in between to
-                // simulate the ledger making progress while online delete waits
-                // for the missing ledger to be filled in.
-                // This ensures the healthWait check has time to run and
-                // detect the gap.
-                for (int l = 0; l < 5; ++l)
-                {
-                    env.close();
-                    // DO NOT CALL rendezvous()! You'll end up with a deadlock.
-                    ++maxSeq;
-                    // Nothing has changed
-                    BEAST_EXPECTS(
-                        store.getLastRotated() == lastRotated,
-                        failureMessage(
-                            "lastRotated",
-                            lastRotated,
-                            store.getLastRotated()));
-                    BEAST_EXPECTS(
-                        lm.getCompleteLedgers() ==
-                            expectedRange(minSeq, deleteSeq, maxSeq),
-                        failureMessage(
-                            "Complete Ledgers",
-                            expectedRange(minSeq, deleteSeq, maxSeq),
-                            lm.getCompleteLedgers()));
-                    std::this_thread::sleep_for(1s);
-                }
-
-                // Put the missing ledger back in LedgerMaster
-                lm.setLedgerRangePresent(deleteSeq, deleteSeq);
-
-                // Wait for the rotation to finish
-                store.rendezvous();
-
-                minSeq = lastRotated;
-                lastRotated = deleteSeq + 1;
-            }
-            BEAST_EXPECT(maxSeq != lastRotated + deleteInterval);
-            BEAST_EXPECTS(
-                env.closed()->info().seq == maxSeq,
-                failureMessage("maxSeq", maxSeq, env.closed()->info().seq));
-            BEAST_EXPECTS(
-                store.getLastRotated() == lastRotated,
-                failureMessage(
-                    "lastRotated", lastRotated, store.getLastRotated()));
-            std::stringstream expectedRange;
-            expectedRange << minSeq << "-" << maxSeq;
-            BEAST_EXPECTS(
-                lm.getCompleteLedgers() == expectedRange.str(),
-                failureMessage(
-                    "CompleteLedgers",
-                    expectedRange.str(),
-                    lm.getCompleteLedgers()));
-            BEAST_EXPECT(
-                lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
-            BEAST_EXPECT(
-                lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
-            BEAST_EXPECT(
-                lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
-            BEAST_EXPECT(
-                lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
-            BEAST_EXPECT(
-                lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
-        }
-    }
-
     void
     run() override
     {
@@ -812,7 +633,6 @@ public:
         testAutomatic();
         testCanDelete();
         testRotate();
-        testLedgerGaps();
     }
 };
 

src/test/jtx/envconfig.h

@@ -58,17 +58,6 @@ envconfig(F&& modfunc, Args&&... args)
     return modfunc(envconfig(), std::forward<Args>(args)...);
 }
 
-/// @brief adjust config to enable online_delete
-///
-/// @param cfg config instance to be modified
-///
-/// @param deleteInterval how many new ledgers should be available before
-/// rotating. Defaults to 8, because the standalone minimum is 8.
-///
-/// @return unique_ptr to Config instance
-std::unique_ptr<Config>
-online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval = 8);
-
 /// @brief adjust config so no admin ports are enabled
 ///
 /// this is intended for use with envconfig, as in

src/test/jtx/impl/envconfig.cpp

@@ -53,15 +53,6 @@ setupConfigForUnitTests(Config& cfg)
 
 namespace jtx {
 
-std::unique_ptr<Config>
-online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval)
-{
-    cfg->LEDGER_HISTORY = deleteInterval;
-    auto& section = cfg->section(ConfigSection::nodeDatabase());
-    section.set("online_delete", std::to_string(deleteInterval));
-    return cfg;
-}
-
 std::unique_ptr<Config>
 no_admin(std::unique_ptr<Config> cfg)
 {

src/xrpld/app/ledger/LedgerMaster.h

@@ -108,10 +108,7 @@ public:
     failedSave(std::uint32_t seq, uint256 const& hash);
 
     std::string
-    getCompleteLedgers() const;
+    getCompleteLedgers();
-
-    std::size_t
-    missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const;
 
     /** Apply held transactions to the open ledger
         This is normally called as we close the ledger.
@@ -328,7 +325,7 @@ private:
     // A set of transactions to replay during the next close
     std::unique_ptr<LedgerReplay> replayData;
 
-    std::recursive_mutex mutable mCompleteLock;
+    std::recursive_mutex mCompleteLock;
     RangeSet<std::uint32_t> mCompleteLedgers;
 
     // Publish thread is running.

src/xrpld/app/ledger/detail/LedgerMaster.cpp

@@ -1571,36 +1571,12 @@ LedgerMaster::getPublishedLedger()
 }
 
 std::string
-LedgerMaster::getCompleteLedgers() const
+LedgerMaster::getCompleteLedgers()
 {
     std::lock_guard sl(mCompleteLock);
     return to_string(mCompleteLedgers);
 }
 
-std::size_t
-LedgerMaster::missingFromCompleteLedgerRange(
-    LedgerIndex first,
-    LedgerIndex last) const
-{
-    // Make a copy of the range to avoid holding the lock
-    auto const range = [&] {
-        std::lock_guard sl(mCompleteLock);
-        return mCompleteLedgers;
-    }();
-
-    std::size_t missing = 0;
-
-    for (LedgerIndex idx = first; idx <= last; ++idx)
-    {
-        if (!boost::icl::contains(range, idx))
-        {
-            ++missing;
-        }
-    }
-
-    return missing;
-}
-
 std::optional<NetClock::time_point>
 LedgerMaster::getCloseTimeBySeq(LedgerIndex ledgerIndex)
 {

src/xrpld/app/misc/SHAMapStoreImp.cpp

@@ -289,18 +289,6 @@ SHAMapStoreImp::run()
             validatedSeq >= lastRotated + deleteInterval_ &&
             canDelete_ >= lastRotated - 1 && healthWait() == keepGoing;
 
-        JLOG(journal_.debug())
-            << "run: Setting lastGoodValidatedLedger_ to " << validatedSeq;
-
-        {
-            // Note that this is set after the healthWait() check, so that we
-            // don't start the rotation until the validated ledger is fully
-            // processed. It is not guaranteed to be done at this point. It also
-            // allows the testLedgerGaps unit test to work.
-            std::unique_lock<std::mutex> lock(mutex_);
-            lastGoodValidatedLedger_ = validatedSeq;
-        }
-
         // will delete up to (not including) lastRotated
         if (readyToRotate)
         {
@@ -309,9 +297,7 @@ SHAMapStoreImp::run()
                 << lastRotated << " deleteInterval " << deleteInterval_
                 << " canDelete_ " << canDelete_ << " state "
                 << app_.getOPs().strOperatingMode(false) << " age "
-                << ledgerMaster_->getValidatedLedgerAge().count()
+                << ledgerMaster_->getValidatedLedgerAge().count() << 's';
-                << "s. Complete ledgers: "
-                << ledgerMaster_->getCompleteLedgers();
 
             clearPrior(lastRotated);
             if (healthWait() == stopping)
@@ -374,10 +360,7 @@ SHAMapStoreImp::run()
                     clearCaches(validatedSeq);
                 });
 
-            JLOG(journal_.warn())
+            JLOG(journal_.warn()) << "finished rotation " << validatedSeq;
-                << "finished rotation. validatedSeq: " << validatedSeq
-                << ", lastRotated: " << lastRotated << ". Complete ledgers: "
-                << ledgerMaster_->getCompleteLedgers();
         }
     }
 }
@@ -632,47 +615,22 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
 SHAMapStoreImp::HealthResult
 SHAMapStoreImp::healthWait()
 {
-    auto index = ledgerMaster_->getValidLedgerIndex();
     auto age = ledgerMaster_->getValidatedLedgerAge();
     OperatingMode mode = netOPs_->getOperatingMode();
     std::unique_lock lock(mutex_);
-
+    while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_))
-    auto numMissing = ledgerMaster_->missingFromCompleteLedgerRange(
-        lastGoodValidatedLedger_, index);
-    while (
-        !stop_ &&
-        (mode != OperatingMode::FULL || age > ageThreshold_ || numMissing > 0))
     {
-        // this value shouldn't change, so grab it while we have the
-        // lock
-        auto const lastGood = lastGoodValidatedLedger_;
-
         lock.unlock();
-
+        JLOG(journal_.warn()) << "Waiting " << recoveryWaitTime_.count()
-        auto const stream = mode != OperatingMode::FULL || age > ageThreshold_
+                              << "s for node to stabilize. state: "
-            ? journal_.warn()
+                              << app_.getOPs().strOperatingMode(mode, false)
-            : journal_.info();
+                              << ". age " << age.count() << 's';
-        JLOG(stream) << "Waiting " << recoveryWaitTime_.count()
-                     << "s for node to stabilize. state: "
-                     << app_.getOPs().strOperatingMode(mode, false) << ". age "
-                     << age.count() << "s. Missing ledgers: " << numMissing
-                     << ". Expect: " << lastGood << "-" << index
-                     << ". Complete ledgers: "
-                     << ledgerMaster_->getCompleteLedgers();
         std::this_thread::sleep_for(recoveryWaitTime_);
-        index = ledgerMaster_->getValidLedgerIndex();
         age = ledgerMaster_->getValidatedLedgerAge();
         mode = netOPs_->getOperatingMode();
-        numMissing =
-            ledgerMaster_->missingFromCompleteLedgerRange(lastGood, index);
-
         lock.lock();
     }
 
-    JLOG(journal_.debug()) << "healthWait: Setting lastGoodValidatedLedger_ to "
-                           << index;
-    lastGoodValidatedLedger_ = index;
-
     return stop_ ? stopping : keepGoing;
 }
 

src/xrpld/app/misc/SHAMapStoreImp.h

@@ -71,11 +71,6 @@ private:
     std::thread thread_;
     bool stop_ = false;
     bool healthy_ = true;
-    // Used to prevent ledger gaps from forming during online deletion. Keeps
-    // track of the last validated ledger that was processed without gaps. There
-    // are no guarantees about gaps while online delete is not running. For
-    // that, use advisory_delete and check for gaps externally.
-    LedgerIndex lastGoodValidatedLedger_ = 0;
     mutable std::condition_variable cond_;
     mutable std::condition_variable rendezvous_;
     mutable std::mutex mutex_;
@@ -89,11 +84,11 @@ private:
     std::uint32_t deleteBatch_ = 100;
     std::chrono::milliseconds backOff_{100};
     std::chrono::seconds ageThreshold_{60};
-    /// If  the node is out of sync, or any recent ledgers are not
+    /// If  the node is out of sync during an online_delete healthWait()
-    /// available during an online_delete healthWait() call, sleep
+    /// call, sleep the thread for this time, and continue checking until
-    /// the thread for this time, and continue checking until recovery.
+    /// recovery.
     /// See also: "recovery_wait_seconds" in rippled-example.cfg
-    std::chrono::seconds recoveryWaitTime_{1};
+    std::chrono::seconds recoveryWaitTime_{5};
 
     // these do not exist upon SHAMapStore creation, but do exist
     // as of run() or before
@@ -217,8 +212,6 @@ private:
     enum HealthResult { stopping, keepGoing };
     [[nodiscard]] HealthResult
     healthWait();
-    bool
-    hasCompleteRange(LedgerIndex first, LedgerIndex last);
 
 public:
     void