Merge branch 'develop' into ximinez/directory

This change updates a script in the documentation to automatically infer the version of a patched Conan dependency from the conan.lock file.

.gitignore

@@ -111,3 +111,6 @@ bld.rippled/
 
 # Suggested in-tree build directory
 /.build*/
+
+# Locally patched Conan recipes
+external/conan-center-index/

BUILD.md

@@ -141,21 +141,37 @@ Alternatively, you can pull the patched recipes into the repository and use them
 locally:
 
 ```bash
+# Extract the version number from the lockfile.
+function extract_version {
+  version=$(cat conan.lock | sed -nE "s@.+${1}/(.+)#.+@\1@p" | head -n1)
+  echo ${version}
+}
+
+# Define which recipes to export.
+recipes=(ed25519 grpc secp256k1 snappy soci)
+
+# Selectively check out the recipes from our CCI fork.
 cd external
+mkdir -p conan-center-index
+cd conan-center-index
 git init
 git remote add origin git@github.com:XRPLF/conan-center-index.git
 git sparse-checkout init
-git sparse-checkout set recipes/ed25519
-git sparse-checkout add recipes/secp256k1
-git sparse-checkout add recipes/snappy
-git sparse-checkout add recipes/soci
+for recipe in ${recipes[@]}; do
+  echo "Checking out ${recipe}..."
+  git sparse-checkout add recipes/${recipe}/all
+done
 git fetch origin master
 git checkout master
-conan export --version 2015.03 recipes/ed25519/all
-conan export --version 0.7.0 recipes/secp256k1/all
-conan export --version 1.1.10 recipes/snappy/all
-conan export --version 4.0.3 recipes/soci/all
-rm -rf .git
+cd ../..
+
+# Export the recipes into the local cache.
+for recipe in ${recipes[@]}; do
+  version=$(extract_version ${recipe})
+  echo "Exporting ${recipe}/${version}..."
+  conan export --version $(extract_version ${recipe}) \
+    external/conan-center-index/recipes/${recipe}/all
+done
 ```
 
 In the case we switch to a newer version of a dependency that still requires a
@@ -385,19 +401,6 @@ tools.build:cxxflags=['-DBOOST_ASIO_DISABLE_CONCEPTS']
    `--settings build_type=$BUILD_TYPE` or in the profile itself,
    under the section `[settings]` with the key `build_type`.
 
-   If you are using a Microsoft Visual C++ compiler,
-   then you will need to ensure consistency between the `build_type` setting
-   and the `compiler.runtime` setting.
-
-   When `build_type` is `Release`, `compiler.runtime` should be `MT`.
-
-   When `build_type` is `Debug`, `compiler.runtime` should be `MTd`.
-
-   ```
-   conan install .. --output-folder . --build missing --settings build_type=Release --settings compiler.runtime=MT
-   conan install .. --output-folder . --build missing --settings build_type=Debug --settings compiler.runtime=MTd
-   ```
-
 3. Configure CMake and pass the toolchain file generated by Conan, located at
    `$OUTPUT_FOLDER/build/generators/conan_toolchain.cmake`.
 

conan.lock

@@ -9,7 +9,7 @@
         "secp256k1/0.7.0#9210e4c655d0a665400378a663960d17%1764261276.764",
         "rocksdb/10.5.1#4a197eca381a3e5ae8adf8cffa5aacd0%1762797952.535",
         "re2/20230301#ca3b241baec15bd31ea9187150e0b333%1764175362.029",
-        "protobuf/3.21.12#44ee56c0a6eea0c19aeeaca680370b88%1764175361.456",
+        "protobuf/6.32.1#f481fd276fc23a33b85a3ed1e898b693%1759134873.45",
         "openssl/3.5.4#a1d5835cc6ed5c5b8f3cd5b9b5d24205%1760106486.594",
         "nudb/2.0.9#fb8dfd1a5557f5e0528114c2da17721e%1763150366.909",
         "lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504%1756234228.999",
@@ -17,19 +17,19 @@
         "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1756230911.03",
         "libarchive/3.8.1#ffee18995c706e02bf96e7a2f7042e0d%1764175360.142",
         "jemalloc/5.3.0#e951da9cf599e956cebc117880d2d9f8%1729241615.244",
-        "grpc/1.50.1#02291451d1e17200293a409410d1c4e1%1756234248.958",
-        "ed25519/2015.03#17c1f1910e769f368025267a53c23c13%1764259445.491",
+        "grpc/1.72.0#f244a57bff01e708c55a1100b12e1589%1763158050.628",
+        "ed25519/2015.03#ae761bdc52730a843f0809bdf6c1b1f6%1764270189.893",
         "doctest/2.4.12#eb9fb352fb2fdfc8abb17ec270945165%1762797941.757",
         "date/3.0.4#862e11e80030356b53c2c38599ceb32b%1763584497.32",
         "c-ares/1.34.5#5581c2b62a608b40bb85d965ab3ec7c8%1764175359.429",
         "bzip2/1.0.8#c470882369c2d95c5c77e970c0c7e321%1764175359.429",
         "boost/1.88.0#8852c0b72ce8271fb8ff7c53456d4983%1756223752.326",
-        "abseil/20230802.1#90ba607d4ee8fb5fb157c3db540671fc%1764175359.429"
+        "abseil/20250127.0#9e8e8cfc89a1324139fc0ee3bd4d8c8c%1753819045.301"
     ],
     "build_requires": [
         "zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1756234269.497",
         "strawberryperl/5.32.1.1#707032463aa0620fa17ec0d887f5fe41%1756234281.733",
-        "protobuf/3.21.12#44ee56c0a6eea0c19aeeaca680370b88%1764175361.456",
+        "protobuf/6.32.1#f481fd276fc23a33b85a3ed1e898b693%1759134873.45",
         "nasm/2.16.01#31e26f2ee3c4346ecd347911bd126904%1756234232.901",
         "msys2/cci.latest#5b73b10144f73cc5bfe0572ed9be39e1%1751977009.857",
         "m4/1.4.19#70dc8bbb33e981d119d2acc0175cf381%1763158052.846",
@@ -37,13 +37,13 @@
         "cmake/3.31.10#313d16a1aa16bbdb2ca0792467214b76%1764175359.429",
         "b2/5.3.3#107c15377719889654eb9a162a673975%1756234226.28",
         "automake/1.16.5#b91b7c384c3deaa9d535be02da14d04f%1755524470.56",
-        "autoconf/2.71#51077f068e61700d65bb05541ea1e4b0%1731054366.86"
+        "autoconf/2.71#51077f068e61700d65bb05541ea1e4b0%1731054366.86",
+        "abseil/20250127.0#9e8e8cfc89a1324139fc0ee3bd4d8c8c%1753819045.301"
     ],
     "python_requires": [],
     "overrides": {
-        "protobuf/3.21.12": [
-            null,
-            "protobuf/3.21.12"
+        "protobuf/5.27.0": [
+            "protobuf/6.32.1"
         ],
         "lz4/1.9.4": [
             "lz4/1.10.0"

conan/profiles/default

@@ -20,12 +20,6 @@ compiler.libcxx={{detect_api.detect_libcxx(compiler, version, compiler_exe)}}
 {% endif %}
 
 [conf]
-{% if compiler == "clang" and compiler_version >= 19 %}
-grpc/1.50.1:tools.build:cxxflags+=['-Wno-missing-template-arg-list-after-template-kw']
-{% endif %}
-{% if compiler == "apple-clang" and compiler_version >= 17 %}
-grpc/1.50.1:tools.build:cxxflags+=['-Wno-missing-template-arg-list-after-template-kw']
-{% endif %}
 {% if compiler == "gcc" and compiler_version < 13 %}
 tools.build:cxxflags+=['-Wno-restrict']
 {% endif %}

conanfile.py

@@ -29,7 +29,7 @@ class Xrpl(ConanFile):
 
     requires = [
         "ed25519/2015.03",
-        "grpc/1.50.1",
+        "grpc/1.72.0",
         "libarchive/3.8.1",
         "nudb/2.0.9",
         "openssl/3.5.4",
@@ -43,7 +43,7 @@ class Xrpl(ConanFile):
     ]
 
     tool_requires = [
-        "protobuf/3.21.12",
+        "protobuf/6.32.1",
     ]
 
     default_options = {
@@ -61,6 +61,16 @@ class Xrpl(ConanFile):
         "ed25519/*:shared": False,
         "grpc/*:shared": False,
         "grpc/*:secure": True,
+        "grpc/*:codegen": True,
+        "grpc/*:cpp_plugin": True,
+        "grpc/*:csharp_ext": False,
+        "grpc/*:csharp_plugin": False,
+        "grpc/*:node_plugin": False,
+        "grpc/*:objective_c_plugin": False,
+        "grpc/*:php_plugin": False,
+        "grpc/*:python_plugin": False,
+        "grpc/*:ruby_plugin": False,
+        "grpc/*:otel_plugin": False,
         "libarchive/*:shared": False,
         "libarchive/*:with_acl": False,
         "libarchive/*:with_bzip2": False,
@@ -118,7 +128,7 @@ class Xrpl(ConanFile):
         self.requires("boost/1.88.0", force=True, **transitive_headers_opt)
         self.requires("date/3.0.4", **transitive_headers_opt)
         self.requires("lz4/1.10.0", force=True)
-        self.requires("protobuf/3.21.12", force=True)
+        self.requires("protobuf/6.32.1", force=True)
         self.requires("sqlite3/3.49.1", force=True)
         if self.options.jemalloc:
             self.requires("jemalloc/5.3.0")

include/xrpl/protocol/detail/features.macro

@@ -16,6 +16,7 @@
 // Add new amendments to the top of this list.
 // Keep it sorted in reverse chronological order.
 
+XRPL_FEATURE(DefragDirectories,          Supported::no,  VoteBehavior::DefaultNo)
 XRPL_FEATURE(LendingProtocol,            Supported::no,  VoteBehavior::DefaultNo)
 XRPL_FEATURE(PermissionDelegationV1_1,   Supported::no,  VoteBehavior::DefaultNo)
 XRPL_FIX    (DirectoryLimit,             Supported::yes, VoteBehavior::DefaultNo)

src/libxrpl/ledger/ApplyView.cpp

@@ -10,6 +10,14 @@ namespace ripple {
 
 namespace directory {
 
+struct Gap
+{
+    uint64_t const page;
+    SLE::pointer node;
+    uint64_t const nextPage;
+    SLE::pointer next;
+};
+
 std::uint64_t
 createRoot(
     ApplyView& view,
@@ -112,7 +120,9 @@ insertPage(
         return std::nullopt;
     if (!view.rules().enabled(fixDirectoryLimit) &&
         page >= dirNodeMaxPages)  // Old pages limit
+    {
         return std::nullopt;
+    }
 
     // We are about to create a new node; we'll link it to
     // the chain first:
@@ -134,15 +144,8 @@ insertPage(
     // it's the default.
     if (page != 1)
         node->setFieldU64(sfIndexPrevious, page - 1);
-    XRPL_ASSERT_PARTS(
-        !nextPage,
-        "ripple::directory::insertPage",
-        "nextPage has default value");
-    /* Reserved for future use when directory pages may be inserted in
-     * between two other pages instead of only at the end of the chain.
     if (nextPage)
         node->setFieldU64(sfIndexNext, nextPage);
-    */
     describe(node);
     view.insert(node);
 
@@ -158,7 +161,7 @@ ApplyView::dirAdd(
     uint256 const& key,
     std::function<void(std::shared_ptr<SLE> const&)> const& describe)
 {
-    auto root = peek(directory);
+    auto const root = peek(directory);
 
     if (!root)
     {
@@ -169,6 +172,44 @@ ApplyView::dirAdd(
     auto [page, node, indexes] =
         directory::findPreviousPage(*this, directory, root);
 
+    if (rules().enabled(featureDefragDirectories))
+    {
+        // If there are more nodes than just the root, and there's no space in
+        // the last one, walk backwards to find one with space, or to find one
+        // missing.
+        std::optional<directory::Gap> gapPages;
+        while (page && indexes.size() >= dirNodeMaxEntries)
+        {
+            // Find a page with space, or a gap in pages.
+            auto [prevPage, prevNode, prevIndexes] =
+                directory::findPreviousPage(*this, directory, node);
+            if (!gapPages && prevPage != page - 1)
+                gapPages.emplace(prevPage, prevNode, page, node);
+            page = prevPage;
+            node = prevNode;
+            indexes = prevIndexes;
+        }
+        // We looped through all the pages back to the root.
+        if (!page)
+        {
+            // If we found a gap, use it.
+            if (gapPages)
+            {
+                return directory::insertPage(
+                    *this,
+                    gapPages->page,
+                    gapPages->node,
+                    gapPages->nextPage,
+                    gapPages->next,
+                    key,
+                    directory,
+                    describe);
+            }
+            std::tie(page, node, indexes) =
+                directory::findPreviousPage(*this, directory, root);
+        }
+    }
+
     // If there's space, we use it:
     if (indexes.size() < dirNodeMaxEntries)
     {