Merge remote-tracking branch 'XRPLF/develop' into ximinez/directory

* XRPLF/develop: (41 commits)
  release: Bump version to 3.2.0-rc2 (7348)
  refactor: Enable support for `fixCleanup3_2_0` amendment (7347)
  release: Bump version to 3.2.0-rc1 (7335)
  fix: Fix a rounding error at the `Number::maxRep` cusp (7051)
  ci: Only push docker images in XRPLF/rippled (7330)
  ci: [DEPENDABOT] bump docker/setup-buildx-action from 4.0.0 to 4.1.0 (7322)
  ci: [DEPENDABOT] bump codecov/codecov-action from 6.0.0 to 6.0.1 (7321)
  ci: [DEPENDABOT] bump docker/build-push-action from 7.1.0 to 7.2.0 (7320)
  ci: [DEPENDABOT] bump docker/metadata-action from 6.0.0 to 6.1.0 (7319)
  ci: [DEPENDABOT] bump docker/login-action from 4.1.0 to 4.2.0 (7318)
  fix: Update `clang-tidy` to include `src/tests` directory header check (7307)
  chore: Pin Python packages for codegen using uv (7329)
  style: Use shfmt instead of bashate (7326)
  fix: Fix edge-case where vault-depositor may get stuck (7139)
  fix: Fix `VaultInvariant` and `VaultDeposit` precision bugs at IOU scale boundaries (7272)
  ci: Add clang to nix images (7308)
  fix: Include management-fee delta in doOverpayment assertion (7039)
  fix: Fix clang-tidy pre-commit hook to locate compile_commands.json from repo root (7325)
  fix: Use consistent scale for `debtTotal` (7093)
  fix: Skip deleted book directories and non-root modifications in `ValidBookDirectory` invariant (7312)
  ...

include/xrpl/protocol/detail/features.macro

@@ -15,6 +15,7 @@
 // Add new amendments to the top of this list.
 // Keep it sorted in reverse chronological order.
 
+XRPL_FEATURE(DefragDirectories,           Supported::No,  VoteBehavior::DefaultNo)
 XRPL_FIX    (Cleanup3_2_0,                Supported::Yes, VoteBehavior::DefaultNo)
 XRPL_FEATURE(MPTokensV2,                  Supported::No,  VoteBehavior::DefaultNo)
 XRPL_FIX    (Cleanup3_1_3,                Supported::Yes, VoteBehavior::DefaultYes)

src/libxrpl/ledger/ApplyView.cpp

@@ -26,6 +26,14 @@ namespace xrpl {
 
 namespace directory {
 
+struct Gap
+{
+    uint64_t const page;
+    SLE::pointer node;
+    uint64_t const nextPage;
+    SLE::pointer next;
+};
+
 std::uint64_t
 createRoot(
     ApplyView& view,
@@ -126,7 +134,9 @@ insertPage(
     if (page == 0)
         return std::nullopt;
     if (!view.rules().enabled(fixDirectoryLimit) && page >= kDirNodeMaxPages)  // Old pages limit
+    {
         return std::nullopt;
+    }
 
     // We are about to create a new node; we'll link it to
     // the chain first:
@@ -147,12 +157,8 @@ insertPage(
     // Save some space by not specifying the value 0 since it's the default.
     if (page != 1)
         node->setFieldU64(sfIndexPrevious, page - 1);
-    XRPL_ASSERT_PARTS(!nextPage, "xrpl::directory::insertPage", "nextPage has default value");
-    /* Reserved for future use when directory pages may be inserted in
-     * between two other pages instead of only at the end of the chain.
     if (nextPage)
         node->setFieldU64(sfIndexNext, nextPage);
-    */
     describe(node);
     view.insert(node);
 
@@ -168,7 +174,7 @@ ApplyView::dirAdd(
     uint256 const& key,
     std::function<void(std::shared_ptr<SLE> const&)> const& describe)
 {
-    auto root = peek(directory);
+    auto const root = peek(directory);
 
     if (!root)
     {
@@ -178,6 +184,43 @@ ApplyView::dirAdd(
 
     auto [page, node, indexes] = directory::findPreviousPage(*this, directory, root);
 
+    if (rules().enabled(featureDefragDirectories))
+    {
+        // If there are more nodes than just the root, and there's no space in
+        // the last one, walk backwards to find one with space, or to find one
+        // missing.
+        std::optional<directory::Gap> gapPages;
+        while (page && indexes.size() >= kDIR_NODE_MAX_PAGES)
+        {
+            // Find a page with space, or a gap in pages.
+            auto [prevPage, prevNode, prevIndexes] =
+                directory::findPreviousPage(*this, directory, node);
+            if (!gapPages && prevPage != page - 1)
+                gapPages.emplace(prevPage, prevNode, page, node);
+            page = prevPage;
+            node = prevNode;
+            indexes = prevIndexes;
+        }
+        // We looped through all the pages back to the root.
+        if (!page)
+        {
+            // If we found a gap, use it.
+            if (gapPages)
+            {
+                return directory::insertPage(
+                    *this,
+                    gapPages->page,
+                    gapPages->node,
+                    gapPages->nextPage,
+                    gapPages->next,
+                    key,
+                    directory,
+                    describe);
+            }
+            std::tie(page, node, indexes) = directory::findPreviousPage(*this, directory, root);
+        }
+    }
+
     // If there's space, we use it:
     if (indexes.size() < kDirNodeMaxEntries)
     {