more detailed tracing

include/xrpl/basics/IntrusiveRefCounts.h

@@ -7,6 +7,24 @@
 
 namespace xrpl {
 
+namespace detail {
+// All-time peak strong/weak ref counts ever observed across all
+// IntrusiveRefCounts instances. Read from doSweep periodically.
+inline std::atomic<std::uint32_t> kPeakStrongObserved{0};
+inline std::atomic<std::uint32_t> kPeakWeakObserved{0};
+
+inline void
+updateRefCountPeak(std::atomic<std::uint32_t>& peak, std::uint32_t v) noexcept
+{
+    auto cur = peak.load(std::memory_order_relaxed);
+    while (v > cur && !peak.compare_exchange_weak(
+                          cur, v, std::memory_order_relaxed))
+    {
+        // retry; cur is updated by compare_exchange_weak on failure
+    }
+}
+}  // namespace detail
+
 /** Action to perform when releasing a strong pointer.
 
     noop: Do nothing. For example, a `noop` action will occur when a count is
@@ -226,13 +244,18 @@ private:
 inline void
 IntrusiveRefCounts::addStrongRef() const noexcept
 {
-    refCounts_.fetch_add(kStrongDelta, std::memory_order_acq_rel);
+    auto const prev = refCounts_.fetch_add(kStrongDelta, std::memory_order_acq_rel);
+    auto const newStrong = static_cast<std::uint32_t>((prev & kStrongMask) + 1);
+    detail::updateRefCountPeak(detail::kPeakStrongObserved, newStrong);
 }
 
 inline void
 IntrusiveRefCounts::addWeakRef() const noexcept
 {
-    refCounts_.fetch_add(kWeakDelta, std::memory_order_acq_rel);
+    auto const prev = refCounts_.fetch_add(kWeakDelta, std::memory_order_acq_rel);
+    auto const newWeak =
+        static_cast<std::uint32_t>(((prev & kWeakMask) >> kStrongCountNumBits) + 1);
+    detail::updateRefCountPeak(detail::kPeakWeakObserved, newWeak);
 }
 
 inline ReleaseStrongRefAction
@@ -331,6 +354,11 @@ IntrusiveRefCounts::addWeakReleaseStrongRef() const
                 (!(prevIntVal & kPartialDestroyStartedMask)),
                 "xrpl::IntrusiveRefCounts::addWeakReleaseStrongRef : not "
                 "started partial destroy");
+            // This op converts one strong into one weak: capture the new
+            // weak count for the leak-proof peak.
+            auto const newWeak = static_cast<std::uint32_t>(
+                ((prevIntVal & kWeakMask) >> kStrongCountNumBits) + 1);
+            detail::updateRefCountPeak(detail::kPeakWeakObserved, newWeak);
             return action;
         }
     }
@@ -377,6 +405,10 @@ IntrusiveRefCounts::checkoutStrongRefFromWeak() const noexcept
 
         desiredValue = curValue + kStrongDelta;
     }
+    // Successful CAS promoted a weak ref to strong. Capture the new strong
+    // count for the leak-proof peak.
+    auto const newStrong = static_cast<std::uint32_t>(desiredValue & kStrongMask);
+    detail::updateRefCountPeak(detail::kPeakStrongObserved, newStrong);
     return true;
 }
 
@@ -413,6 +445,8 @@ inline IntrusiveRefCounts::RefCountPair::RefCountPair(IntrusiveRefCounts::FieldT
     , partialDestroyStartedBit{v & kPartialDestroyStartedMask}
     , partialDestroyFinishedBit{v & kPartialDestroyFinishedMask}
 {
+    detail::updateRefCountPeak(detail::kPeakStrongObserved, strong);
+    detail::updateRefCountPeak(detail::kPeakWeakObserved, weak);
     XRPL_ASSERT(
         (strong < kCheckStrongMaxValue && weak < kCheckWeakMaxValue),
         "xrpl::IntrusiveRefCounts::RefCountPair(FieldType) : inputs inside "

include/xrpl/protocol/detail/features.macro

@@ -15,7 +15,6 @@
 // Add new amendments to the top of this list.
 // Keep it sorted in reverse chronological order.
 
-XRPL_FEATURE(DefragDirectories,           Supported::No,  VoteBehavior::DefaultNo)
 XRPL_FIX    (Cleanup3_2_0,                Supported::Yes, VoteBehavior::DefaultNo)
 XRPL_FEATURE(MPTokensV2,                  Supported::No,  VoteBehavior::DefaultNo)
 XRPL_FIX    (Cleanup3_1_3,                Supported::Yes, VoteBehavior::DefaultYes)

src/libxrpl/ledger/ApplyView.cpp

@@ -26,14 +26,6 @@ namespace xrpl {
 
 namespace directory {
 
-struct Gap
-{
-    uint64_t const page;
-    SLE::pointer node;
-    uint64_t const nextPage;
-    SLE::pointer next;
-};
-
 std::uint64_t
 createRoot(
     ApplyView& view,
@@ -134,9 +126,7 @@ insertPage(
     if (page == 0)
         return std::nullopt;
     if (!view.rules().enabled(fixDirectoryLimit) && page >= kDirNodeMaxPages)  // Old pages limit
-    {
         return std::nullopt;
-    }
 
     // We are about to create a new node; we'll link it to
     // the chain first:
@@ -157,8 +147,12 @@ insertPage(
     // Save some space by not specifying the value 0 since it's the default.
     if (page != 1)
         node->setFieldU64(sfIndexPrevious, page - 1);
+    XRPL_ASSERT_PARTS(!nextPage, "xrpl::directory::insertPage", "nextPage has default value");
+    /* Reserved for future use when directory pages may be inserted in
+     * between two other pages instead of only at the end of the chain.
     if (nextPage)
         node->setFieldU64(sfIndexNext, nextPage);
+    */
     describe(node);
     view.insert(node);
 
@@ -174,7 +168,7 @@ ApplyView::dirAdd(
     uint256 const& key,
     std::function<void(std::shared_ptr<SLE> const&)> const& describe)
 {
-    auto const root = peek(directory);
+    auto root = peek(directory);
 
     if (!root)
     {
@@ -184,43 +178,6 @@ ApplyView::dirAdd(
 
     auto [page, node, indexes] = directory::findPreviousPage(*this, directory, root);
 
-    if (rules().enabled(featureDefragDirectories))
-    {
-        // If there are more nodes than just the root, and there's no space in
-        // the last one, walk backwards to find one with space, or to find one
-        // missing.
-        std::optional<directory::Gap> gapPages;
-        while (page && indexes.size() >= kDIR_NODE_MAX_PAGES)
-        {
-            // Find a page with space, or a gap in pages.
-            auto [prevPage, prevNode, prevIndexes] =
-                directory::findPreviousPage(*this, directory, node);
-            if (!gapPages && prevPage != page - 1)
-                gapPages.emplace(prevPage, prevNode, page, node);
-            page = prevPage;
-            node = prevNode;
-            indexes = prevIndexes;
-        }
-        // We looped through all the pages back to the root.
-        if (!page)
-        {
-            // If we found a gap, use it.
-            if (gapPages)
-            {
-                return directory::insertPage(
-                    *this,
-                    gapPages->page,
-                    gapPages->node,
-                    gapPages->nextPage,
-                    gapPages->next,
-                    key,
-                    directory,
-                    describe);
-            }
-            std::tie(page, node, indexes) = directory::findPreviousPage(*this, directory, root);
-        }
-    }
-
     // If there's space, we use it:
     if (indexes.size() < kDirNodeMaxEntries)
     {

src/xrpld/app/main/Application.cpp

@@ -1084,6 +1084,21 @@ public:
                                    << "; size after: " << cachedSLEs_.size();
         }
 
+        {
+            auto const peakStrong =
+                xrpl::detail::kPeakStrongObserved.load(std::memory_order_relaxed);
+            auto const peakWeak =
+                xrpl::detail::kPeakWeakObserved.load(std::memory_order_relaxed);
+            constexpr std::uint32_t kStrongLimit = 65503;  // kCheckStrongMaxValue
+            constexpr std::uint32_t kWeakLimit = 16351;    // kCheckWeakMaxValue
+            JLOG(journal_.warn())
+                << "RefCount peak since startup: "
+                << "strong=" << peakStrong << "/" << kStrongLimit
+                << " (" << (peakStrong * 100 / kStrongLimit) << "%); "
+                << "weak=" << peakWeak << "/" << kWeakLimit
+                << " (" << (peakWeak * 100 / kWeakLimit) << "%)";
+        }
+
         mallocTrim("doSweep", journal_);
 
         // Set timer to do another sweep later.