log tree structure

include/xrpl/basics/MallocTrim.h

@@ -0,0 +1,70 @@
+#ifndef XRPL_BASICS_MALLOCTRIM_H_INCLUDED
+#define XRPL_BASICS_MALLOCTRIM_H_INCLUDED
+
+#include <xrpl/beast/utility/Journal.h>
+
+#include <optional>
+#include <string>
+
+namespace ripple {
+
+// -----------------------------------------------------------------------------
+// Allocator interaction note:
+// - This facility invokes glibc's malloc_trim(0) on Linux/glibc to request that
+//   ptmalloc return free heap pages to the OS.
+// - If an alternative allocator (e.g. jemalloc or tcmalloc) is linked or
+//   preloaded (LD_PRELOAD), calling glibc's malloc_trim typically has no effect
+//   on the *active* heap. The call is harmless but may not reclaim memory
+//   because those allocators manage their own arenas.
+// - Only glibc sbrk/arena space is eligible for trimming; large mmap-backed
+//   allocations are usually returned to the OS on free regardless of trimming.
+// - Call at known reclamation points (e.g., after cache sweeps / online delete)
+//   and consider rate limiting to avoid churn.
+// -----------------------------------------------------------------------------
+
+struct MallocTrimReport
+{
+    bool supported{false};
+    int trimResult{-1};
+    long rssBeforeKB{-1};
+    long rssAfterKB{-1};
+
+    [[nodiscard]] long
+    deltaKB() const noexcept
+    {
+        if (rssBeforeKB < 0 || rssAfterKB < 0)
+            return 0;
+        return rssAfterKB - rssBeforeKB;
+    }
+};
+
+/**
+ * @brief Attempt to return freed memory to the operating system.
+ *
+ * On Linux with glibc malloc, this issues ::malloc_trim(0), which may release
+ * free space from ptmalloc arenas back to the kernel. On other platforms, or if
+ * a different allocator is in use, this function is a no-op and the report will
+ * indicate that trimming is unsupported or had no effect.
+ *
+ * @param tag     Optional identifier for logging/debugging purposes.
+ * @param journal Journal for diagnostic logging.
+ * @return Report containing before/after metrics and the trim result.
+ *
+ * @note If an alternative allocator (jemalloc/tcmalloc) is linked or preloaded,
+ *       calling glibc's malloc_trim may have no effect on the active heap. The
+ *       call is harmless but typically does not reclaim memory under those
+ *       allocators.
+ *
+ * @note Only memory served from glibc's sbrk/arena heaps is eligible for trim.
+ *       Large allocations satisfied via mmap are usually returned on free
+ *       independently of trimming.
+ *
+ * @note Intended for use after operations that free significant memory (e.g.,
+ *       cache sweeps, ledger cleanup, online delete). Consider rate limiting.
+ */
+MallocTrimReport
+mallocTrim(std::optional<std::string> const& tag, beast::Journal journal);
+
+}  // namespace ripple
+
+#endif

include/xrpl/shamap/SHAMap.h

@@ -347,6 +347,13 @@ public:
     void
     invariants() const;
 
+    /** Log tree structure statistics for debugging/monitoring
+        @param j Journal to log to
+        @param mapName Name to identify this map in logs
+    */
+    void
+    logTreeStats(beast::Journal j, std::string const& mapName) const;
+
 private:
     using SharedPtrNodeStack = std::stack<
         std::pair<intr_ptr::SharedPtr<SHAMapTreeNode>, SHAMapNodeID>>;

src/libxrpl/basics/MallocTrim.cpp

@@ -0,0 +1,121 @@
+#include <xrpl/basics/Log.h>
+#include <xrpl/basics/MallocTrim.h>
+
+#include <boost/predef.h>
+
+#include <cstdio>
+#include <fstream>
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+#include <malloc.h>
+#include <unistd.h>
+
+namespace {
+pid_t const cachedPid = ::getpid();
+}  // namespace
+#endif
+
+namespace ripple {
+
+namespace detail {
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+
+long
+parseVmRSSkB(std::string const& status)
+{
+    std::istringstream iss(status);
+    std::string line;
+
+    while (std::getline(iss, line))
+    {
+        // Allow leading spaces/tabs before the key.
+        auto const firstNonWs = line.find_first_not_of(" \t");
+        if (firstNonWs == std::string::npos)
+            continue;
+
+        constexpr char key[] = "VmRSS:";
+        constexpr auto keyLen = sizeof(key) - 1;
+
+        // Require the line (after leading whitespace) to start with "VmRSS:".
+        // Check if we have enough characters and the substring matches.
+        if (firstNonWs + keyLen > line.size() ||
+            line.substr(firstNonWs, keyLen) != key)
+            continue;
+
+        // Move past "VmRSS:" and any following whitespace.
+        auto pos = firstNonWs + keyLen;
+        while (pos < line.size() &&
+               std::isspace(static_cast<unsigned char>(line[pos])))
+        {
+            ++pos;
+        }
+
+        long value = -1;
+        if (std::sscanf(line.c_str() + pos, "%ld", &value) == 1)
+            return value;
+
+        // Found the key but couldn't parse a number.
+        return -1;
+    }
+
+    // No VmRSS line found.
+    return -1;
+}
+
+#endif  // __GLIBC__ && BOOST_OS_LINUX
+
+}  // namespace detail
+
+MallocTrimReport
+mallocTrim(
+    [[maybe_unused]] std::optional<std::string> const& tag,
+    beast::Journal journal)
+{
+    MallocTrimReport report;
+
+#if !(defined(__GLIBC__) && BOOST_OS_LINUX)
+    JLOG(journal.debug()) << "malloc_trim not supported on this platform";
+#else
+
+    report.supported = true;
+
+    if (journal.debug())
+    {
+        auto readFile = [](std::string const& path) -> std::string {
+            std::ifstream ifs(path);
+            if (!ifs.is_open())
+                return {};
+            return std::string(
+                std::istreambuf_iterator<char>(ifs),
+                std::istreambuf_iterator<char>());
+        };
+
+        std::string const tagStr = tag.value_or("default");
+        std::string const statusPath =
+            "/proc/" + std::to_string(cachedPid) + "/status";
+
+        auto const statusBefore = readFile(statusPath);
+        report.rssBeforeKB = detail::parseVmRSSkB(statusBefore);
+
+        report.trimResult = ::malloc_trim(0);
+
+        auto const statusAfter = readFile(statusPath);
+        report.rssAfterKB = detail::parseVmRSSkB(statusAfter);
+
+        JLOG(journal.debug())
+            << "malloc_trim tag=" << tagStr << " result=" << report.trimResult
+            << " rss_before=" << report.rssBeforeKB << "kB"
+            << " rss_after=" << report.rssAfterKB << "kB"
+            << " delta=" << report.deltaKB() << "kB";
+    }
+    else
+    {
+        report.trimResult = ::malloc_trim(0);
+    }
+#endif
+
+    return report;
+}
+
+}  // namespace ripple

src/libxrpl/shamap/SHAMap.cpp

@@ -1240,4 +1240,72 @@ SHAMap::invariants() const
     node->invariants(true);
 }
 
+void
+SHAMap::logTreeStats(beast::Journal j, std::string const& mapName) const
+{
+    struct Stats
+    {
+        std::uint32_t totalNodes = 0;
+        std::uint32_t innerNodes = 0;
+        std::uint32_t leafNodes = 0;
+        std::uint32_t maxDepth = 0;
+        std::array<std::uint32_t, 65> depthCount;
+        Stats()
+        {
+            depthCount.fill(0);
+        }
+    } stats;
+
+    std::function<void(
+        intr_ptr::SharedPtr<SHAMapTreeNode> const&, std::uint32_t)>
+        traverse;
+    traverse = [&](intr_ptr::SharedPtr<SHAMapTreeNode> const& node,
+                   std::uint32_t depth) {
+        if (!node)
+            return;
+
+        stats.totalNodes++;
+        stats.depthCount[depth]++;
+        stats.maxDepth = std::max(stats.maxDepth, depth);
+
+        if (node->isInner())
+        {
+            stats.innerNodes++;
+            auto inner = dynamic_cast<SHAMapInnerNode*>(node.get());
+            if (inner)
+            {
+                for (int i = 0; i < branchFactor; ++i)
+                {
+                    if (auto child = inner->getChild(i))
+                    {
+                        traverse(child, depth + 1);
+                    }
+                }
+            }
+        }
+        else
+        {
+            stats.leafNodes++;
+        }
+    };
+
+    traverse(root_, 0);
+
+    JLOG(j.info()) << "SHAMap (" << mapName
+                   << ") stats: total_nodes=" << stats.totalNodes
+                   << ", inner_nodes=" << stats.innerNodes
+                   << ", leaf_nodes=" << stats.leafNodes
+                   << ", max_depth=" << stats.maxDepth;
+
+    std::ostringstream hist;
+    hist << "Depth histogram: { ";
+    for (std::uint32_t d = 0; d <= stats.maxDepth; ++d)
+    {
+        if (stats.depthCount[d] > 0)
+            hist << d << ": " << stats.depthCount[d] << ", ";
+    }
+    hist << "}";
+    JLOG(j.debug()) << "SHAMap (" << mapName << ") " << hist.str();
+}
+
 }  // namespace ripple

src/tests/libxrpl/basics/MallocTrim.cpp

@@ -0,0 +1,207 @@
+#include <xrpl/basics/MallocTrim.h>
+
+#include <boost/predef.h>
+
+#include <doctest/doctest.h>
+
+using namespace ripple;
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+namespace ripple::detail {
+long
+parseVmRSSkB(std::string const& status);
+}  // namespace ripple::detail
+#endif
+
+TEST_CASE("MallocTrimReport structure")
+{
+    // Test default construction
+    MallocTrimReport report;
+    CHECK(report.supported == false);
+    CHECK(report.trimResult == -1);
+    CHECK(report.rssBeforeKB == -1);
+    CHECK(report.rssAfterKB == -1);
+    CHECK(report.deltaKB() == 0);
+
+    // Test deltaKB calculation - memory freed
+    report.rssBeforeKB = 1000;
+    report.rssAfterKB = 800;
+    CHECK(report.deltaKB() == -200);
+
+    // Test deltaKB calculation - memory increased
+    report.rssBeforeKB = 500;
+    report.rssAfterKB = 600;
+    CHECK(report.deltaKB() == 100);
+
+    // Test deltaKB calculation - no change
+    report.rssBeforeKB = 1234;
+    report.rssAfterKB = 1234;
+    CHECK(report.deltaKB() == 0);
+}
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+TEST_CASE("parseVmRSSkB")
+{
+    using ripple::detail::parseVmRSSkB;
+
+    // Test standard format
+    {
+        std::string status = "VmRSS:      123456 kB\n";
+        long result = parseVmRSSkB(status);
+        CHECK(result == 123456);
+    }
+
+    // Test with multiple lines
+    {
+        std::string status =
+            "Name:   rippled\n"
+            "VmPeak:  1234567 kB\n"
+            "VmSize:  1234567 kB\n"
+            "VmRSS:      987654 kB\n"
+            "VmData:   123456 kB\n";
+        long result = parseVmRSSkB(status);
+        CHECK(result == 987654);
+    }
+
+    // Test with minimal whitespace
+    {
+        std::string status = "VmRSS: 42 kB";
+        long result = parseVmRSSkB(status);
+        CHECK(result == 42);
+    }
+
+    // Test with extra whitespace
+    {
+        std::string status = "VmRSS:          999999 kB";
+        long result = parseVmRSSkB(status);
+        CHECK(result == 999999);
+    }
+
+    // Test with tabs
+    {
+        std::string status = "VmRSS:\t\t12345 kB";
+        long result = parseVmRSSkB(status);
+        // Note: tabs are not explicitly handled as spaces, this documents
+        // current behavior
+        CHECK(result == 12345);
+    }
+
+    // Test zero value
+    {
+        std::string status = "VmRSS:      0 kB\n";
+        long result = parseVmRSSkB(status);
+        CHECK(result == 0);
+    }
+
+    // Test missing VmRSS
+    {
+        std::string status =
+            "Name:   rippled\n"
+            "VmPeak:  1234567 kB\n"
+            "VmSize:  1234567 kB\n";
+        long result = parseVmRSSkB(status);
+        CHECK(result == -1);
+    }
+
+    // Test empty string
+    {
+        std::string status = "";
+        long result = parseVmRSSkB(status);
+        CHECK(result == -1);
+    }
+
+    // Test malformed data (VmRSS but no number)
+    {
+        std::string status = "VmRSS:      \n";
+        long result = parseVmRSSkB(status);
+        // sscanf should fail to parse and return -1 unchanged
+        CHECK(result == -1);
+    }
+
+    // Test malformed data (VmRSS but invalid number)
+    {
+        std::string status = "VmRSS:      abc kB\n";
+        long result = parseVmRSSkB(status);
+        // sscanf should fail and return -1 unchanged
+        CHECK(result == -1);
+    }
+
+    // Test partial match (should not match "NotVmRSS:")
+    {
+        std::string status = "NotVmRSS:      123456 kB\n";
+        long result = parseVmRSSkB(status);
+        CHECK(result == -1);
+    }
+}
+#endif
+
+TEST_CASE("mallocTrim basic functionality")
+{
+    beast::Journal journal{beast::Journal::getNullSink()};
+
+    // Test with no tag
+    {
+        MallocTrimReport report = mallocTrim(std::nullopt, journal);
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+        // On Linux with glibc, should be supported
+        CHECK(report.supported == true);
+        // trimResult should be 0 or 1 (success indicators)
+        CHECK(report.trimResult >= 0);
+#else
+        // On other platforms, should be unsupported
+        CHECK(report.supported == false);
+        CHECK(report.trimResult == -1);
+        CHECK(report.rssBeforeKB == -1);
+        CHECK(report.rssAfterKB == -1);
+#endif
+    }
+
+    // Test with tag
+    {
+        MallocTrimReport report =
+            mallocTrim(std::optional<std::string>("test_tag"), journal);
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+        CHECK(report.supported == true);
+        CHECK(report.trimResult >= 0);
+#else
+        CHECK(report.supported == false);
+#endif
+    }
+}
+
+TEST_CASE("mallocTrim with debug logging")
+{
+    beast::Journal journal{beast::Journal::getNullSink()};
+
+    MallocTrimReport report =
+        mallocTrim(std::optional<std::string>("debug_test"), journal);
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+    CHECK(report.supported == true);
+    // The function should complete without crashing
+#else
+    CHECK(report.supported == false);
+#endif
+}
+
+TEST_CASE("mallocTrim repeated calls")
+{
+    beast::Journal journal{beast::Journal::getNullSink()};
+
+    // Call malloc_trim multiple times to ensure it's safe
+    for (int i = 0; i < 5; ++i)
+    {
+        MallocTrimReport report = mallocTrim(
+            std::optional<std::string>("iteration_" + std::to_string(i)),
+            journal);
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+        CHECK(report.supported == true);
+        CHECK(report.trimResult >= 0);
+#else
+        CHECK(report.supported == false);
+#endif
+    }
+}

src/xrpld/app/ledger/detail/LedgerMaster.cpp

@@ -261,6 +261,13 @@ LedgerMaster::setValidLedger(std::shared_ptr<Ledger const> const& l)
     (void)max_ledger_difference_;
     mValidLedgerSeq = l->info().seq;
 
+    if (l->info().seq % 100 == 0)
+    {
+        beast::Journal statsJournal = app_.journal("SHAMapStats");
+        l->stateMap().logTreeStats(statsJournal, "AccountStateMap");
+        l->txMap().logTreeStats(statsJournal, "TransactionMap");
+    }
+
     app_.getOPs().updateLocalTx(*l);
     app_.getSHAMapStore().onLedgerClosed(getValidatedLedger());
     mLedgerHistory.validatedLedger(l, consensusHash);

src/xrpld/app/main/Application.cpp

@@ -37,6 +37,7 @@
 #include <xrpld/shamap/NodeFamily.h>
 
 #include <xrpl/basics/ByteUtilities.h>
+#include <xrpl/basics/MallocTrim.h>
 #include <xrpl/basics/ResolverAsio.h>
 #include <xrpl/basics/random.h>
 #include <xrpl/beast/asio/io_latency_probe.h>
@@ -1106,6 +1107,8 @@ public:
                 << "; size after: " << cachedSLEs_.size();
         }
 
+        mallocTrim(std::optional<std::string>("doSweep"), m_journal);
+
         // Set timer to do another sweep later.
         setSweepTimer();
     }

src/xrpld/app/misc/NetworkOPs.cpp

@@ -34,6 +34,7 @@
 #include <xrpld/rpc/MPTokenIssuanceID.h>
 #include <xrpld/rpc/ServerHandler.h>
 
+#include <xrpl/basics/MallocTrim.h>
 #include <xrpl/basics/UptimeClock.h>
 #include <xrpl/basics/mulDiv.h>
 #include <xrpl/basics/safe_cast.h>
@@ -2546,10 +2547,14 @@ NetworkOPsImp::setMode(OperatingMode om)
     if (mMode == om)
         return;
 
+    auto const oldMode = mMode.load(std::memory_order_relaxed);
     mMode = om;
 
     accounting_.mode(om);
 
+    if (oldMode != OperatingMode::FULL && om == OperatingMode::FULL)
+        mallocTrim(std::optional<std::string>("SyncComplete"), m_journal);
+
     JLOG(m_journal.info()) << "STATE->" << strOperatingMode();
     pubServer();
 }

src/xrpld/app/misc/SHAMapStoreImp.cpp

@@ -5,6 +5,7 @@
 #include <xrpld/app/rdb/backend/SQLiteDatabase.h>
 #include <xrpld/core/ConfigSections.h>
 
+#include <xrpl/basics/MallocTrim.h>
 #include <xrpl/beast/core/CurrentThreadName.h>
 #include <xrpl/nodestore/Scheduler.h>
 #include <xrpl/nodestore/detail/DatabaseRotatingImp.h>
@@ -545,6 +546,8 @@ SHAMapStoreImp::clearCaches(LedgerIndex validatedSeq)
 {
     ledgerMaster_->clearLedgerCachePrior(validatedSeq);
     fullBelowCache_->clear();
+
+    mallocTrim(std::optional<std::string>("clearCaches"), journal_);
 }
 
 void
@@ -610,6 +613,8 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
         });
     if (healthWait() == stopping)
         return;
+
+    mallocTrim(std::optional<std::string>("clearPrior"), journal_);
 }
 
 SHAMapStoreImp::HealthResult