Merge branch 'develop' into vlntb/malloc-trim

include/xrpl/basics/MallocTrim.h

@@ -0,0 +1,70 @@
+#ifndef XRPL_BASICS_MALLOCTRIM_H_INCLUDED
+#define XRPL_BASICS_MALLOCTRIM_H_INCLUDED
+
+#include <xrpl/beast/utility/Journal.h>
+
+#include <optional>
+#include <string>
+
+namespace ripple {
+
+// -----------------------------------------------------------------------------
+// Allocator interaction note:
+// - This facility invokes glibc's malloc_trim(0) on Linux/glibc to request that
+//   ptmalloc return free heap pages to the OS.
+// - If an alternative allocator (e.g. jemalloc or tcmalloc) is linked or
+//   preloaded (LD_PRELOAD), calling glibc's malloc_trim typically has no effect
+//   on the *active* heap. The call is harmless but may not reclaim memory
+//   because those allocators manage their own arenas.
+// - Only glibc sbrk/arena space is eligible for trimming; large mmap-backed
+//   allocations are usually returned to the OS on free regardless of trimming.
+// - Call at known reclamation points (e.g., after cache sweeps / online delete)
+//   and consider rate limiting to avoid churn.
+// -----------------------------------------------------------------------------
+
+struct MallocTrimReport
+{
+    bool supported{false};
+    int trimResult{-1};
+    long rssBeforeKB{-1};
+    long rssAfterKB{-1};
+
+    [[nodiscard]] long
+    deltaKB() const noexcept
+    {
+        if (rssBeforeKB < 0 || rssAfterKB < 0)
+            return 0;
+        return rssAfterKB - rssBeforeKB;
+    }
+};
+
+/**
+ * @brief Attempt to return freed memory to the operating system.
+ *
+ * On Linux with glibc malloc, this issues ::malloc_trim(0), which may release
+ * free space from ptmalloc arenas back to the kernel. On other platforms, or if
+ * a different allocator is in use, this function is a no-op and the report will
+ * indicate that trimming is unsupported or had no effect.
+ *
+ * @param tag     Optional identifier for logging/debugging purposes.
+ * @param journal Journal for diagnostic logging.
+ * @return Report containing before/after metrics and the trim result.
+ *
+ * @note If an alternative allocator (jemalloc/tcmalloc) is linked or preloaded,
+ *       calling glibc's malloc_trim may have no effect on the active heap. The
+ *       call is harmless but typically does not reclaim memory under those
+ *       allocators.
+ *
+ * @note Only memory served from glibc's sbrk/arena heaps is eligible for trim.
+ *       Large allocations satisfied via mmap are usually returned on free
+ *       independently of trimming.
+ *
+ * @note Intended for use after operations that free significant memory (e.g.,
+ *       cache sweeps, ledger cleanup, online delete). Consider rate limiting.
+ */
+MallocTrimReport
+mallocTrim(std::optional<std::string> const& tag, beast::Journal journal);
+
+}  // namespace ripple
+
+#endif

src/libxrpl/basics/MallocTrim.cpp

@@ -0,0 +1,121 @@
+#include <xrpl/basics/Log.h>
+#include <xrpl/basics/MallocTrim.h>
+
+#include <boost/predef.h>
+
+#include <cstdio>
+#include <fstream>
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+#include <malloc.h>
+#include <unistd.h>
+
+namespace {
+pid_t const cachedPid = ::getpid();
+}  // namespace
+#endif
+
+namespace ripple {
+
+namespace detail {
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+
+long
+parseVmRSSkB(std::string const& status)
+{
+    std::istringstream iss(status);
+    std::string line;
+
+    while (std::getline(iss, line))
+    {
+        // Allow leading spaces/tabs before the key.
+        auto const firstNonWs = line.find_first_not_of(" \t");
+        if (firstNonWs == std::string::npos)
+            continue;
+
+        constexpr char key[] = "VmRSS:";
+        constexpr auto keyLen = sizeof(key) - 1;
+
+        // Require the line (after leading whitespace) to start with "VmRSS:".
+        // Check if we have enough characters and the substring matches.
+        if (firstNonWs + keyLen > line.size() ||
+            line.substr(firstNonWs, keyLen) != key)
+            continue;
+
+        // Move past "VmRSS:" and any following whitespace.
+        auto pos = firstNonWs + keyLen;
+        while (pos < line.size() &&
+               std::isspace(static_cast<unsigned char>(line[pos])))
+        {
+            ++pos;
+        }
+
+        long value = -1;
+        if (std::sscanf(line.c_str() + pos, "%ld", &value) == 1)
+            return value;
+
+        // Found the key but couldn't parse a number.
+        return -1;
+    }
+
+    // No VmRSS line found.
+    return -1;
+}
+
+#endif  // __GLIBC__ && BOOST_OS_LINUX
+
+}  // namespace detail
+
+MallocTrimReport
+mallocTrim(
+    [[maybe_unused]] std::optional<std::string> const& tag,
+    beast::Journal journal)
+{
+    MallocTrimReport report;
+
+#if !(defined(__GLIBC__) && BOOST_OS_LINUX)
+    JLOG(journal.debug()) << "malloc_trim not supported on this platform";
+#else
+
+    report.supported = true;
+
+    if (journal.debug())
+    {
+        auto readFile = [](std::string const& path) -> std::string {
+            std::ifstream ifs(path);
+            if (!ifs.is_open())
+                return {};
+            return std::string(
+                std::istreambuf_iterator<char>(ifs),
+                std::istreambuf_iterator<char>());
+        };
+
+        std::string const tagStr = tag.value_or("default");
+        std::string const statusPath =
+            "/proc/" + std::to_string(cachedPid) + "/status";
+
+        auto const statusBefore = readFile(statusPath);
+        report.rssBeforeKB = detail::parseVmRSSkB(statusBefore);
+
+        report.trimResult = ::malloc_trim(0);
+
+        auto const statusAfter = readFile(statusPath);
+        report.rssAfterKB = detail::parseVmRSSkB(statusAfter);
+
+        JLOG(journal.debug())
+            << "malloc_trim tag=" << tagStr << " result=" << report.trimResult
+            << " rss_before=" << report.rssBeforeKB << "kB"
+            << " rss_after=" << report.rssAfterKB << "kB"
+            << " delta=" << report.deltaKB() << "kB";
+    }
+    else
+    {
+        report.trimResult = ::malloc_trim(0);
+    }
+#endif
+
+    return report;
+}
+
+}  // namespace ripple

src/tests/libxrpl/basics/MallocTrim.cpp

@@ -0,0 +1,207 @@
+#include <xrpl/basics/MallocTrim.h>
+
+#include <boost/predef.h>
+
+#include <doctest/doctest.h>
+
+using namespace ripple;
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+namespace ripple::detail {
+long
+parseVmRSSkB(std::string const& status);
+}  // namespace ripple::detail
+#endif
+
+TEST_CASE("MallocTrimReport structure")
+{
+    // Test default construction
+    MallocTrimReport report;
+    CHECK(report.supported == false);
+    CHECK(report.trimResult == -1);
+    CHECK(report.rssBeforeKB == -1);
+    CHECK(report.rssAfterKB == -1);
+    CHECK(report.deltaKB() == 0);
+
+    // Test deltaKB calculation - memory freed
+    report.rssBeforeKB = 1000;
+    report.rssAfterKB = 800;
+    CHECK(report.deltaKB() == -200);
+
+    // Test deltaKB calculation - memory increased
+    report.rssBeforeKB = 500;
+    report.rssAfterKB = 600;
+    CHECK(report.deltaKB() == 100);
+
+    // Test deltaKB calculation - no change
+    report.rssBeforeKB = 1234;
+    report.rssAfterKB = 1234;
+    CHECK(report.deltaKB() == 0);
+}
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+TEST_CASE("parseVmRSSkB")
+{
+    using ripple::detail::parseVmRSSkB;
+
+    // Test standard format
+    {
+        std::string status = "VmRSS:      123456 kB\n";
+        long result = parseVmRSSkB(status);
+        CHECK(result == 123456);
+    }
+
+    // Test with multiple lines
+    {
+        std::string status =
+            "Name:   rippled\n"
+            "VmPeak:  1234567 kB\n"
+            "VmSize:  1234567 kB\n"
+            "VmRSS:      987654 kB\n"
+            "VmData:   123456 kB\n";
+        long result = parseVmRSSkB(status);
+        CHECK(result == 987654);
+    }
+
+    // Test with minimal whitespace
+    {
+        std::string status = "VmRSS: 42 kB";
+        long result = parseVmRSSkB(status);
+        CHECK(result == 42);
+    }
+
+    // Test with extra whitespace
+    {
+        std::string status = "VmRSS:          999999 kB";
+        long result = parseVmRSSkB(status);
+        CHECK(result == 999999);
+    }
+
+    // Test with tabs
+    {
+        std::string status = "VmRSS:\t\t12345 kB";
+        long result = parseVmRSSkB(status);
+        // Note: tabs are not explicitly handled as spaces, this documents
+        // current behavior
+        CHECK(result == 12345);
+    }
+
+    // Test zero value
+    {
+        std::string status = "VmRSS:      0 kB\n";
+        long result = parseVmRSSkB(status);
+        CHECK(result == 0);
+    }
+
+    // Test missing VmRSS
+    {
+        std::string status =
+            "Name:   rippled\n"
+            "VmPeak:  1234567 kB\n"
+            "VmSize:  1234567 kB\n";
+        long result = parseVmRSSkB(status);
+        CHECK(result == -1);
+    }
+
+    // Test empty string
+    {
+        std::string status = "";
+        long result = parseVmRSSkB(status);
+        CHECK(result == -1);
+    }
+
+    // Test malformed data (VmRSS but no number)
+    {
+        std::string status = "VmRSS:      \n";
+        long result = parseVmRSSkB(status);
+        // sscanf should fail to parse and return -1 unchanged
+        CHECK(result == -1);
+    }
+
+    // Test malformed data (VmRSS but invalid number)
+    {
+        std::string status = "VmRSS:      abc kB\n";
+        long result = parseVmRSSkB(status);
+        // sscanf should fail and return -1 unchanged
+        CHECK(result == -1);
+    }
+
+    // Test partial match (should not match "NotVmRSS:")
+    {
+        std::string status = "NotVmRSS:      123456 kB\n";
+        long result = parseVmRSSkB(status);
+        CHECK(result == -1);
+    }
+}
+#endif
+
+TEST_CASE("mallocTrim basic functionality")
+{
+    beast::Journal journal{beast::Journal::getNullSink()};
+
+    // Test with no tag
+    {
+        MallocTrimReport report = mallocTrim(std::nullopt, journal);
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+        // On Linux with glibc, should be supported
+        CHECK(report.supported == true);
+        // trimResult should be 0 or 1 (success indicators)
+        CHECK(report.trimResult >= 0);
+#else
+        // On other platforms, should be unsupported
+        CHECK(report.supported == false);
+        CHECK(report.trimResult == -1);
+        CHECK(report.rssBeforeKB == -1);
+        CHECK(report.rssAfterKB == -1);
+#endif
+    }
+
+    // Test with tag
+    {
+        MallocTrimReport report =
+            mallocTrim(std::optional<std::string>("test_tag"), journal);
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+        CHECK(report.supported == true);
+        CHECK(report.trimResult >= 0);
+#else
+        CHECK(report.supported == false);
+#endif
+    }
+}
+
+TEST_CASE("mallocTrim with debug logging")
+{
+    beast::Journal journal{beast::Journal::getNullSink()};
+
+    MallocTrimReport report =
+        mallocTrim(std::optional<std::string>("debug_test"), journal);
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+    CHECK(report.supported == true);
+    // The function should complete without crashing
+#else
+    CHECK(report.supported == false);
+#endif
+}
+
+TEST_CASE("mallocTrim repeated calls")
+{
+    beast::Journal journal{beast::Journal::getNullSink()};
+
+    // Call malloc_trim multiple times to ensure it's safe
+    for (int i = 0; i < 5; ++i)
+    {
+        MallocTrimReport report = mallocTrim(
+            std::optional<std::string>("iteration_" + std::to_string(i)),
+            journal);
+
+#if defined(__GLIBC__) && BOOST_OS_LINUX
+        CHECK(report.supported == true);
+        CHECK(report.trimResult >= 0);
+#else
+        CHECK(report.supported == false);
+#endif
+    }
+}

src/xrpld/app/ledger/OrderBookDB.cpp

@@ -7,6 +7,7 @@
 #include <xrpld/core/JobQueue.h>
 
 #include <xrpl/basics/Log.h>
+#include <xrpl/basics/MallocTrim.h>
 #include <xrpl/protocol/Indexes.h>
 
 namespace ripple {
@@ -154,6 +155,8 @@ OrderBookDB::update(std::shared_ptr<ReadView const> const& ledger)
     }
 
     app_.getLedgerMaster().newOrderBookDB();
+
+    mallocTrim(std::optional<std::string>("OrderBookUpdate"), j_);
 }
 
 void

src/xrpld/app/main/Application.cpp

@@ -37,6 +37,7 @@
 #include <xrpld/shamap/NodeFamily.h>
 
 #include <xrpl/basics/ByteUtilities.h>
+#include <xrpl/basics/MallocTrim.h>
 #include <xrpl/basics/ResolverAsio.h>
 #include <xrpl/basics/random.h>
 #include <xrpl/beast/asio/io_latency_probe.h>
@@ -1106,6 +1107,8 @@ public:
                 << "; size after: " << cachedSLEs_.size();
         }
 
+        mallocTrim(std::optional<std::string>("doSweep"), m_journal);
+
         // Set timer to do another sweep later.
         setSweepTimer();
     }

src/xrpld/app/misc/NetworkOPs.cpp

@@ -34,6 +34,7 @@
 #include <xrpld/rpc/MPTokenIssuanceID.h>
 #include <xrpld/rpc/ServerHandler.h>
 
+#include <xrpl/basics/MallocTrim.h>
 #include <xrpl/basics/UptimeClock.h>
 #include <xrpl/basics/mulDiv.h>
 #include <xrpl/basics/safe_cast.h>
@@ -2547,10 +2548,14 @@ NetworkOPsImp::setMode(OperatingMode om)
     if (mMode == om)
         return;
 
+    auto const oldMode = mMode.load(std::memory_order_relaxed);
     mMode = om;
 
     accounting_.mode(om);
 
+    if (oldMode != OperatingMode::FULL && om == OperatingMode::FULL)
+        mallocTrim(std::optional<std::string>("SyncComplete"), m_journal);
+
     JLOG(m_journal.info()) << "STATE->" << strOperatingMode();
     pubServer();
 }

src/xrpld/app/misc/SHAMapStoreImp.cpp

@@ -5,6 +5,7 @@
 #include <xrpld/app/rdb/backend/SQLiteDatabase.h>
 #include <xrpld/core/ConfigSections.h>
 
+#include <xrpl/basics/MallocTrim.h>
 #include <xrpl/beast/core/CurrentThreadName.h>
 #include <xrpl/nodestore/Scheduler.h>
 #include <xrpl/nodestore/detail/DatabaseRotatingImp.h>
@@ -545,6 +546,8 @@ SHAMapStoreImp::clearCaches(LedgerIndex validatedSeq)
 {
     ledgerMaster_->clearLedgerCachePrior(validatedSeq);
     fullBelowCache_->clear();
+
+    mallocTrim(std::optional<std::string>("clearCaches"), journal_);
 }
 
 void
@@ -610,6 +613,8 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
         });
     if (healthWait() == stopping)
         return;
+
+    mallocTrim(std::optional<std::string>("clearPrior"), journal_);
 }
 
 SHAMapStoreImp::HealthResult