mirror of
https://github.com/XRPLF/rippled.git
synced 2026-02-02 21:15:25 +00:00
Compare commits
3 Commits
vlntb/mall
...
ximinez/wi
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e7e252b108 | ||
|
|
ff4520cc45 | ||
|
|
16929f80bc |
2
.github/workflows/pre-commit.yml
vendored
2
.github/workflows/pre-commit.yml
vendored
@@ -11,7 +11,7 @@ on:
|
||||
jobs:
|
||||
# Call the workflow in the XRPLF/actions repo that runs the pre-commit hooks.
|
||||
run-hooks:
|
||||
uses: XRPLF/actions/.github/workflows/pre-commit.yml@282890f46d6921249d5659dd38babcb0bd8aef48
|
||||
uses: XRPLF/actions/.github/workflows/pre-commit.yml@320be44621ca2a080f05aeb15817c44b84518108
|
||||
with:
|
||||
runs_on: ubuntu-latest
|
||||
container: '{ "image": "ghcr.io/xrplf/ci/tools-rippled-pre-commit:sha-ab4d1f0" }'
|
||||
|
||||
2
.github/workflows/publish-docs.yml
vendored
2
.github/workflows/publish-docs.yml
vendored
@@ -36,7 +36,7 @@ jobs:
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
|
||||
- name: Get number of processors
|
||||
uses: XRPLF/actions/get-nproc@2ece4ec6ab7de266859a6f053571425b2bd684b6
|
||||
uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf
|
||||
id: nproc
|
||||
with:
|
||||
subtract: ${{ env.NPROC_SUBTRACT }}
|
||||
|
||||
@@ -101,13 +101,13 @@ jobs:
|
||||
steps:
|
||||
- name: Cleanup workspace (macOS and Windows)
|
||||
if: ${{ runner.os == 'macOS' || runner.os == 'Windows' }}
|
||||
uses: XRPLF/actions/cleanup-workspace@2ece4ec6ab7de266859a6f053571425b2bd684b6
|
||||
uses: XRPLF/actions/cleanup-workspace@cf0433aa74563aead044a1e395610c96d65a37cf
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
|
||||
- name: Prepare runner
|
||||
uses: XRPLF/actions/prepare-runner@f05cab7b8541eee6473aa42beb9d2fe35608a190
|
||||
uses: XRPLF/actions/prepare-runner@2cbf481018d930656e9276fcc20dc0e3a0be5b6d
|
||||
with:
|
||||
enable_ccache: ${{ inputs.ccache_enabled }}
|
||||
|
||||
@@ -119,7 +119,7 @@ jobs:
|
||||
uses: ./.github/actions/print-env
|
||||
|
||||
- name: Get number of processors
|
||||
uses: XRPLF/actions/get-nproc@2ece4ec6ab7de266859a6f053571425b2bd684b6
|
||||
uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf
|
||||
id: nproc
|
||||
with:
|
||||
subtract: ${{ inputs.nproc_subtract }}
|
||||
|
||||
6
.github/workflows/upload-conan-deps.yml
vendored
6
.github/workflows/upload-conan-deps.yml
vendored
@@ -64,13 +64,13 @@ jobs:
|
||||
steps:
|
||||
- name: Cleanup workspace (macOS and Windows)
|
||||
if: ${{ runner.os == 'macOS' || runner.os == 'Windows' }}
|
||||
uses: XRPLF/actions/cleanup-workspace@2ece4ec6ab7de266859a6f053571425b2bd684b6
|
||||
uses: XRPLF/actions/cleanup-workspace@cf0433aa74563aead044a1e395610c96d65a37cf
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
|
||||
- name: Prepare runner
|
||||
uses: XRPLF/actions/prepare-runner@f05cab7b8541eee6473aa42beb9d2fe35608a190
|
||||
uses: XRPLF/actions/prepare-runner@2cbf481018d930656e9276fcc20dc0e3a0be5b6d
|
||||
with:
|
||||
enable_ccache: false
|
||||
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
uses: ./.github/actions/print-env
|
||||
|
||||
- name: Get number of processors
|
||||
uses: XRPLF/actions/get-nproc@2ece4ec6ab7de266859a6f053571425b2bd684b6
|
||||
uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf
|
||||
id: nproc
|
||||
with:
|
||||
subtract: ${{ env.NPROC_SUBTRACT }}
|
||||
|
||||
@@ -51,7 +51,7 @@ set(is_amd64 FALSE)
|
||||
set(is_arm64 FALSE)
|
||||
if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64")
|
||||
set(is_amd64 TRUE)
|
||||
elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
|
||||
elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64")
|
||||
set(is_arm64 TRUE)
|
||||
else ()
|
||||
message(FATAL_ERROR "Unknown architecture: ${CMAKE_SYSTEM_PROCESSOR}")
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
#ifndef XRPL_BASICS_MALLOCTRIM_H_INCLUDED
|
||||
#define XRPL_BASICS_MALLOCTRIM_H_INCLUDED
|
||||
|
||||
#include <xrpl/beast/utility/Journal.h>
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
// cSpell:ignore ptmalloc
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Allocator interaction note:
|
||||
// - This facility invokes glibc's malloc_trim(0) on Linux/glibc to request that
|
||||
// ptmalloc return free heap pages to the OS.
|
||||
// - If an alternative allocator (e.g. jemalloc or tcmalloc) is linked or
|
||||
// preloaded (LD_PRELOAD), calling glibc's malloc_trim typically has no effect
|
||||
// on the *active* heap. The call is harmless but may not reclaim memory
|
||||
// because those allocators manage their own arenas.
|
||||
// - Only glibc sbrk/arena space is eligible for trimming; large mmap-backed
|
||||
// allocations are usually returned to the OS on free regardless of trimming.
|
||||
// - Call at known reclamation points (e.g., after cache sweeps / online delete)
|
||||
// and consider rate limiting to avoid churn.
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
struct MallocTrimReport
|
||||
{
|
||||
bool supported{false};
|
||||
int trimResult{-1};
|
||||
long rssBeforeKB{-1};
|
||||
long rssAfterKB{-1};
|
||||
long long durationUs{-1};
|
||||
long minfltDelta{-1};
|
||||
long majfltDelta{-1};
|
||||
|
||||
[[nodiscard]] long
|
||||
deltaKB() const noexcept
|
||||
{
|
||||
if (rssBeforeKB < 0 || rssAfterKB < 0)
|
||||
return 0;
|
||||
return rssAfterKB - rssBeforeKB;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Attempt to return freed memory to the operating system.
|
||||
*
|
||||
* On Linux with glibc malloc, this issues ::malloc_trim(0), which may release
|
||||
* free space from ptmalloc arenas back to the kernel. On other platforms, or if
|
||||
* a different allocator is in use, this function is a no-op and the report will
|
||||
* indicate that trimming is unsupported or had no effect.
|
||||
*
|
||||
* @param tag Optional identifier for logging/debugging purposes.
|
||||
* @param journal Journal for diagnostic logging.
|
||||
* @return Report containing before/after metrics and the trim result.
|
||||
*
|
||||
* @note If an alternative allocator (jemalloc/tcmalloc) is linked or preloaded,
|
||||
* calling glibc's malloc_trim may have no effect on the active heap. The
|
||||
* call is harmless but typically does not reclaim memory under those
|
||||
* allocators.
|
||||
*
|
||||
* @note Only memory served from glibc's sbrk/arena heaps is eligible for trim.
|
||||
* Large allocations satisfied via mmap are usually returned on free
|
||||
* independently of trimming.
|
||||
*
|
||||
* @note Intended for use after operations that free significant memory (e.g.,
|
||||
* cache sweeps, ledger cleanup, online delete). Consider rate limiting.
|
||||
*/
|
||||
MallocTrimReport
|
||||
mallocTrim(std::optional<std::string> const& tag, beast::Journal journal);
|
||||
|
||||
} // namespace xrpl
|
||||
|
||||
#endif
|
||||
@@ -1,173 +0,0 @@
|
||||
#include <xrpl/basics/Log.h>
|
||||
#include <xrpl/basics/MallocTrim.h>
|
||||
|
||||
#include <boost/predef.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <fstream>
|
||||
// #include <thread>
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
#include <sys/resource.h>
|
||||
|
||||
#include <malloc.h>
|
||||
#include <unistd.h>
|
||||
|
||||
// Require RUSAGE_THREAD for thread-scoped page fault tracking
|
||||
#ifndef RUSAGE_THREAD
|
||||
#error "MallocTrim rusage instrumentation requires RUSAGE_THREAD on Linux/glibc"
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
pid_t const cachedPid = ::getpid();
|
||||
|
||||
bool
|
||||
getRusageThread(struct rusage& ru)
|
||||
{
|
||||
return ::getrusage(RUSAGE_THREAD, &ru) == 0;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
#endif
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
namespace detail {
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
|
||||
long
|
||||
parseVmRSSkB(std::string const& status)
|
||||
{
|
||||
std::istringstream iss(status);
|
||||
std::string line;
|
||||
|
||||
while (std::getline(iss, line))
|
||||
{
|
||||
// Allow leading spaces/tabs before the key.
|
||||
auto const firstNonWs = line.find_first_not_of(" \t");
|
||||
if (firstNonWs == std::string::npos)
|
||||
continue;
|
||||
|
||||
constexpr char key[] = "VmRSS:";
|
||||
constexpr auto keyLen = sizeof(key) - 1;
|
||||
|
||||
// Require the line (after leading whitespace) to start with "VmRSS:".
|
||||
// Check if we have enough characters and the substring matches.
|
||||
if (firstNonWs + keyLen > line.size() ||
|
||||
line.substr(firstNonWs, keyLen) != key)
|
||||
continue;
|
||||
|
||||
// Move past "VmRSS:" and any following whitespace.
|
||||
auto pos = firstNonWs + keyLen;
|
||||
while (pos < line.size() &&
|
||||
std::isspace(static_cast<unsigned char>(line[pos])))
|
||||
{
|
||||
++pos;
|
||||
}
|
||||
|
||||
long value = -1;
|
||||
if (std::sscanf(line.c_str() + pos, "%ld", &value) == 1)
|
||||
return value;
|
||||
|
||||
// Found the key but couldn't parse a number.
|
||||
return -1;
|
||||
}
|
||||
|
||||
// No VmRSS line found.
|
||||
return -1;
|
||||
}
|
||||
|
||||
#endif // __GLIBC__ && BOOST_OS_LINUX
|
||||
|
||||
} // namespace detail
|
||||
|
||||
MallocTrimReport
|
||||
mallocTrim(
|
||||
[[maybe_unused]] std::optional<std::string> const& tag,
|
||||
beast::Journal journal)
|
||||
{
|
||||
MallocTrimReport report;
|
||||
|
||||
#if !(defined(__GLIBC__) && BOOST_OS_LINUX)
|
||||
JLOG(journal.debug()) << "malloc_trim not supported on this platform";
|
||||
#else
|
||||
|
||||
report.supported = true;
|
||||
|
||||
// Launch malloc_trim on a separate thread (async, fire-and-forget)
|
||||
// std::thread trimThread([tag, journal]() {
|
||||
if (journal.debug())
|
||||
{
|
||||
auto readFile = [](std::string const& path) -> std::string {
|
||||
std::ifstream ifs(path);
|
||||
if (!ifs.is_open())
|
||||
return {};
|
||||
return std::string(
|
||||
std::istreambuf_iterator<char>(ifs),
|
||||
std::istreambuf_iterator<char>());
|
||||
};
|
||||
|
||||
std::string const tagStr = tag.value_or("default");
|
||||
std::string const statusPath =
|
||||
"/proc/" + std::to_string(cachedPid) + "/status";
|
||||
|
||||
auto const statusBefore = readFile(statusPath);
|
||||
long const rssBeforeKB = detail::parseVmRSSkB(statusBefore);
|
||||
|
||||
struct rusage ru0
|
||||
{
|
||||
};
|
||||
bool const have_ru0 = getRusageThread(ru0);
|
||||
|
||||
auto const t0 = std::chrono::steady_clock::now();
|
||||
int const trimResult = ::malloc_trim(0);
|
||||
auto const t1 = std::chrono::steady_clock::now();
|
||||
|
||||
struct rusage ru1
|
||||
{
|
||||
};
|
||||
bool const have_ru1 = getRusageThread(ru1);
|
||||
|
||||
auto const statusAfter = readFile(statusPath);
|
||||
long const rssAfterKB = detail::parseVmRSSkB(statusAfter);
|
||||
|
||||
long long const durationUs =
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0)
|
||||
.count();
|
||||
|
||||
long minfltDelta = -1;
|
||||
long majfltDelta = -1;
|
||||
if (have_ru0 && have_ru1)
|
||||
{
|
||||
minfltDelta = ru1.ru_minflt - ru0.ru_minflt;
|
||||
majfltDelta = ru1.ru_majflt - ru0.ru_majflt;
|
||||
}
|
||||
|
||||
long const deltaKB = (rssBeforeKB < 0 || rssAfterKB < 0)
|
||||
? 0
|
||||
: (rssAfterKB - rssBeforeKB);
|
||||
|
||||
JLOG(journal.debug())
|
||||
<< "malloc_trim tag=" << tagStr << " result=" << trimResult
|
||||
<< " rss_before=" << rssBeforeKB << "kB"
|
||||
<< " rss_after=" << rssAfterKB << "kB"
|
||||
<< " delta=" << deltaKB << "kB"
|
||||
<< " duration_us=" << durationUs << " minflt_delta=" << minfltDelta
|
||||
<< " majflt_delta=" << majfltDelta;
|
||||
}
|
||||
else
|
||||
{
|
||||
::malloc_trim(0);
|
||||
}
|
||||
// });
|
||||
|
||||
// trimThread.detach();
|
||||
|
||||
#endif
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -1,203 +0,0 @@
|
||||
#include <xrpl/basics/MallocTrim.h>
|
||||
|
||||
#include <boost/predef.h>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
using namespace xrpl;
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
namespace xrpl::detail {
|
||||
long
|
||||
parseVmRSSkB(std::string const& status);
|
||||
} // namespace xrpl::detail
|
||||
#endif
|
||||
|
||||
TEST(MallocTrimReport, structure)
|
||||
{
|
||||
// Test default construction
|
||||
MallocTrimReport report;
|
||||
EXPECT_EQ(report.supported, false);
|
||||
EXPECT_EQ(report.trimResult, -1);
|
||||
EXPECT_EQ(report.rssBeforeKB, -1);
|
||||
EXPECT_EQ(report.rssAfterKB, -1);
|
||||
EXPECT_EQ(report.deltaKB(), 0);
|
||||
|
||||
// Test deltaKB calculation - memory freed
|
||||
report.rssBeforeKB = 1000;
|
||||
report.rssAfterKB = 800;
|
||||
EXPECT_EQ(report.deltaKB(), -200);
|
||||
|
||||
// Test deltaKB calculation - memory increased
|
||||
report.rssBeforeKB = 500;
|
||||
report.rssAfterKB = 600;
|
||||
EXPECT_EQ(report.deltaKB(), 100);
|
||||
|
||||
// Test deltaKB calculation - no change
|
||||
report.rssBeforeKB = 1234;
|
||||
report.rssAfterKB = 1234;
|
||||
EXPECT_EQ(report.deltaKB(), 0);
|
||||
}
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
TEST(parseVmRSSkB, standard_format)
|
||||
{
|
||||
using xrpl::detail::parseVmRSSkB;
|
||||
|
||||
// Test standard format
|
||||
{
|
||||
std::string status = "VmRSS: 123456 kB\n";
|
||||
long result = parseVmRSSkB(status);
|
||||
EXPECT_EQ(result, 123456);
|
||||
}
|
||||
|
||||
// Test with multiple lines
|
||||
{
|
||||
std::string status =
|
||||
"Name: xrpld\n"
|
||||
"VmPeak: 1234567 kB\n"
|
||||
"VmSize: 1234567 kB\n"
|
||||
"VmRSS: 987654 kB\n"
|
||||
"VmData: 123456 kB\n";
|
||||
long result = parseVmRSSkB(status);
|
||||
EXPECT_EQ(result, 987654);
|
||||
}
|
||||
|
||||
// Test with minimal whitespace
|
||||
{
|
||||
std::string status = "VmRSS: 42 kB";
|
||||
long result = parseVmRSSkB(status);
|
||||
EXPECT_EQ(result, 42);
|
||||
}
|
||||
|
||||
// Test with extra whitespace
|
||||
{
|
||||
std::string status = "VmRSS: 999999 kB";
|
||||
long result = parseVmRSSkB(status);
|
||||
EXPECT_EQ(result, 999999);
|
||||
}
|
||||
|
||||
// Test with tabs
|
||||
{
|
||||
std::string status = "VmRSS:\t\t12345 kB";
|
||||
long result = parseVmRSSkB(status);
|
||||
// Note: tabs are not explicitly handled as spaces, this documents
|
||||
// current behavior
|
||||
EXPECT_EQ(result, 12345);
|
||||
}
|
||||
|
||||
// Test zero value
|
||||
{
|
||||
std::string status = "VmRSS: 0 kB\n";
|
||||
long result = parseVmRSSkB(status);
|
||||
EXPECT_EQ(result, 0);
|
||||
}
|
||||
|
||||
// Test missing VmRSS
|
||||
{
|
||||
std::string status =
|
||||
"Name: xrpld\n"
|
||||
"VmPeak: 1234567 kB\n"
|
||||
"VmSize: 1234567 kB\n";
|
||||
long result = parseVmRSSkB(status);
|
||||
EXPECT_EQ(result, -1);
|
||||
}
|
||||
|
||||
// Test empty string
|
||||
{
|
||||
std::string status = "";
|
||||
long result = parseVmRSSkB(status);
|
||||
EXPECT_EQ(result, -1);
|
||||
}
|
||||
|
||||
// Test malformed data (VmRSS but no number)
|
||||
{
|
||||
std::string status = "VmRSS: \n";
|
||||
long result = parseVmRSSkB(status);
|
||||
// sscanf should fail to parse and return -1 unchanged
|
||||
EXPECT_EQ(result, -1);
|
||||
}
|
||||
|
||||
// Test malformed data (VmRSS but invalid number)
|
||||
{
|
||||
std::string status = "VmRSS: abc kB\n";
|
||||
long result = parseVmRSSkB(status);
|
||||
// sscanf should fail and return -1 unchanged
|
||||
EXPECT_EQ(result, -1);
|
||||
}
|
||||
|
||||
// Test partial match (should not match "NotVmRSS:")
|
||||
{
|
||||
std::string status = "NotVmRSS: 123456 kB\n";
|
||||
long result = parseVmRSSkB(status);
|
||||
EXPECT_EQ(result, -1);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST(mallocTrim, basic_functionality)
|
||||
{
|
||||
beast::Journal journal{beast::Journal::getNullSink()};
|
||||
|
||||
// Test with no tag
|
||||
{
|
||||
MallocTrimReport report = mallocTrim(std::nullopt, journal);
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
// On Linux with glibc, should be supported
|
||||
EXPECT_EQ(report.supported, true);
|
||||
// trimResult should be 0 or 1 (success indicators)
|
||||
EXPECT_GE(report.trimResult, 0);
|
||||
#else
|
||||
// On other platforms, should be unsupported
|
||||
EXPECT_EQ(report.supported, false);
|
||||
EXPECT_EQ(report.trimResult, -1);
|
||||
EXPECT_EQ(report.rssBeforeKB, -1);
|
||||
EXPECT_EQ(report.rssAfterKB, -1);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Test with tag
|
||||
{
|
||||
MallocTrimReport report = mallocTrim(std::optional<std::string>("test_tag"), journal);
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
EXPECT_EQ(report.supported, true);
|
||||
EXPECT_GE(report.trimResult, 0);
|
||||
#else
|
||||
EXPECT_EQ(report.supported, false);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
TEST(mallocTrim, debug_logging)
|
||||
{
|
||||
beast::Journal journal{beast::Journal::getNullSink()};
|
||||
|
||||
MallocTrimReport report = mallocTrim(std::optional<std::string>("debug_test"), journal);
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
EXPECT_EQ(report.supported, true);
|
||||
// The function should complete without crashing
|
||||
#else
|
||||
EXPECT_EQ(report.supported, false);
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST(mallocTrim, repeated_calls)
|
||||
{
|
||||
beast::Journal journal{beast::Journal::getNullSink()};
|
||||
|
||||
// Call malloc_trim multiple times to ensure it's safe
|
||||
for (int i = 0; i < 5; ++i)
|
||||
{
|
||||
MallocTrimReport report = mallocTrim(std::optional<std::string>("iteration_" + std::to_string(i)), journal);
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
EXPECT_EQ(report.supported, true);
|
||||
EXPECT_GE(report.trimResult, 0);
|
||||
#else
|
||||
EXPECT_EQ(report.supported, false);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,6 @@
|
||||
#include <xrpld/shamap/NodeFamily.h>
|
||||
|
||||
#include <xrpl/basics/ByteUtilities.h>
|
||||
#include <xrpl/basics/MallocTrim.h>
|
||||
#include <xrpl/basics/ResolverAsio.h>
|
||||
#include <xrpl/basics/random.h>
|
||||
#include <xrpl/beast/asio/io_latency_probe.h>
|
||||
@@ -991,8 +990,6 @@ public:
|
||||
<< "; size after: " << cachedSLEs_.size();
|
||||
}
|
||||
|
||||
mallocTrim(std::optional<std::string>("doSweep"), m_journal);
|
||||
|
||||
// Set timer to do another sweep later.
|
||||
setSweepTimer();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user