mirror of
https://github.com/XRPLF/rippled.git
synced 2026-02-22 23:02:45 +00:00
Compare commits
23 Commits
vlntb/mall
...
pratik/Fix
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2201264858 | ||
|
|
76a49000e1 | ||
|
|
ec64579e22 | ||
|
|
7319b8f46b | ||
|
|
0893228637 | ||
|
|
2397ad7e9b | ||
|
|
e8ca732fa4 | ||
|
|
f04338a1e1 | ||
|
|
b926559def | ||
|
|
550ba266fc | ||
|
|
6b94f7ae91 | ||
|
|
09b8602bd7 | ||
|
|
cf3bedb007 | ||
|
|
9bad60173e | ||
|
|
580de04cfc | ||
|
|
4a8c6c884f | ||
|
|
8eda79000d | ||
|
|
5d58331ad6 | ||
|
|
376abfc80b | ||
|
|
17479dd209 | ||
|
|
b405d2326b | ||
|
|
45e53a885a | ||
|
|
78f4ef1a65 |
4
.github/scripts/strategy-matrix/generate.py
vendored
4
.github/scripts/strategy-matrix/generate.py
vendored
@@ -243,14 +243,14 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
|
||||
# Add ASAN + UBSAN configuration.
|
||||
configurations.append(
|
||||
{
|
||||
"config_name": config_name + "-asan-ubsan",
|
||||
"config_name": config_name + "-asan",
|
||||
"cmake_args": cmake_args,
|
||||
"cmake_target": cmake_target,
|
||||
"build_only": build_only,
|
||||
"build_type": build_type,
|
||||
"os": os,
|
||||
"architecture": architecture,
|
||||
"sanitizers": "address,undefinedbehavior",
|
||||
"sanitizers": "address",
|
||||
}
|
||||
)
|
||||
# TSAN is deactivated due to seg faults with latest compilers.
|
||||
|
||||
11
.github/workflows/reusable-build-test-config.yml
vendored
11
.github/workflows/reusable-build-test-config.yml
vendored
@@ -76,7 +76,7 @@ jobs:
|
||||
name: ${{ inputs.config_name }}
|
||||
runs-on: ${{ fromJSON(inputs.runs_on) }}
|
||||
container: ${{ inputs.image != '' && inputs.image || null }}
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: ${{ inputs.sanitizers != '' && 360 || 60 }}
|
||||
env:
|
||||
# Use a namespace to keep the objects separate for each configuration.
|
||||
CCACHE_NAMESPACE: ${{ inputs.config_name }}
|
||||
@@ -205,7 +205,11 @@ jobs:
|
||||
- name: Set sanitizer options
|
||||
if: ${{ !inputs.build_only && env.SANITIZERS_ENABLED == 'true' }}
|
||||
run: |
|
||||
echo "ASAN_OPTIONS=print_stacktrace=1:detect_container_overflow=0:suppressions=${GITHUB_WORKSPACE}/sanitizers/suppressions/asan.supp" >> ${GITHUB_ENV}
|
||||
ASAN_OPTS="halt_on_error=0:use_sigaltstack=0:print_stacktrace=1:detect_container_overflow=0:detect_stack_use_after_return=0:suppressions=${GITHUB_WORKSPACE}/sanitizers/suppressions/asan.supp"
|
||||
if [[ "${{ inputs.config_name }}" == *gcc* ]]; then
|
||||
ASAN_OPTS="${ASAN_OPTS}:alloc_dealloc_mismatch=0"
|
||||
fi
|
||||
echo "ASAN_OPTIONS=${ASAN_OPTS}" >> ${GITHUB_ENV}
|
||||
echo "TSAN_OPTIONS=second_deadlock_stack=1:halt_on_error=0:suppressions=${GITHUB_WORKSPACE}/sanitizers/suppressions/tsan.supp" >> ${GITHUB_ENV}
|
||||
echo "UBSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/sanitizers/suppressions/ubsan.supp" >> ${GITHUB_ENV}
|
||||
echo "LSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/sanitizers/suppressions/lsan.supp" >> ${GITHUB_ENV}
|
||||
@@ -228,8 +232,9 @@ jobs:
|
||||
working-directory: ${{ runner.os == 'Windows' && format('{0}/{1}', env.BUILD_DIR, inputs.build_type) || env.BUILD_DIR }}
|
||||
env:
|
||||
BUILD_NPROC: ${{ steps.nproc.outputs.nproc }}
|
||||
PARALLELISM: ${{ env.SANITIZERS_ENABLED == 'true' && '1' || steps.nproc.outputs.nproc }}
|
||||
run: |
|
||||
./xrpld --unittest --unittest-jobs "${BUILD_NPROC}"
|
||||
./xrpld --unittest --unittest-jobs 1
|
||||
|
||||
- name: Debug failure (Linux)
|
||||
if: ${{ failure() && runner.os == 'Linux' && !inputs.build_only }}
|
||||
|
||||
@@ -17,12 +17,10 @@ find_dependency(Boost
|
||||
chrono
|
||||
container
|
||||
context
|
||||
coroutine
|
||||
date_time
|
||||
filesystem
|
||||
program_options
|
||||
regex
|
||||
system
|
||||
thread)
|
||||
#[=========================================================[
|
||||
OpenSSL
|
||||
|
||||
@@ -22,7 +22,7 @@ target_compile_definitions(
|
||||
BOOST_FILESYSTEM_NO_DEPRECATED
|
||||
>
|
||||
$<$<NOT:$<BOOL:${boost_show_deprecated}>>:
|
||||
BOOST_COROUTINES_NO_DEPRECATION_WARNING
|
||||
BOOST_COROUTINES2_NO_DEPRECATION_WARNING
|
||||
BOOST_BEAST_ALLOW_DEPRECATED
|
||||
BOOST_FILESYSTEM_DEPRECATED
|
||||
>
|
||||
|
||||
@@ -4,13 +4,12 @@ include(XrplSanitizers)
|
||||
find_package(Boost REQUIRED
|
||||
COMPONENTS chrono
|
||||
container
|
||||
coroutine
|
||||
context
|
||||
date_time
|
||||
filesystem
|
||||
json
|
||||
program_options
|
||||
regex
|
||||
system
|
||||
thread)
|
||||
|
||||
add_library(xrpl_boost INTERFACE)
|
||||
@@ -21,7 +20,7 @@ target_link_libraries(
|
||||
INTERFACE Boost::headers
|
||||
Boost::chrono
|
||||
Boost::container
|
||||
Boost::coroutine
|
||||
Boost::context
|
||||
Boost::date_time
|
||||
Boost::filesystem
|
||||
Boost::json
|
||||
@@ -32,6 +31,26 @@ target_link_libraries(
|
||||
if (Boost_COMPILER)
|
||||
target_link_libraries(xrpl_boost INTERFACE Boost::disable_autolinking)
|
||||
endif ()
|
||||
|
||||
# GCC 14+ has a false positive -Wuninitialized warning in Boost.Coroutine2's
|
||||
# state.hpp when compiled with -O3. This is due to GCC's intentional behavior
|
||||
# change (Bug #98871, #119388) where warnings from inlined system header code
|
||||
# are no longer suppressed by -isystem. The warning occurs in operator|= in
|
||||
# boost/coroutine2/detail/state.hpp when inlined from push_control_block::destroy().
|
||||
# See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119388
|
||||
if (is_gcc AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 14)
|
||||
target_compile_options(xrpl_boost INTERFACE -Wno-uninitialized)
|
||||
endif ()
|
||||
|
||||
# Boost.Context's ucontext backend has ASAN fiber-switching annotations
|
||||
# (start/finish_switch_fiber) that are compiled in when BOOST_USE_ASAN is defined.
|
||||
# This tells ASAN about coroutine stack switches, preventing false positive
|
||||
# stack-use-after-scope errors. BOOST_USE_UCONTEXT ensures the ucontext backend
|
||||
# is selected (fcontext does not support ASAN annotations).
|
||||
# These defines must match what Boost was compiled with (see conan/profiles/sanitizers).
|
||||
if (enable_asan)
|
||||
target_compile_definitions(xrpl_boost INTERFACE BOOST_USE_ASAN BOOST_USE_UCONTEXT)
|
||||
endif ()
|
||||
if (SANITIZERS_ENABLED AND is_clang)
|
||||
# TODO: gcc does not support -fsanitize-blacklist...can we do something else for gcc ?
|
||||
if (NOT Boost_INCLUDE_DIRS AND TARGET Boost::headers)
|
||||
|
||||
@@ -7,16 +7,21 @@ include(default)
|
||||
{% if compiler == "gcc" %}
|
||||
{% if "address" in sanitizers or "thread" in sanitizers or "undefinedbehavior" in sanitizers %}
|
||||
{% set sanitizer_list = [] %}
|
||||
{% set defines = [] %}
|
||||
{% set model_code = "" %}
|
||||
{% set extra_cxxflags = ["-fno-omit-frame-pointer", "-O1", "-Wno-stringop-overflow"] %}
|
||||
|
||||
{% if "address" in sanitizers %}
|
||||
{% set _ = sanitizer_list.append("address") %}
|
||||
{% set model_code = "-mcmodel=large" %}
|
||||
{% set _ = defines.append("BOOST_USE_ASAN")%}
|
||||
{% set _ = defines.append("BOOST_USE_UCONTEXT")%}
|
||||
{% elif "thread" in sanitizers %}
|
||||
{% set _ = sanitizer_list.append("thread") %}
|
||||
{% set model_code = "-mcmodel=medium" %}
|
||||
{% set _ = extra_cxxflags.append("-Wno-tsan") %}
|
||||
{% set _ = defines.append("BOOST_USE_TSAN")%}
|
||||
{% set _ = defines.append("BOOST_USE_UCONTEXT")%}
|
||||
{% endif %}
|
||||
|
||||
{% if "undefinedbehavior" in sanitizers %}
|
||||
@@ -29,16 +34,22 @@ include(default)
|
||||
tools.build:cxxflags+=['{{sanitizer_flags}} {{" ".join(extra_cxxflags)}}']
|
||||
tools.build:sharedlinkflags+=['{{sanitizer_flags}}']
|
||||
tools.build:exelinkflags+=['{{sanitizer_flags}}']
|
||||
tools.build:defines+={{defines}}
|
||||
{% endif %}
|
||||
{% elif compiler == "apple-clang" or compiler == "clang" %}
|
||||
{% if "address" in sanitizers or "thread" in sanitizers or "undefinedbehavior" in sanitizers %}
|
||||
{% set sanitizer_list = [] %}
|
||||
{% set defines = [] %}
|
||||
{% set extra_cxxflags = ["-fno-omit-frame-pointer", "-O1"] %}
|
||||
|
||||
{% if "address" in sanitizers %}
|
||||
{% set _ = sanitizer_list.append("address") %}
|
||||
{% set _ = defines.append("BOOST_USE_ASAN")%}
|
||||
{% set _ = defines.append("BOOST_USE_UCONTEXT")%}
|
||||
{% elif "thread" in sanitizers %}
|
||||
{% set _ = sanitizer_list.append("thread") %}
|
||||
{% set _ = defines.append("BOOST_USE_TSAN")%}
|
||||
{% set _ = defines.append("BOOST_USE_UCONTEXT")%}
|
||||
{% endif %}
|
||||
|
||||
{% if "undefinedbehavior" in sanitizers %}
|
||||
@@ -52,8 +63,24 @@ include(default)
|
||||
tools.build:cxxflags+=['{{sanitizer_flags}} {{" ".join(extra_cxxflags)}}']
|
||||
tools.build:sharedlinkflags+=['{{sanitizer_flags}}']
|
||||
tools.build:exelinkflags+=['{{sanitizer_flags}}']
|
||||
tools.build:defines+={{defines}}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
tools.info.package_id:confs+=["tools.build:cxxflags", "tools.build:exelinkflags", "tools.build:sharedlinkflags"]
|
||||
tools.info.package_id:confs+=["tools.build:cxxflags", "tools.build:exelinkflags", "tools.build:sharedlinkflags", "tools.build:defines"]
|
||||
|
||||
[options]
|
||||
{% if sanitizers %}
|
||||
{% if "address" in sanitizers %}
|
||||
# Build Boost.Context with ucontext backend (not fcontext) so that
|
||||
# ASAN fiber-switching annotations (__sanitizer_start/finish_switch_fiber)
|
||||
# are compiled into the library. fcontext (assembly) has no ASAN support.
|
||||
# define=BOOST_USE_ASAN=1 is critical: it must be defined when building
|
||||
# Boost.Context itself so the ucontext backend compiles in the ASAN annotations.
|
||||
boost/*:extra_b2_flags=context-impl=ucontext address-sanitizer=on define=BOOST_USE_ASAN=1
|
||||
boost/*:without_context=False
|
||||
# Boost stacktrace fails to build with some sanitizers
|
||||
boost/*:without_stacktrace=True
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
15
conanfile.py
15
conanfile.py
@@ -1,4 +1,5 @@
|
||||
import re
|
||||
import os
|
||||
|
||||
from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
|
||||
|
||||
@@ -57,6 +58,9 @@ class Xrpl(ConanFile):
|
||||
"tests": False,
|
||||
"unity": False,
|
||||
"xrpld": False,
|
||||
"boost/*:without_context": False,
|
||||
"boost/*:without_coroutine": True,
|
||||
"boost/*:without_coroutine2": False,
|
||||
"date/*:header_only": True,
|
||||
"ed25519/*:shared": False,
|
||||
"grpc/*:shared": False,
|
||||
@@ -125,6 +129,14 @@ class Xrpl(ConanFile):
|
||||
self.options["boost"].visibility = "global"
|
||||
if self.settings.compiler in ["clang", "gcc"]:
|
||||
self.options["boost"].without_cobalt = True
|
||||
self.options["boost"].without_context = False
|
||||
self.options["boost"].without_coroutine = True
|
||||
self.options["boost"].without_coroutine2 = False
|
||||
# Check if environment variable exists
|
||||
if "SANITIZERS" in os.environ:
|
||||
sanitizers = os.environ["SANITIZERS"]
|
||||
if "address" in sanitizers.lower():
|
||||
self.default_options["fPIC"] = False
|
||||
|
||||
def requirements(self):
|
||||
# Conan 2 requires transitive headers to be specified
|
||||
@@ -196,7 +208,8 @@ class Xrpl(ConanFile):
|
||||
"boost::headers",
|
||||
"boost::chrono",
|
||||
"boost::container",
|
||||
"boost::coroutine",
|
||||
"boost::context",
|
||||
"boost::coroutine2",
|
||||
"boost::date_time",
|
||||
"boost::filesystem",
|
||||
"boost::json",
|
||||
|
||||
@@ -99,6 +99,7 @@ words:
|
||||
- endmacro
|
||||
- exceptioned
|
||||
- Falco
|
||||
- fcontext
|
||||
- finalizers
|
||||
- firewalled
|
||||
- fmtdur
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <xrpl/beast/utility/Journal.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
// cSpell:ignore ptmalloc
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Allocator interaction note:
|
||||
// - This facility invokes glibc's malloc_trim(0) on Linux/glibc to request that
|
||||
// ptmalloc return free heap pages to the OS.
|
||||
// - If an alternative allocator (e.g. jemalloc or tcmalloc) is linked or
|
||||
// preloaded (LD_PRELOAD), calling glibc's malloc_trim typically has no effect
|
||||
// on the *active* heap. The call is harmless but may not reclaim memory
|
||||
// because those allocators manage their own arenas.
|
||||
// - Only glibc sbrk/arena space is eligible for trimming; large mmap-backed
|
||||
// allocations are usually returned to the OS on free regardless of trimming.
|
||||
// - Call at known reclamation points (e.g., after cache sweeps / online delete)
|
||||
// and consider rate limiting to avoid churn.
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
struct MallocTrimReport
|
||||
{
|
||||
bool supported{false};
|
||||
int trimResult{-1};
|
||||
long rssBeforeKB{-1};
|
||||
long rssAfterKB{-1};
|
||||
std::chrono::microseconds durationUs{-1};
|
||||
long minfltDelta{-1};
|
||||
long majfltDelta{-1};
|
||||
|
||||
[[nodiscard]] long
|
||||
deltaKB() const noexcept
|
||||
{
|
||||
if (rssBeforeKB < 0 || rssAfterKB < 0)
|
||||
return 0;
|
||||
return rssAfterKB - rssBeforeKB;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Attempt to return freed memory to the operating system.
|
||||
*
|
||||
* On Linux with glibc malloc, this issues ::malloc_trim(0), which may release
|
||||
* free space from ptmalloc arenas back to the kernel. On other platforms, or if
|
||||
* a different allocator is in use, this function is a no-op and the report will
|
||||
* indicate that trimming is unsupported or had no effect.
|
||||
*
|
||||
* @param tag Optional identifier for logging/debugging purposes.
|
||||
* @param journal Journal for diagnostic logging.
|
||||
* @return Report containing before/after metrics and the trim result.
|
||||
*
|
||||
* @note If an alternative allocator (jemalloc/tcmalloc) is linked or preloaded,
|
||||
* calling glibc's malloc_trim may have no effect on the active heap. The
|
||||
* call is harmless but typically does not reclaim memory under those
|
||||
* allocators.
|
||||
*
|
||||
* @note Only memory served from glibc's sbrk/arena heaps is eligible for trim.
|
||||
* Large allocations satisfied via mmap are usually returned on free
|
||||
* independently of trimming.
|
||||
*
|
||||
* @note Intended for use after operations that free significant memory (e.g.,
|
||||
* cache sweeps, ledger cleanup, online delete). Consider rate limiting.
|
||||
*/
|
||||
MallocTrimReport
|
||||
mallocTrim(std::optional<std::string> const& tag, beast::Journal journal);
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -1,7 +1,5 @@
|
||||
#pragma once
|
||||
|
||||
#include <xrpl/basics/ByteUtilities.h>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
template <class F>
|
||||
@@ -10,17 +8,15 @@ JobQueue::Coro::Coro(Coro_create_t, JobQueue& jq, JobType type, std::string cons
|
||||
, type_(type)
|
||||
, name_(name)
|
||||
, running_(false)
|
||||
, coro_(
|
||||
[this, fn = std::forward<F>(f)](
|
||||
boost::coroutines::asymmetric_coroutine<void>::push_type& do_yield) {
|
||||
yield_ = &do_yield;
|
||||
yield();
|
||||
fn(shared_from_this());
|
||||
, coro_([this, fn = std::forward<F>(f)](
|
||||
boost::coroutines2::asymmetric_coroutine<void>::push_type& do_yield) {
|
||||
yield_ = &do_yield;
|
||||
yield();
|
||||
fn(shared_from_this());
|
||||
#ifndef NDEBUG
|
||||
finished_ = true;
|
||||
finished_ = true;
|
||||
#endif
|
||||
},
|
||||
boost::coroutines::attributes(megabytes(1)))
|
||||
})
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
#include <xrpl/core/detail/Workers.h>
|
||||
#include <xrpl/json/json_value.h>
|
||||
|
||||
#include <boost/coroutine/all.hpp>
|
||||
#include <boost/coroutine2/all.hpp>
|
||||
|
||||
#include <set>
|
||||
|
||||
@@ -48,8 +48,8 @@ public:
|
||||
std::mutex mutex_;
|
||||
std::mutex mutex_run_;
|
||||
std::condition_variable cv_;
|
||||
boost::coroutines::asymmetric_coroutine<void>::pull_type coro_;
|
||||
boost::coroutines::asymmetric_coroutine<void>::push_type* yield_;
|
||||
boost::coroutines2::coroutine<void>::pull_type coro_;
|
||||
boost::coroutines2::coroutine<void>::push_type* yield_;
|
||||
#ifndef NDEBUG
|
||||
bool finished_ = false;
|
||||
#endif
|
||||
|
||||
@@ -1,15 +1,7 @@
|
||||
# The idea is to empty this file gradually by fixing the underlying issues and removing suppressions.
|
||||
#
|
||||
# ASAN_OPTIONS="print_stacktrace=1:detect_container_overflow=0:suppressions=sanitizers/suppressions/asan.supp:halt_on_error=0"
|
||||
#
|
||||
# The detect_container_overflow=0 option disables false positives from:
|
||||
# - Boost intrusive containers (slist_iterator.hpp, hashtable.hpp, aged_unordered_container.h)
|
||||
# - Boost context/coroutine stack switching (Workers.cpp, thread.h)
|
||||
#
|
||||
# See: https://github.com/google/sanitizers/wiki/AddressSanitizerContainerOverflow
|
||||
|
||||
# Boost
|
||||
interceptor_name:boost/asio
|
||||
|
||||
# Leaks in Doctest tests: xrpl.test.*
|
||||
interceptor_name:src/libxrpl/net/HTTPClient.cpp
|
||||
@@ -20,6 +12,23 @@ interceptor_name:xrpl/net/HTTPClient.h
|
||||
interceptor_name:xrpl/net/HTTPClientSSLContext.h
|
||||
interceptor_name:xrpl/net/RegisterSSLCerts.h
|
||||
|
||||
# Boost.Context fiber/coroutine false positives
|
||||
# ASan doesn't fully support makecontext/swapcontext (see first warning in output)
|
||||
# The "attempting free on address which was not malloc()-ed" errors are false positives
|
||||
# caused by Boost.Context's fiber stack management.
|
||||
#
|
||||
# Suppress bad-free errors in Boost.Context fiber/coroutine stack deallocation
|
||||
# These are triggered when fiber stacks are deallocated but ASan doesn't recognize them
|
||||
# as having been allocated via malloc (because they use mmap via boost's stack allocator)
|
||||
interceptor_via_fun:swapcontext
|
||||
interceptor_via_fun:makecontext
|
||||
interceptor_via_fun:boost::context::basic_fixedsize_stack*deallocate
|
||||
interceptor_via_fun:boost::context::fiber::~fiber
|
||||
interceptor_name:boost/context/fiber_ucontext.hpp
|
||||
interceptor_name:boost/context/fixedsize_stack.hpp
|
||||
interceptor_name:Coro.ipp
|
||||
|
||||
|
||||
# Suppress false positive stack-buffer errors in thread stack allocation
|
||||
# Related to ASan's __asan_handle_no_return warnings (github.com/google/sanitizers/issues/189)
|
||||
# These occur during multi-threaded test initialization on macOS
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
#include <xrpl/basics/Log.h>
|
||||
#include <xrpl/basics/MallocTrim.h>
|
||||
|
||||
#include <boost/predef.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
#include <sys/resource.h>
|
||||
|
||||
#include <malloc.h>
|
||||
#include <unistd.h>
|
||||
|
||||
// Require RUSAGE_THREAD for thread-scoped page fault tracking
|
||||
#ifndef RUSAGE_THREAD
|
||||
#error "MallocTrim rusage instrumentation requires RUSAGE_THREAD on Linux/glibc"
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
bool
|
||||
getRusageThread(struct rusage& ru)
|
||||
{
|
||||
return ::getrusage(RUSAGE_THREAD, &ru) == 0; // LCOV_EXCL_LINE
|
||||
}
|
||||
|
||||
} // namespace
|
||||
#endif
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
namespace detail {
|
||||
|
||||
// cSpell:ignore statm
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
|
||||
inline int
|
||||
mallocTrimWithPad(std::size_t padBytes)
|
||||
{
|
||||
return ::malloc_trim(padBytes);
|
||||
}
|
||||
|
||||
long
|
||||
parseStatmRSSkB(std::string const& statm)
|
||||
{
|
||||
// /proc/self/statm format: size resident shared text lib data dt
|
||||
// We want the second field (resident) which is in pages
|
||||
std::istringstream iss(statm);
|
||||
long size, resident;
|
||||
if (!(iss >> size >> resident))
|
||||
return -1;
|
||||
|
||||
// Convert pages to KB
|
||||
long const pageSize = ::sysconf(_SC_PAGESIZE);
|
||||
if (pageSize <= 0)
|
||||
return -1;
|
||||
|
||||
return (resident * pageSize) / 1024;
|
||||
}
|
||||
|
||||
#endif // __GLIBC__ && BOOST_OS_LINUX
|
||||
|
||||
} // namespace detail
|
||||
|
||||
MallocTrimReport
|
||||
mallocTrim([[maybe_unused]] std::optional<std::string> const& tag, beast::Journal journal)
|
||||
{
|
||||
// LCOV_EXCL_START
|
||||
|
||||
MallocTrimReport report;
|
||||
|
||||
#if !(defined(__GLIBC__) && BOOST_OS_LINUX)
|
||||
JLOG(journal.debug()) << "malloc_trim not supported on this platform";
|
||||
#else
|
||||
// Keep glibc malloc_trim padding at 0 (default): 12h Mainnet tests across 0/256KB/1MB/16MB
|
||||
// showed no clear, consistent benefit from custom padding—0 provided the best overall balance
|
||||
// of RSS reduction and trim-latency stability without adding a tuning surface.
|
||||
constexpr std::size_t TRIM_PAD = 0;
|
||||
|
||||
report.supported = true;
|
||||
|
||||
if (journal.debug())
|
||||
{
|
||||
auto readFile = [](std::string const& path) -> std::string {
|
||||
std::ifstream ifs(path, std::ios::in | std::ios::binary);
|
||||
if (!ifs.is_open())
|
||||
return {};
|
||||
|
||||
// /proc files are often not seekable; read as a stream.
|
||||
std::ostringstream oss;
|
||||
oss << ifs.rdbuf();
|
||||
return oss.str();
|
||||
};
|
||||
|
||||
std::string const tagStr = tag.value_or("default");
|
||||
std::string const statmPath = "/proc/self/statm";
|
||||
|
||||
auto const statmBefore = readFile(statmPath);
|
||||
long const rssBeforeKB = detail::parseStatmRSSkB(statmBefore);
|
||||
|
||||
struct rusage ru0{};
|
||||
bool const have_ru0 = getRusageThread(ru0);
|
||||
|
||||
auto const t0 = std::chrono::steady_clock::now();
|
||||
|
||||
report.trimResult = detail::mallocTrimWithPad(TRIM_PAD);
|
||||
|
||||
auto const t1 = std::chrono::steady_clock::now();
|
||||
|
||||
struct rusage ru1{};
|
||||
bool const have_ru1 = getRusageThread(ru1);
|
||||
|
||||
auto const statmAfter = readFile(statmPath);
|
||||
long const rssAfterKB = detail::parseStatmRSSkB(statmAfter);
|
||||
|
||||
// Populate report fields
|
||||
report.rssBeforeKB = rssBeforeKB;
|
||||
report.rssAfterKB = rssAfterKB;
|
||||
report.durationUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0);
|
||||
|
||||
if (have_ru0 && have_ru1)
|
||||
{
|
||||
report.minfltDelta = ru1.ru_minflt - ru0.ru_minflt;
|
||||
report.majfltDelta = ru1.ru_majflt - ru0.ru_majflt;
|
||||
}
|
||||
|
||||
long const deltaKB = (rssBeforeKB < 0 || rssAfterKB < 0) ? 0 : (rssAfterKB - rssBeforeKB);
|
||||
|
||||
JLOG(journal.debug()) << "malloc_trim tag=" << tagStr << " result=" << report.trimResult << " pad=" << TRIM_PAD
|
||||
<< " bytes"
|
||||
<< " rss_before=" << rssBeforeKB << "kB"
|
||||
<< " rss_after=" << rssAfterKB << "kB"
|
||||
<< " delta=" << deltaKB << "kB"
|
||||
<< " duration_us=" << report.durationUs.count() << " minflt_delta=" << report.minfltDelta
|
||||
<< " majflt_delta=" << report.majfltDelta;
|
||||
}
|
||||
else
|
||||
{
|
||||
report.trimResult = detail::mallocTrimWithPad(TRIM_PAD);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
return report;
|
||||
|
||||
// LCOV_EXCL_STOP
|
||||
}
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -1,198 +0,0 @@
|
||||
#include <xrpl/basics/MallocTrim.h>
|
||||
|
||||
#include <boost/predef.h>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
using namespace xrpl;
|
||||
|
||||
// cSpell:ignore statm
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
namespace xrpl::detail {
|
||||
long
|
||||
parseStatmRSSkB(std::string const& statm);
|
||||
} // namespace xrpl::detail
|
||||
#endif
|
||||
|
||||
TEST(MallocTrimReport, structure)
|
||||
{
|
||||
// Test default construction
|
||||
MallocTrimReport report;
|
||||
EXPECT_EQ(report.supported, false);
|
||||
EXPECT_EQ(report.trimResult, -1);
|
||||
EXPECT_EQ(report.rssBeforeKB, -1);
|
||||
EXPECT_EQ(report.rssAfterKB, -1);
|
||||
EXPECT_EQ(report.durationUs, std::chrono::microseconds{-1});
|
||||
EXPECT_EQ(report.minfltDelta, -1);
|
||||
EXPECT_EQ(report.majfltDelta, -1);
|
||||
EXPECT_EQ(report.deltaKB(), 0);
|
||||
|
||||
// Test deltaKB calculation - memory freed
|
||||
report.rssBeforeKB = 1000;
|
||||
report.rssAfterKB = 800;
|
||||
EXPECT_EQ(report.deltaKB(), -200);
|
||||
|
||||
// Test deltaKB calculation - memory increased
|
||||
report.rssBeforeKB = 500;
|
||||
report.rssAfterKB = 600;
|
||||
EXPECT_EQ(report.deltaKB(), 100);
|
||||
|
||||
// Test deltaKB calculation - no change
|
||||
report.rssBeforeKB = 1234;
|
||||
report.rssAfterKB = 1234;
|
||||
EXPECT_EQ(report.deltaKB(), 0);
|
||||
}
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
TEST(parseStatmRSSkB, standard_format)
|
||||
{
|
||||
using xrpl::detail::parseStatmRSSkB;
|
||||
|
||||
// Test standard format: size resident shared text lib data dt
|
||||
// Assuming 4KB page size: resident=1000 pages = 4000 KB
|
||||
{
|
||||
std::string statm = "25365 1000 2377 0 0 5623 0";
|
||||
long result = parseStatmRSSkB(statm);
|
||||
// Note: actual result depends on system page size
|
||||
// On most systems it's 4KB, so 1000 pages = 4000 KB
|
||||
EXPECT_GT(result, 0);
|
||||
}
|
||||
|
||||
// Test with newline
|
||||
{
|
||||
std::string statm = "12345 2000 1234 0 0 3456 0\n";
|
||||
long result = parseStatmRSSkB(statm);
|
||||
EXPECT_GT(result, 0);
|
||||
}
|
||||
|
||||
// Test with tabs
|
||||
{
|
||||
std::string statm = "12345\t2000\t1234\t0\t0\t3456\t0";
|
||||
long result = parseStatmRSSkB(statm);
|
||||
EXPECT_GT(result, 0);
|
||||
}
|
||||
|
||||
// Test zero resident pages
|
||||
{
|
||||
std::string statm = "25365 0 2377 0 0 5623 0";
|
||||
long result = parseStatmRSSkB(statm);
|
||||
EXPECT_EQ(result, 0);
|
||||
}
|
||||
|
||||
// Test with extra whitespace
|
||||
{
|
||||
std::string statm = " 25365 1000 2377 ";
|
||||
long result = parseStatmRSSkB(statm);
|
||||
EXPECT_GT(result, 0);
|
||||
}
|
||||
|
||||
// Test empty string
|
||||
{
|
||||
std::string statm = "";
|
||||
long result = parseStatmRSSkB(statm);
|
||||
EXPECT_EQ(result, -1);
|
||||
}
|
||||
|
||||
// Test malformed data (only one field)
|
||||
{
|
||||
std::string statm = "25365";
|
||||
long result = parseStatmRSSkB(statm);
|
||||
EXPECT_EQ(result, -1);
|
||||
}
|
||||
|
||||
// Test malformed data (non-numeric)
|
||||
{
|
||||
std::string statm = "abc def ghi";
|
||||
long result = parseStatmRSSkB(statm);
|
||||
EXPECT_EQ(result, -1);
|
||||
}
|
||||
|
||||
// Test malformed data (second field non-numeric)
|
||||
{
|
||||
std::string statm = "25365 abc 2377";
|
||||
long result = parseStatmRSSkB(statm);
|
||||
EXPECT_EQ(result, -1);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST(mallocTrim, basic_functionality)
|
||||
{
|
||||
beast::Journal journal{beast::Journal::getNullSink()};
|
||||
|
||||
// Test with no tag
|
||||
{
|
||||
MallocTrimReport report = mallocTrim(std::nullopt, journal);
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
// On Linux with glibc, should be supported
|
||||
EXPECT_EQ(report.supported, true);
|
||||
// trimResult should be 0 or 1 (success indicators)
|
||||
EXPECT_GE(report.trimResult, 0);
|
||||
EXPECT_EQ(report.durationUs, std::chrono::microseconds{-1});
|
||||
EXPECT_EQ(report.minfltDelta, -1);
|
||||
EXPECT_EQ(report.majfltDelta, -1);
|
||||
#else
|
||||
// On other platforms, should be unsupported
|
||||
EXPECT_EQ(report.supported, false);
|
||||
EXPECT_EQ(report.trimResult, -1);
|
||||
EXPECT_EQ(report.rssBeforeKB, -1);
|
||||
EXPECT_EQ(report.rssAfterKB, -1);
|
||||
EXPECT_EQ(report.durationUs, std::chrono::microseconds{-1});
|
||||
EXPECT_EQ(report.minfltDelta, -1);
|
||||
EXPECT_EQ(report.majfltDelta, -1);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Test with tag
|
||||
{
|
||||
MallocTrimReport report = mallocTrim(std::optional<std::string>("test_tag"), journal);
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
EXPECT_EQ(report.supported, true);
|
||||
EXPECT_GE(report.trimResult, 0);
|
||||
#else
|
||||
EXPECT_EQ(report.supported, false);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
TEST(mallocTrim, debug_logging)
|
||||
{
|
||||
beast::severities::Severity const thresh = beast::severities::kDebug;
|
||||
beast::Journal journal{beast::Journal::getNullSink()};
|
||||
journal.threshold(thresh);
|
||||
|
||||
MallocTrimReport report = mallocTrim(std::optional<std::string>("debug_test"), journal);
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
EXPECT_EQ(report.supported, true);
|
||||
EXPECT_GE(report.durationUs.count(), 0);
|
||||
EXPECT_GE(report.minfltDelta, 0);
|
||||
EXPECT_GE(report.majfltDelta, 0);
|
||||
#else
|
||||
EXPECT_EQ(report.supported, false);
|
||||
EXPECT_EQ(report.durationUs, std::chrono::microseconds{-1});
|
||||
EXPECT_EQ(report.minfltDelta, -1);
|
||||
EXPECT_EQ(report.majfltDelta, -1);
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST(mallocTrim, repeated_calls)
|
||||
{
|
||||
beast::Journal journal{beast::Journal::getNullSink()};
|
||||
|
||||
// Call malloc_trim multiple times to ensure it's safe
|
||||
for (int i = 0; i < 5; ++i)
|
||||
{
|
||||
MallocTrimReport report = mallocTrim(std::optional<std::string>("iteration_" + std::to_string(i)), journal);
|
||||
|
||||
#if defined(__GLIBC__) && BOOST_OS_LINUX
|
||||
EXPECT_EQ(report.supported, true);
|
||||
EXPECT_GE(report.trimResult, 0);
|
||||
#else
|
||||
EXPECT_EQ(report.supported, false);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
@@ -31,7 +31,6 @@
|
||||
#include <xrpld/shamap/NodeFamily.h>
|
||||
|
||||
#include <xrpl/basics/ByteUtilities.h>
|
||||
#include <xrpl/basics/MallocTrim.h>
|
||||
#include <xrpl/basics/ResolverAsio.h>
|
||||
#include <xrpl/basics/random.h>
|
||||
#include <xrpl/beast/asio/io_latency_probe.h>
|
||||
@@ -1054,8 +1053,6 @@ public:
|
||||
<< "; size after: " << cachedSLEs_.size();
|
||||
}
|
||||
|
||||
mallocTrim(std::optional<std::string>("doSweep"), m_journal);
|
||||
|
||||
// Set timer to do another sweep later.
|
||||
setSweepTimer();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user