diff --git a/.github/scripts/strategy-matrix/generate.py b/.github/scripts/strategy-matrix/generate.py index 230c18c2df..0915a9a3d2 100755 --- a/.github/scripts/strategy-matrix/generate.py +++ b/.github/scripts/strategy-matrix/generate.py @@ -156,15 +156,107 @@ def generate_strategy_matrix(all: bool, config: Config) -> list: # Add the configuration to the list, with the most unique fields first, # so that they are easier to identify in the GitHub Actions UI, as long # names get truncated. - configurations.append({ - 'config_name': config_name, - 'cmake_args': cmake_args, - 'cmake_target': cmake_target, - 'build_only': build_only, - 'build_type': build_type, - 'os': os, - 'architecture': architecture, - }) + # Add Address and Thread (both coupled with UB) sanitizers when the distro is bookworm. + if os['distro_version'] == 'bookworm' and f'{os["compiler_name"]}-{os["compiler_version"]}' in {'gcc-15', 'clang-20'}: + extra_warning_flags = '' + linker_relocation_flags = '' + linker_flags = '' + cxx_flags += ' -DBOOST_USE_TSAN -DBOOST_USE_UBSAN -DBOOST_USE_UCONTEXT' + + # Use large code model to avoid relocation errors with large binaries + # Only for x86-64 (amd64) - ARM64 doesn't support -mcmodel=large + if architecture['platform'] == 'linux/amd64' and os['compiler_name'] == 'gcc': + # Add -mcmodel=large and -fPIC to both compiler AND linker flags + # This is needed because sanitizers create very large binaries + # -fPIC enables position independent code to avoid relocation range issues + # large model removes the 2GB limitation that medium model has + cxx_flags += ' -mcmodel=large -fno-PIC' + linker_relocation_flags+=' -mcmodel=large -fno-PIC' + + # Create default sanitizer flags + sanitizers_flags = 'undefined,float-divide-by-zero' + + if os['compiler_name'] == 'gcc': + sanitizers_flags = f'{sanitizers_flags},signed-integer-overflow' + # Suppress false positive warnings in GCC with stringop-overflow + extra_warning_flags += ' -Wno-stringop-overflow' + # Disable mold, gold and lld linkers. + # Use default linker (bfd/ld) which is more lenient with mixed code models + cmake_args += ' -Duse_mold=OFF -Duse_gold=OFF -Duse_lld=OFF' + # Add linker flags for Sanitizers + linker_flags += f' -DCMAKE_EXE_LINKER_FLAGS="{linker_relocation_flags} -fsanitize=address,{sanitizers_flags}"' + linker_flags += f' -DCMAKE_SHARED_LINKER_FLAGS="{linker_relocation_flags} -fsanitize=address,{sanitizers_flags}"' + elif os['compiler_name'] == 'clang': + sanitizers_flags = f'{sanitizers_flags},signed-integer-overflow,unsigned-integer-overflow' + linker_flags += f' -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,{sanitizers_flags}"' + linker_flags += f' -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=address,{sanitizers_flags}"' + + # Sanitizers recommend minimum of -O1 for reasonable performance + if "-O0" in cxx_flags: + cxx_flags = cxx_flags.replace("-O0", "-O1") + else: + cxx_flags += " -O1" + + # First create config for asan + cmake_args_flags = f'{cmake_args} -DCMAKE_CXX_FLAGS="-fsanitize=address,{sanitizers_flags} -fno-omit-frame-pointer {cxx_flags} {extra_warning_flags}" {linker_flags}' + + # Add config with asan + configurations.append({ + 'config_name': config_name + "_asan", + 'cmake_args': cmake_args_flags, + 'cmake_target': cmake_target, + 'build_only': build_only, + 'build_type': build_type, + 'os': os, + 'architecture': architecture, + 'sanitizers': 'Address' + }) + + linker_flags = '' + # Update configs for tsan + # gcc doesn't supports atomic_thread_fence with tsan. Suppress warnings. + # Also tsan doesn't work well with mcmode=large and bfd linker + if os['compiler_name'] == 'gcc': + extra_warning_flags += ' -Wno-tsan' + cxx_flags = cxx_flags.replace('-mcmodel=large', '-mcmodel=medium') + linker_relocation_flags = linker_relocation_flags.replace('-mcmodel=large', '-mcmodel=medium') + # Add linker flags for Sanitizers + linker_flags += f' -DCMAKE_EXE_LINKER_FLAGS="{linker_relocation_flags} -fsanitize=thread,{sanitizers_flags}"' + linker_flags += f' -DCMAKE_SHARED_LINKER_FLAGS="{linker_relocation_flags} -fsanitize=thread,{sanitizers_flags}"' + elif os['compiler_name'] == 'clang': + cxx_flags += ' -fsanitize-blacklist=$GITHUB_WORKSPACE/external/sanitizer-blacklist.txt' + linker_flags += f' -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=thread,{sanitizers_flags}"' + linker_flags += f' -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=thread,{sanitizers_flags}"' + + # Note: We use $GITHUB_WORKSPACE environment variable which will be expanded by the shell + # before CMake processes it. This ensures the compiler receives an absolute path. + # CMAKE_SOURCE_DIR won't work here because it's inside CMAKE_CXX_FLAGS string. + cmake_args_flags = f'{cmake_args} -DCMAKE_CXX_FLAGS="-fsanitize=thread,{sanitizers_flags} -fno-omit-frame-pointer {cxx_flags} {extra_warning_flags}" {linker_flags}' + + configurations.append({ + 'config_name': config_name+ "_tsan", + 'cmake_args': cmake_args_flags, + 'cmake_target': cmake_target, + 'build_only': build_only, + 'build_type': build_type, + 'os': os, + 'architecture': architecture, + 'sanitizers': 'Thread' + }) + else: + if cxx_flags: + cmake_args_flags = f'{cmake_args} -DCMAKE_CXX_FLAGS={cxx_flags}' + else: + cmake_args_flags = f'{cmake_args}' + configurations.append({ + 'config_name': config_name, + 'cmake_args': cmake_args_flags, + 'cmake_target': cmake_target, + 'build_only': build_only, + 'build_type': build_type, + 'os': os, + 'architecture': architecture + }) return configurations diff --git a/cmake/XrplConfig.cmake b/cmake/XrplConfig.cmake index 8a739d48a3..ed64050d53 100644 --- a/cmake/XrplConfig.cmake +++ b/cmake/XrplConfig.cmake @@ -17,7 +17,7 @@ find_dependency (Boost chrono container context - coroutine + coroutine2 date_time filesystem program_options diff --git a/cmake/deps/Boost.cmake b/cmake/deps/Boost.cmake index 475c1033b2..b7e9443375 100644 --- a/cmake/deps/Boost.cmake +++ b/cmake/deps/Boost.cmake @@ -2,7 +2,7 @@ find_package(Boost 1.82 REQUIRED COMPONENTS chrono container - coroutine + context date_time filesystem json @@ -20,7 +20,7 @@ target_link_libraries(xrpl_boost Boost::headers Boost::chrono Boost::container - Boost::coroutine + Boost::context Boost::date_time Boost::filesystem Boost::json diff --git a/conan/profiles/ci b/conan/profiles/ci new file mode 100644 index 0000000000..c4c0898ad5 --- /dev/null +++ b/conan/profiles/ci @@ -0,0 +1 @@ + include(sanitizers) diff --git a/conan/profiles/sanitizers b/conan/profiles/sanitizers new file mode 100644 index 0000000000..8939911cc1 --- /dev/null +++ b/conan/profiles/sanitizers @@ -0,0 +1,69 @@ +include(default) +{% set compiler, version, compiler_exe = detect_api.detect_default_compiler() %} +{% set default_sanitizer_flags = "undefined,float-divide-by-zero,signed-integer-overflow" %} +{% set sanitizers = os.getenv("SANITIZERS") %} + +[settings] + +[conf] + +{% if sanitizers == "Address" or sanitizers == "Thread" %} +user.package:sanitizers={{ sanitizers }} +tools.info.package_id:confs+=["user.package:sanitizers"] +{% endif %} + +{% if compiler == "gcc" %} + +{% set asan_sanitizer_flags = "-fsanitize=address,"~default_sanitizer_flags~" -mcmodel=large -fno-PIC" %} +{% set tsan_sanitizer_flags = "-fsanitize=thread,"~default_sanitizer_flags~" -mcmodel=medium -fno-PIC" %} + +{% if sanitizers == "Address" %} +tools.build:cxxflags+=['{{asan_sanitizer_flags}} -fno-omit-frame-pointer -O1 -Wno-stringop-overflow -DBOOST_USE_ASAN -DBOOST_USE_UBSAN -DBOOST_USE_UCONTEXT'] +tools.build:sharedlinkflags+=['{{asan_sanitizer_flags}}'] +tools.build:exelinkflags+=['{{asan_sanitizer_flags}}'] +tools.cmake.cmaketoolchain:extra_variables={"use_mold": "OFF", "use_gold": "OFF", "use_lld": "OFF"} +tools.build:defines+=["BOOST_USE_ASAN", "BOOST_USE_UBSAN", "BOOST_USE_UCONTEXT"] + +{% elif sanitizers == "Thread" %} +tools.build:cxxflags+=['{{tsan_sanitizer_flags}} -fno-omit-frame-pointer -O1 -Wno-stringop-overflow -Wno-tsan -DBOOST_USE_TSAN -DBOOST_USE_UBSAN -DBOOST_USE_UCONTEXT'] +tools.build:sharedlinkflags+=['{{tsan_sanitizer_flags}}'] +tools.build:exelinkflags+=['{{tsan_sanitizer_flags}}'] +tools.build:defines+=["BOOST_USE_TSAN", "BOOST_USE_UBSAN", "BOOST_USE_UCONTEXT"] + +{% endif %} + +{% elif compiler == "apple-clang" or compiler == "clang" %} + +{% set asan_sanitizer_flags = "-fsanitize=address,"~default_sanitizer_flags~",unsigned-integer-overflow" %} +{% set tsan_sanitizer_flags = "-fsanitize=thread,"~default_sanitizer_flags~",unsigned-integer-overflow" %} +{% if sanitizers == "Address" %} +tools.build:cxxflags+=['{{asan_sanitizer_flags}} -fno-omit-frame-pointer -O1 -DBOOST_USE_ASAN -DBOOST_USE_UBSAN -DBOOST_USE_UCONTEXT'] +tools.build:sharedlinkflags+=['{{asan_sanitizer_flags}}'] +tools.build:exelinkflags+=['{{asan_sanitizer_flags}}'] +tools.build:defines+=["BOOST_USE_ASAN", "BOOST_USE_UBSAN", "BOOST_USE_UCONTEXT"] + +{% elif sanitizers == "Thread" %} +tools.build:cxxflags+=['{{tsan_sanitizer_flags}} -fno-omit-frame-pointer -O1 -DBOOST_USE_TSAN -DBOOST_USE_UBSAN -DBOOST_USE_UCONTEXT'] +tools.build:sharedlinkflags+=['{{tsan_sanitizer_flags}}'] +tools.build:exelinkflags+=['{{tsan_sanitizer_flags}}'] +tools.build:defines+=["BOOST_USE_TSAN", "BOOST_USE_UBSAN", "BOOST_USE_UCONTEXT"] + +{% endif %} + +{% endif %} + +[options] +{% if compiler == "gcc" or compiler == "apple-clang" or compiler == "clang" %} +{% if sanitizers == "Address" or sanitizers == "Thread" %} +boost/*:without_context=False +boost/*:without_stacktrace=True +boost/*:without_coroutine2=False + +{% if sanitizers == "Address" %} +boost/*:extra_b2_flags="context-impl=ucontext address-sanitizer=norecover undefined-sanitizer=norecover --with-coroutine2" +{% elif sanitizers == "Thread" %} +boost/*:extra_b2_flags="context-impl=ucontext thread-sanitizer=norecover undefined-sanitizer=norecover --with-coroutine2" +{% endif %} + +{% endif %} +{% endif %} diff --git a/conanfile.py b/conanfile.py index 41ec5d35f3..353b14c8dd 100644 --- a/conanfile.py +++ b/conanfile.py @@ -102,6 +102,7 @@ class Xrpl(ConanFile): self.options['boost'].visibility = 'global' if self.settings.compiler in ['clang', 'gcc']: self.options['boost'].without_cobalt = True + self.options['boost'].without_coroutine2 = False def requirements(self): # Conan 2 requires transitive headers to be specified @@ -172,7 +173,8 @@ class Xrpl(ConanFile): 'boost::headers', 'boost::chrono', 'boost::container', - 'boost::coroutine', + 'boost::context', + 'boost::coroutine2', 'boost::date_time', 'boost::filesystem', 'boost::json', diff --git a/src/xrpld/core/Coro.ipp b/src/xrpld/core/Coro.ipp index 8e14e88592..1972bb1481 100644 --- a/src/xrpld/core/Coro.ipp +++ b/src/xrpld/core/Coro.ipp @@ -16,18 +16,16 @@ JobQueue::Coro::Coro( , type_(type) , name_(name) , running_(false) - , coro_( - [this, fn = std::forward(f)]( - boost::coroutines::asymmetric_coroutine::push_type& - do_yield) { - yield_ = &do_yield; - yield(); - fn(shared_from_this()); + , coro_([this, fn = std::forward(f)]( + boost::coroutines2::asymmetric_coroutine::push_type& + do_yield) { + yield_ = &do_yield; + yield(); + fn(shared_from_this()); #ifndef NDEBUG - finished_ = true; + finished_ = true; #endif - }, - boost::coroutines::attributes(megabytes(1))) + }) { } diff --git a/src/xrpld/core/JobQueue.h b/src/xrpld/core/JobQueue.h index c5d36cd993..2ef323f4cc 100644 --- a/src/xrpld/core/JobQueue.h +++ b/src/xrpld/core/JobQueue.h @@ -9,7 +9,7 @@ #include #include -#include +#include #include @@ -50,8 +50,8 @@ public: std::mutex mutex_; std::mutex mutex_run_; std::condition_variable cv_; - boost::coroutines::asymmetric_coroutine::pull_type coro_; - boost::coroutines::asymmetric_coroutine::push_type* yield_; + boost::coroutines2::asymmetric_coroutine::pull_type coro_; + boost::coroutines2::asymmetric_coroutine::push_type* yield_; #ifndef NDEBUG bool finished_ = false; #endif @@ -334,7 +334,7 @@ private: other requests while the RPC command completes its work asynchronously. postCoro() creates a Coro object. When the Coro ctor is called, and its - coro_ member is initialized (a boost::coroutines::pull_type), execution + coro_ member is initialized (a boost::coroutines2::pull_type), execution automatically passes to the coroutine, which we don't want at this point, since we are still in the handler thread context. It's important to note here that construction of a boost pull_type automatically passes execution to