Implement sanitizer support via CMake (#822)

Fixes #302
This commit is contained in:
Alex Kremer
2023-08-15 15:20:50 +01:00
committed by GitHub
parent ec70127050
commit bf3b24867c
28 changed files with 340 additions and 318 deletions

View File

@@ -1,15 +1,42 @@
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14)
message(FATAL_ERROR "Clang 14+ required for building clio")
endif()
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14)
message (FATAL_ERROR "Clang 14+ required for building clio")
endif ()
set (is_clang TRUE)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14)
message(FATAL_ERROR "AppleClang 14+ required for building clio")
endif()
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14)
message (FATAL_ERROR "AppleClang 14+ required for building clio")
endif ()
set (is_appleclang TRUE)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11)
message(FATAL_ERROR "GCC 11+ required for building clio")
endif()
else()
message(FATAL_ERROR "Supported compilers: AppleClang 14+, Clang 14+, GCC 11+")
endif()
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11)
message (FATAL_ERROR "GCC 11+ required for building clio")
endif ()
set (is_gcc TRUE)
else ()
message (FATAL_ERROR "Supported compilers: AppleClang 14+, Clang 14+, GCC 11+")
endif ()
if (san)
string (TOLOWER ${san} san)
set (SAN_FLAG "-fsanitize=${san}")
set (SAN_LIB "")
if (is_gcc)
if (san STREQUAL "address")
set (SAN_LIB "asan")
elseif (san STREQUAL "thread")
set (SAN_LIB "tsan")
elseif (san STREQUAL "memory")
set (SAN_LIB "msan")
elseif (san STREQUAL "undefined")
set (SAN_LIB "ubsan")
endif ()
endif ()
set (_saved_CRL ${CMAKE_REQUIRED_LIBRARIES})
set (CMAKE_REQUIRED_LIBRARIES "${SAN_FLAG};${SAN_LIB}")
CHECK_CXX_COMPILER_FLAG (${SAN_FLAG} COMPILER_SUPPORTS_SAN)
set (CMAKE_REQUIRED_LIBRARIES ${_saved_CRL})
if (NOT COMPILER_SUPPORTS_SAN)
message (FATAL_ERROR "${san} sanitizer does not seem to be supported by your compiler")
endif ()
endif ()

View File

@@ -2,32 +2,32 @@
write version to source
#]===================================================================]
find_package(Git REQUIRED)
find_package (Git REQUIRED)
set(GIT_COMMAND rev-parse --short HEAD)
execute_process(COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE REV OUTPUT_STRIP_TRAILING_WHITESPACE)
set (GIT_COMMAND rev-parse --short HEAD)
execute_process (COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE REV OUTPUT_STRIP_TRAILING_WHITESPACE)
set(GIT_COMMAND branch --show-current)
execute_process(COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE BRANCH OUTPUT_STRIP_TRAILING_WHITESPACE)
set (GIT_COMMAND branch --show-current)
execute_process (COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE BRANCH OUTPUT_STRIP_TRAILING_WHITESPACE)
if(BRANCH STREQUAL "")
set(BRANCH "dev")
endif()
if (BRANCH STREQUAL "")
set (BRANCH "dev")
endif ()
if(NOT (BRANCH MATCHES master OR BRANCH MATCHES release/*)) # for develop and any other branch name YYYYMMDDHMS-<branch>-<git-ref>
execute_process(COMMAND date +%Y%m%d%H%M%S OUTPUT_VARIABLE DATE OUTPUT_STRIP_TRAILING_WHITESPACE)
set(VERSION "${DATE}-${BRANCH}-${REV}")
else()
set(GIT_COMMAND describe --tags)
execute_process(COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE TAG_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
set(VERSION "${TAG_VERSION}-${REV}")
endif()
if (NOT (BRANCH MATCHES master OR BRANCH MATCHES release/*)) # for develop and any other branch name YYYYMMDDHMS-<branch>-<git-ref>
execute_process (COMMAND date +%Y%m%d%H%M%S OUTPUT_VARIABLE DATE OUTPUT_STRIP_TRAILING_WHITESPACE)
set (VERSION "${DATE}-${BRANCH}-${REV}")
else ()
set (GIT_COMMAND describe --tags)
execute_process (COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE TAG_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
set (VERSION "${TAG_VERSION}-${REV}")
endif ()
if(CMAKE_BUILD_TYPE MATCHES Debug)
set(VERSION "${VERSION}+DEBUG")
endif()
if (CMAKE_BUILD_TYPE MATCHES Debug)
set (VERSION "${VERSION}+DEBUG")
endif ()
message(STATUS "Build version: ${VERSION}")
set(clio_version "${VERSION}")
message (STATUS "Build version: ${VERSION}")
set (clio_version "${VERSION}")
configure_file(CMake/Build.cpp.in ${CMAKE_SOURCE_DIR}/src/main/impl/Build.cpp)
configure_file (CMake/Build.cpp.in ${CMAKE_SOURCE_DIR}/src/main/impl/Build.cpp)

View File

@@ -1,48 +1,45 @@
# call add_coverage(module_name) to add coverage targets for the given module
function(add_coverage module)
if("${CMAKE_C_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang"
function (add_coverage module)
if ("${CMAKE_C_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang"
OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang")
message("[Coverage] Building with llvm Code Coverage Tools")
message ("[Coverage] Building with llvm Code Coverage Tools")
# Using llvm gcov ; llvm install by xcode
set(LLVM_COV_PATH /Library/Developer/CommandLineTools/usr/bin)
if(NOT EXISTS ${LLVM_COV_PATH}/llvm-cov)
message(FATAL_ERROR "llvm-cov not found! Aborting.")
endif()
set (LLVM_COV_PATH /Library/Developer/CommandLineTools/usr/bin)
if (NOT EXISTS ${LLVM_COV_PATH}/llvm-cov)
message (FATAL_ERROR "llvm-cov not found! Aborting.")
endif ()
# set Flags
target_compile_options(${module} PRIVATE
target_compile_options (${module} PRIVATE
-fprofile-instr-generate
-fcoverage-mapping)
target_link_options(${module} PUBLIC
target_link_options (${module} PUBLIC
-fprofile-instr-generate
-fcoverage-mapping)
target_compile_options(clio PRIVATE
target_compile_options (clio PRIVATE
-fprofile-instr-generate
-fcoverage-mapping)
target_link_options(clio PUBLIC
target_link_options (clio PUBLIC
-fprofile-instr-generate
-fcoverage-mapping)
# llvm-cov
add_custom_target(
${module}-ccov-preprocessing
add_custom_target (${module}-ccov-preprocessing
COMMAND LLVM_PROFILE_FILE=${module}.profraw $<TARGET_FILE:${module}>
COMMAND ${LLVM_COV_PATH}/llvm-profdata merge -sparse ${module}.profraw -o
${module}.profdata
DEPENDS ${module})
add_custom_target(
${module}-ccov-show
add_custom_target (${module}-ccov-show
COMMAND ${LLVM_COV_PATH}/llvm-cov show $<TARGET_FILE:${module}>
-instr-profile=${module}.profdata -show-line-counts-or-regions
DEPENDS ${module}-ccov-preprocessing)
# add summary for CI parse
add_custom_target(
${module}-ccov-report
add_custom_target (${module}-ccov-report
COMMAND
${LLVM_COV_PATH}/llvm-cov report $<TARGET_FILE:${module}>
-instr-profile=${module}.profdata
@@ -51,8 +48,7 @@ function(add_coverage module)
DEPENDS ${module}-ccov-preprocessing)
# exclude libs and unittests self
add_custom_target(
${module}-ccov
add_custom_target (${module}-ccov
COMMAND
${LLVM_COV_PATH}/llvm-cov show $<TARGET_FILE:${module}>
-instr-profile=${module}.profdata -show-line-counts-or-regions
@@ -60,37 +56,36 @@ function(add_coverage module)
-ignore-filename-regex=".*_makefiles|.*unittests|.*_deps" > /dev/null 2>&1
DEPENDS ${module}-ccov-preprocessing)
add_custom_command(
add_custom_command (
TARGET ${module}-ccov
POST_BUILD
COMMENT
"Open ${module}-llvm-cov/index.html in your browser to view the coverage report."
)
elseif("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
message("[Coverage] Building with Gcc Code Coverage Tools")
elseif ("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
message ("[Coverage] Building with Gcc Code Coverage Tools")
find_program(GCOV_PATH gcov)
if(NOT GCOV_PATH)
message(FATAL_ERROR "gcov not found! Aborting...")
endif() # NOT GCOV_PATH
find_program(GCOVR_PATH gcovr)
if(NOT GCOVR_PATH)
message(FATAL_ERROR "gcovr not found! Aborting...")
endif() # NOT GCOVR_PATH
find_program (GCOV_PATH gcov)
if (NOT GCOV_PATH)
message (FATAL_ERROR "gcov not found! Aborting...")
endif () # NOT GCOV_PATH
find_program (GCOVR_PATH gcovr)
if (NOT GCOVR_PATH)
message (FATAL_ERROR "gcovr not found! Aborting...")
endif () # NOT GCOVR_PATH
set(COV_OUTPUT_PATH ${module}-gcc-cov)
target_compile_options(${module} PRIVATE -fprofile-arcs -ftest-coverage
set (COV_OUTPUT_PATH ${module}-gcc-cov)
target_compile_options (${module} PRIVATE -fprofile-arcs -ftest-coverage
-fPIC)
target_link_libraries(${module} PRIVATE gcov)
target_link_libraries (${module} PRIVATE gcov)
target_compile_options(clio PRIVATE -fprofile-arcs -ftest-coverage
target_compile_options (clio PRIVATE -fprofile-arcs -ftest-coverage
-fPIC)
target_link_libraries(clio PRIVATE gcov)
target_link_libraries (clio PRIVATE gcov)
# this target is used for CI as well generate the summary out.xml will send
# to github action to generate markdown, we can paste it to comments or
# readme
add_custom_target(
${module}-ccov
add_custom_target (${module}-ccov
COMMAND ${module} ${TEST_PARAMETER}
COMMAND rm -rf ${COV_OUTPUT_PATH}
COMMAND mkdir ${COV_OUTPUT_PATH}
@@ -107,8 +102,7 @@ function(add_coverage module)
COMMENT "Running gcovr to produce Cobertura code coverage report.")
# generate the detail report
add_custom_target(
${module}-ccov-report
add_custom_target (${module}-ccov-report
COMMAND ${module} ${TEST_PARAMETER}
COMMAND rm -rf ${COV_OUTPUT_PATH}
COMMAND mkdir ${COV_OUTPUT_PATH}
@@ -119,13 +113,13 @@ function(add_coverage module)
--exclude='${PROJECT_BINARY_DIR}/'
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
COMMENT "Running gcovr to produce Cobertura code coverage report.")
add_custom_command(
add_custom_command (
TARGET ${module}-ccov-report
POST_BUILD
COMMENT
"Open ${COV_OUTPUT_PATH}/index.html in your browser to view the coverage report."
)
else()
message(FATAL_ERROR "Complier not support yet")
endif()
endfunction()
else ()
message (FATAL_ERROR "Complier not support yet")
endif ()
endfunction ()

View File

@@ -1,10 +1,10 @@
find_package(Doxygen REQUIRED)
find_package (Doxygen REQUIRED)
set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile)
set(DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
set (DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile)
set (DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)
add_custom_target(docs
configure_file (${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)
add_custom_target (docs
COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Generating API documentation with Doxygen"

View File

@@ -1,4 +1,4 @@
target_compile_options(clio PUBLIC
target_compile_options (clio PUBLIC
-Wall
-Werror
-Wno-narrowing

View File

@@ -1,11 +1,11 @@
include(CheckIncludeFileCXX)
include (CheckIncludeFileCXX)
check_include_file_cxx("source_location" SOURCE_LOCATION_AVAILABLE)
if(SOURCE_LOCATION_AVAILABLE)
target_compile_definitions(clio PUBLIC "HAS_SOURCE_LOCATION")
endif()
check_include_file_cxx ("source_location" SOURCE_LOCATION_AVAILABLE)
if (SOURCE_LOCATION_AVAILABLE)
target_compile_definitions (clio PUBLIC "HAS_SOURCE_LOCATION")
endif ()
check_include_file_cxx("experimental/source_location" EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE)
if(EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE)
target_compile_definitions(clio PUBLIC "HAS_EXPERIMENTAL_SOURCE_LOCATION")
endif()
check_include_file_cxx ("experimental/source_location" EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE)
if (EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE)
target_compile_definitions (clio PUBLIC "HAS_EXPERIMENTAL_SOURCE_LOCATION")
endif ()

View File

@@ -1,7 +1,7 @@
set(Boost_USE_STATIC_LIBS ON)
set(Boost_USE_STATIC_RUNTIME ON)
set (Boost_USE_STATIC_LIBS ON)
set (Boost_USE_STATIC_RUNTIME ON)
find_package(Boost 1.82 REQUIRED
find_package (Boost 1.82 REQUIRED
COMPONENTS
program_options
coroutine

View File

@@ -1,5 +1,5 @@
find_package(OpenSSL 1.1.1 REQUIRED)
find_package (OpenSSL 1.1.1 REQUIRED)
set_target_properties(OpenSSL::SSL PROPERTIES
set_target_properties (OpenSSL::SSL PROPERTIES
INTERFACE_COMPILE_DEFINITIONS OPENSSL_NO_SSL2
)

View File

@@ -1,2 +1,2 @@
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads)
set (THREADS_PREFER_PTHREAD_FLAG ON)
find_package (Threads)

View File

@@ -1 +1 @@
find_package(cassandra-cpp-driver REQUIRED)
find_package (cassandra-cpp-driver REQUIRED)

View File

@@ -1,4 +1,4 @@
find_package(GTest REQUIRED)
find_package (GTest REQUIRED)
enable_testing()
include(GoogleTest)
enable_testing ()
include (GoogleTest)

View File

@@ -1 +1 @@
find_package(fmt REQUIRED)
find_package (fmt REQUIRED)

View File

@@ -1 +1 @@
find_package(xrpl REQUIRED)
find_package (xrpl REQUIRED)

View File

@@ -1,16 +1,14 @@
set(CLIO_INSTALL_DIR "/opt/clio")
set(CMAKE_INSTALL_PREFIX ${CLIO_INSTALL_DIR})
set (CLIO_INSTALL_DIR "/opt/clio")
set (CMAKE_INSTALL_PREFIX ${CLIO_INSTALL_DIR})
install(TARGETS clio_server DESTINATION bin)
# install(TARGETS clio_tests DESTINATION bin) # NOTE: Do we want to install the tests?
install (TARGETS clio_server DESTINATION bin)
#install(FILES example-config.json DESTINATION etc RENAME config.json)
file(READ example-config.json config)
string(REGEX REPLACE "./clio_log" "/var/log/clio/" config "${config}")
file(WRITE ${CMAKE_BINARY_DIR}/install-config.json "${config}")
install(FILES ${CMAKE_BINARY_DIR}/install-config.json DESTINATION etc RENAME config.json)
file (READ example-config.json config)
string (REGEX REPLACE "./clio_log" "/var/log/clio/" config "${config}")
file (WRITE ${CMAKE_BINARY_DIR}/install-config.json "${config}")
install (FILES ${CMAKE_BINARY_DIR}/install-config.json DESTINATION etc RENAME config.json)
configure_file("${CMAKE_SOURCE_DIR}/CMake/install/clio.service.in" "${CMAKE_BINARY_DIR}/clio.service")
configure_file ("${CMAKE_SOURCE_DIR}/CMake/install/clio.service.in" "${CMAKE_BINARY_DIR}/clio.service")
install(FILES "${CMAKE_BINARY_DIR}/clio.service" DESTINATION /lib/systemd/system)
install (FILES "${CMAKE_BINARY_DIR}/clio.service" DESTINATION /lib/systemd/system)

View File

@@ -1,45 +1,51 @@
cmake_minimum_required(VERSION 3.16.3)
project(clio)
# ==================================================== #
# Options #
# ==================================================== #
option(verbose "Verbose build" FALSE)
option(tests "Build tests" FALSE)
option(docs "Generate doxygen docs" FALSE)
option(coverage "Build test coverage report" FALSE)
option(packaging "Create distribution packages" FALSE)
# ==================================================== #
# ========================================================================== #
# Options #
# ========================================================================== #
option (verbose "Verbose build" FALSE)
option (tests "Build tests" FALSE)
option (docs "Generate doxygen docs" FALSE)
option (coverage "Build test coverage report" FALSE)
option (packaging "Create distribution packages" FALSE)
# ========================================================================== #
set (san "" CACHE STRING "Add sanitizer instrumentation")
set_property (CACHE san PROPERTY STRINGS ";undefined;memory;address;thread")
# ========================================================================== #
if(verbose)
set(CMAKE_VERBOSE_MAKEFILE TRUE)
endif()
# Include required modules
include (CheckCXXCompilerFlag)
if(packaging)
add_definitions(-DPKG=1)
endif()
if (verbose)
set (CMAKE_VERBOSE_MAKEFILE TRUE)
endif ()
add_library(clio)
if (packaging)
add_definitions (-DPKG=1)
endif ()
add_library (clio)
# Clio tweaks and checks
include(CMake/CheckCompiler.cmake)
include(CMake/Settings.cmake)
include(CMake/ClioVersion.cmake)
include(CMake/SourceLocation.cmake)
include (CMake/CheckCompiler.cmake)
include (CMake/Settings.cmake)
include (CMake/ClioVersion.cmake)
include (CMake/SourceLocation.cmake)
# Clio deps
include(CMake/deps/libxrpl.cmake)
include(CMake/deps/Boost.cmake)
include(CMake/deps/OpenSSL.cmake)
include(CMake/deps/Threads.cmake)
include(CMake/deps/libfmt.cmake)
include(CMake/deps/cassandra.cmake)
include (CMake/deps/libxrpl.cmake)
include (CMake/deps/Boost.cmake)
include (CMake/deps/OpenSSL.cmake)
include (CMake/deps/Threads.cmake)
include (CMake/deps/libfmt.cmake)
include (CMake/deps/cassandra.cmake)
# TODO: Include directory will be wrong when installed.
target_include_directories(clio PUBLIC src)
target_compile_features(clio PUBLIC cxx_std_20)
target_include_directories (clio PUBLIC src)
target_compile_features (clio PUBLIC cxx_std_20)
target_link_libraries(clio
target_link_libraries (clio
PUBLIC Boost::boost
PUBLIC Boost::coroutine
PUBLIC Boost::program_options
@@ -55,12 +61,12 @@ target_link_libraries(clio
INTERFACE Threads::Threads
)
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
if (is_gcc)
# FIXME: needed on gcc for now
target_compile_definitions(clio PUBLIC BOOST_ASIO_DISABLE_CONCEPTS)
endif()
target_compile_definitions (clio PUBLIC BOOST_ASIO_DISABLE_CONCEPTS)
endif ()
target_sources(clio PRIVATE
target_sources (clio PRIVATE
## Main
src/main/impl/Build.cpp
## Backend
@@ -83,6 +89,8 @@ target_sources(clio PRIVATE
src/etl/impl/ForwardCache.cpp
## Feed
src/feed/SubscriptionManager.cpp
## Web
src/web/IntervalSweepHandler.cpp
## RPC
src/rpc/Errors.cpp
src/rpc/Factories.cpp
@@ -126,13 +134,13 @@ target_sources(clio PRIVATE
src/util/Taggable.cpp)
# Clio server
add_executable(clio_server src/main/Main.cpp)
target_link_libraries(clio_server PUBLIC clio)
add_executable (clio_server src/main/Main.cpp)
target_link_libraries (clio_server PUBLIC clio)
# Unittesting
if(tests)
set(TEST_TARGET clio_tests)
add_executable(${TEST_TARGET}
if (tests)
set (TEST_TARGET clio_tests)
add_executable (${TEST_TARGET}
# Common
unittests/Main.cpp
unittests/Playground.cpp
@@ -206,38 +214,50 @@ if(tests)
unittests/web/WhitelistHandlerTests.cpp
unittests/web/SweepHandlerTests.cpp)
include(CMake/deps/gtest.cmake)
include (CMake/deps/gtest.cmake)
# See https://github.com/google/googletest/issues/3475
gtest_discover_tests(clio_tests DISCOVERY_TIMEOUT 10)
gtest_discover_tests (clio_tests DISCOVERY_TIMEOUT 10)
# Fix for dwarf5 bug on ci
target_compile_options(clio PUBLIC -gdwarf-4)
target_compile_options (clio PUBLIC -gdwarf-4)
# TODO: support sanitizers properly
# Tmp: uncomment for TSAN
# target_compile_options(${TEST_TARGET} PRIVATE -fsanitize=thread)
# target_link_options(${TEST_TARGET} PRIVATE -fsanitize=thread)
target_compile_definitions(${TEST_TARGET} PUBLIC UNITTEST_BUILD)
target_include_directories(${TEST_TARGET} PRIVATE unittests)
target_link_libraries(${TEST_TARGET} PUBLIC clio gtest::gtest)
target_compile_definitions (${TEST_TARGET} PUBLIC UNITTEST_BUILD)
target_include_directories (${TEST_TARGET} PRIVATE unittests)
target_link_libraries (${TEST_TARGET} PUBLIC clio gtest::gtest)
# Generate `clio_tests-ccov` if coverage is enabled
# Note: use `make clio_tests-ccov` to generate report
if(coverage)
include(CMake/Coverage.cmake)
add_coverage(${TEST_TARGET})
endif()
endif()
if (coverage)
include (CMake/Coverage.cmake)
add_coverage (${TEST_TARGET})
endif ()
endif ()
# Enable selected sanitizer if enabled via `san`
if (san)
target_compile_options (clio
PUBLIC
# Sanitizers recommend minimum of -O1 for reasonable performance
$<$<CONFIG:Debug>:-O1>
${SAN_FLAG}
-fno-omit-frame-pointer)
target_compile_definitions (clio
PUBLIC
$<$<STREQUAL:${san},address>:SANITIZER=ASAN>
$<$<STREQUAL:${san},thread>:SANITIZER=TSAN>
$<$<STREQUAL:${san},memory>:SANITIZER=MSAN>
$<$<STREQUAL:${san},undefined>:SANITIZER=UBSAN>)
target_link_libraries (clio INTERFACE ${SAN_FLAG} ${SAN_LIB})
endif ()
# Generate `docs` target for doxygen documentation if enabled
# Note: use `make docs` to generate the documentation
if(docs)
include(CMake/Docs.cmake)
endif()
if (docs)
include (CMake/Docs.cmake)
endif ()
include(CMake/install/install.cmake)
if(packaging)
include(CMake/packaging.cmake) # This file exists only in build runner
endif()
include (CMake/install/install.cmake)
if (packaging)
include (CMake/packaging.cmake) # This file exists only in build runner
endif ()

View File

@@ -31,12 +31,11 @@ namespace data {
/**
* @brief A factory function that creates the backend based on a config.
*
* @param ioc The boost::asio::io_context to use
* @param config The clio config to use
* @return A shared_ptr<BackendInterface> with the selected implementation
*/
std::shared_ptr<BackendInterface>
make_Backend(boost::asio::io_context& ioc, util::Config const& config)
make_Backend(util::Config const& config)
{
static util::Logger log{"Backend"};
LOG(log.info()) << "Constructing BackendInterface";

View File

@@ -568,7 +568,7 @@ public:
std::vector<Statement> statements;
statements.reserve(numHashes);
auto const timeDiff = util::timed([this, &yield, &results, &hashes, &statements]() {
auto const timeDiff = util::timed([this, yield, &results, &hashes, &statements]() {
// TODO: seems like a job for "hash IN (list of hashes)" instead?
std::transform(
std::cbegin(hashes), std::cend(hashes), std::back_inserter(statements), [this](auto const& hash) {
@@ -634,7 +634,7 @@ public:
std::vector<LedgerObject>
fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const override
{
auto const [keys, timeDiff] = util::timed([this, &ledgerSequence, &yield]() -> std::vector<ripple::uint256> {
auto const [keys, timeDiff] = util::timed([this, &ledgerSequence, yield]() -> std::vector<ripple::uint256> {
auto const res = executor_.read(yield, schema_->selectDiff, ledgerSequence);
if (not res)
{

View File

@@ -236,7 +236,7 @@ void
ETLService::doWork()
{
worker_ = std::thread([this]() {
beast::setCurrentThreadName("rippled: ETLService worker");
beast::setCurrentThreadName("ETLService worker");
if (state_.isReadOnly)
monitorReadOnly();

View File

@@ -59,8 +59,7 @@ PlainSource::close(bool startAgain)
derived().ws().async_close(boost::beast::websocket::close_code::normal, [this, startAgain](auto ec) {
if (ec)
{
LOG(log_.error()) << " async_close : "
<< "error code = " << ec << " - " << toString();
LOG(log_.error()) << "async_close: error code = " << ec << " - " << toString();
}
closing_ = false;
if (startAgain)
@@ -94,8 +93,7 @@ SslSource::close(bool startAgain)
derived().ws().async_close(boost::beast::websocket::close_code::normal, [this, startAgain](auto ec) {
if (ec)
{
LOG(log_.error()) << " async_close : "
<< "error code = " << ec << " - " << toString();
LOG(log_.error()) << "async_close: error code = " << ec << " - " << toString();
}
closing_ = false;
if (startAgain)

View File

@@ -68,7 +68,6 @@ class CacheLoader
std::vector<ClioPeer> clioPeers_;
std::thread thread_;
std::atomic_bool stopping_ = false;
public:
@@ -116,8 +115,6 @@ public:
~CacheLoader()
{
stop();
if (thread_.joinable())
thread_.join();
}
/**
@@ -357,28 +354,28 @@ private:
diff.erase(std::unique(diff.begin(), diff.end(), [](auto a, auto b) { return a.key == b.key; }), diff.end());
cursors.push_back({});
for (auto& obj : diff)
for (auto const& obj : diff)
if (obj.blob.size())
cursors.push_back({obj.key});
cursors.push_back({});
std::stringstream cursorStr;
for (auto& c : cursors)
for (auto const& c : cursors)
if (c)
cursorStr << ripple::strHex(*c) << ", ";
LOG(log_.info()) << "Loading cache. num cursors = " << cursors.size() - 1;
LOG(log_.trace()) << "cursors = " << cursorStr.str();
thread_ = std::thread{[this, seq, cursors]() {
boost::asio::post(ioContext_.get(), [this, seq, cursors = std::move(cursors)]() {
auto startTime = std::chrono::system_clock::now();
auto markers = std::make_shared<std::atomic_int>(0);
auto numRemaining = std::make_shared<std::atomic_int>(cursors.size() - 1);
for (size_t i = 0; i < cursors.size() - 1; ++i)
{
auto const start = cursors[i];
auto const end = cursors[i + 1];
auto const start = cursors.at(i);
auto const end = cursors.at(i + 1);
markers->wait(numCacheMarkers_);
++(*markers);
@@ -386,14 +383,14 @@ private:
boost::asio::spawn(
ioContext_.get(),
[this, seq, start, end, numRemaining, startTime, markers](boost::asio::yield_context yield) {
std::optional<ripple::uint256> cursor = start;
auto cursor = start;
std::string cursorStr =
cursor.has_value() ? ripple::strHex(cursor.value()) : ripple::strHex(data::firstKey);
LOG(log_.debug()) << "Starting a cursor: " << cursorStr << " markers = " << *markers;
while (not stopping_)
{
auto res = data::retryOnTimeout([this, seq, &cursor, &yield]() {
auto res = data::retryOnTimeout([this, seq, &cursor, yield]() {
return backend_->fetchLedgerPage(cursor, seq, cachePageFetchSize_, false, yield);
});
@@ -428,7 +425,7 @@ private:
}
});
}
}};
});
}
};

View File

@@ -184,7 +184,7 @@ try
auto dosGuard = web::DOSGuard{config, whitelistHandler, sweepHandler};
// Interface to the database
auto backend = data::make_Backend(ioc, config);
auto backend = data::make_Backend(config);
// Manages clients subscribed to streams
auto subscriptions = feed::SubscriptionManager::make_SubscriptionManager(config, backend);
@@ -208,7 +208,7 @@ try
auto const rpcEngine = rpc::RPCEngine::make_RPCEngine(
config, backend, subscriptions, balancer, etl, dosGuard, workQueue, counters, handlerProvider);
// init the web server
// Init the web server
auto handler = std::make_shared<web::RPCServerHandler<rpc::RPCEngine, etl::ETLService>>(
config, backend, rpcEngine, etl, subscriptions);
auto ctx = parseCerts(config);

View File

@@ -84,7 +84,7 @@ NFTOffersHandlerBase::iterateOfferDirectory(
cursor = uint256(input.marker->c_str());
// We have a start point. Use limit - 1 from the result and use the very last one for the resume.
auto const sle = [this, &cursor, &lgrInfo, &yield]() -> std::shared_ptr<SLE const> {
auto const sle = [this, &cursor, &lgrInfo, yield]() -> std::shared_ptr<SLE const> {
auto const key = keylet::nftoffer(cursor).key;
if (auto const blob = sharedPtrBackend_->fetchLedgerObject(key, lgrInfo.seq, yield); blob)

View File

@@ -20,6 +20,8 @@
#pragma once
#include <util/config/Config.h>
#include <util/log/Logger.h>
#include <web/IntervalSweepHandler.h>
#include <web/WhitelistHandler.h>
#include <boost/asio.hpp>
@@ -253,66 +255,6 @@ private:
}
};
/**
* @brief Sweep handler using a steady_timer and boost::asio::io_context.
*/
class IntervalSweepHandler
{
std::chrono::milliseconds sweepInterval_;
std::reference_wrapper<boost::asio::io_context> ctx_;
boost::asio::steady_timer timer_;
BaseDOSGuard* dosGuard_ = nullptr;
public:
/**
* @brief Construct a new interval-based sweep handler.
*
* @param config Clio config
* @param ctx The boost::asio::io_context
*/
IntervalSweepHandler(util::Config const& config, boost::asio::io_context& ctx)
: sweepInterval_{std::max(1u, static_cast<uint32_t>(config.valueOr("dos_guard.sweep_interval", 1.0) * 1000.0))}
, ctx_{std::ref(ctx)}
, timer_{ctx.get_executor()}
{
}
~IntervalSweepHandler()
{
timer_.cancel();
}
/**
* @brief This setup member function is called by @ref BasicDOSGuard during its initialization.
*
* @param guard Pointer to the dos guard
*/
void
setup(BaseDOSGuard* guard)
{
assert(dosGuard_ == nullptr);
dosGuard_ = guard;
assert(dosGuard_ != nullptr);
createTimer();
}
private:
void
createTimer()
{
timer_.expires_after(sweepInterval_);
timer_.async_wait([this](boost::system::error_code const& error) {
if (error == boost::asio::error::operation_aborted)
return;
dosGuard_->clear();
boost::asio::post(ctx_.get().get_executor(), [this] { createTimer(); });
});
}
};
using DOSGuard = BasicDOSGuard<web::WhitelistHandler, IntervalSweepHandler>;
using DOSGuard = BasicDOSGuard<web::WhitelistHandler, web::IntervalSweepHandler>;
} // namespace web

View File

@@ -0,0 +1,64 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <web/IntervalSweepHandler.h>
#include <web/DOSGuard.h>
#include <algorithm>
#include <ctime>
namespace web {
IntervalSweepHandler::IntervalSweepHandler(util::Config const& config, boost::asio::io_context& ctx)
: sweepInterval_{std::max(1u, static_cast<uint32_t>(config.valueOr("dos_guard.sweep_interval", 1.0) * 1000.0))}
, ctx_{std::ref(ctx)}
, timer_{ctx.get_executor()}
{
}
IntervalSweepHandler::~IntervalSweepHandler()
{
boost::asio::post(ctx_.get(), [this]() { timer_.cancel(); });
}
void
IntervalSweepHandler::setup(web::BaseDOSGuard* guard)
{
assert(dosGuard_ == nullptr);
dosGuard_ = guard;
assert(dosGuard_ != nullptr);
createTimer();
}
void
IntervalSweepHandler::createTimer()
{
timer_.expires_after(sweepInterval_);
timer_.async_wait([this](boost::system::error_code const& error) {
if (error == boost::asio::error::operation_aborted)
return;
dosGuard_->clear();
boost::asio::post(ctx_.get(), [this] { createTimer(); });
});
}
} // namespace web

View File

@@ -19,15 +19,15 @@
#pragma once
#include <util/config/Config.h>
#include <boost/asio.hpp>
#include <boost/iterator/transform_iterator.hpp>
#include <boost/system/error_code.hpp>
#include <algorithm>
#include <chrono>
#include <ctime>
namespace web::detail {
namespace web {
class BaseDOSGuard;
/**
* @brief Sweep handler using a steady_timer and boost::asio::io_context.
@@ -42,52 +42,29 @@ class IntervalSweepHandler
public:
/**
* @brief Construct a new interval-based sweep handler
* @brief Construct a new interval-based sweep handler.
*
* @param config Clio config
* @param ctx The boost::asio::io_context
* @param config Clio config to use
* @param ctx The boost::asio::io_context to use
*/
IntervalSweepHandler(util::Config const& config, boost::asio::io_context& ctx)
: sweepInterval_{std::max(1u, static_cast<uint32_t>(config.valueOr("dos_guard.sweep_interval", 1.0) * 1000.0))}
, ctx_{std::ref(ctx)}
, timer_{ctx.get_executor()}
{
}
~IntervalSweepHandler()
{
timer_.cancel();
}
IntervalSweepHandler(util::Config const& config, boost::asio::io_context& ctx);
/**
* @brief This setup member function is called by @ref BasicDOSGuard during
* its initialization.
* @brief Cancels the sweep timer.
*/
~IntervalSweepHandler();
/**
* @brief This setup member function is called by @ref BasicDOSGuard during its initialization.
*
* @param guard Pointer to the dos guard
*/
void
setup(web::BaseDOSGuard* guard)
{
assert(dosGuard_ == nullptr);
dosGuard_ = guard;
assert(dosGuard_ != nullptr);
createTimer();
}
setup(web::BaseDOSGuard* guard);
private:
void
createTimer()
{
timer_.expires_after(sweepInterval_);
timer_.async_wait([this](boost::system::error_code const& error) {
if (error == boost::asio::error::operation_aborted)
return;
dosGuard_->clear();
boost::asio::post(ctx_.get().get_executor(), [this] { createTimer(); });
});
}
createTimer();
};
} // namespace web::detail
} // namespace web

View File

@@ -20,6 +20,7 @@
#pragma once
#include <boost/asio.hpp>
#include <boost/iterator/transform_iterator.hpp>
#include <fmt/core.h>
#include <regex>
@@ -30,8 +31,7 @@
namespace web {
/**
* @brief A whitelist to remove rate limits of certain IP addresses
*
* @brief A whitelist to remove rate limits of certain IP addresses.
*/
class Whitelist
{
@@ -41,7 +41,7 @@ class Whitelist
public:
/**
* @brief Add network address to whitelist
* @brief Add network address to whitelist.
*
* @param net Network part of the ip address
* @throws std::runtime::error when the network address is not valid
@@ -66,7 +66,7 @@ public:
}
/**
* @brief Checks to see if ip address is whitelisted
* @brief Checks to see if ip address is whitelisted.
*
* @param ip IP address
* @throws std::runtime::error when the network address is not valid
@@ -130,15 +130,18 @@ private:
};
/**
* @brief A simple handler to add/check elements in a whitelist
*
* @param arr map of net addresses to add to whitelist
* @brief A simple handler to add/check elements in a whitelist.
*/
class WhitelistHandler
{
Whitelist whitelist_;
public:
/**
* @brief Adds all whitelisted IPs and masks from the given config.
*
* @param config The Clio config to use
*/
WhitelistHandler(util::Config const& config)
{
std::unordered_set<std::string> arr = getWhitelist(config);
@@ -146,6 +149,9 @@ public:
whitelist_.add(net);
}
/**
* @return true if the given IP is whitelisted; false otherwise
*/
bool
isWhiteListed(std::string_view ip) const
{

View File

@@ -74,7 +74,7 @@ TEST_F(BackendCassandraFactoryTest, NoSuchBackend)
"type":"unknown"
}
})")};
EXPECT_THROW(make_Backend(ctx, cfg), std::runtime_error);
EXPECT_THROW(make_Backend(cfg), std::runtime_error);
}
TEST_F(BackendCassandraFactoryTest, CreateCassandraBackendDBDisconnect)
@@ -94,7 +94,7 @@ TEST_F(BackendCassandraFactoryTest, CreateCassandraBackendDBDisconnect)
}})",
"127.0.0.2",
keyspace))};
EXPECT_THROW(make_Backend(ctx, cfg), std::runtime_error);
EXPECT_THROW(make_Backend(cfg), std::runtime_error);
}
TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend)
@@ -115,7 +115,7 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend)
keyspace))};
{
auto backend = make_Backend(ctx, cfg);
auto backend = make_Backend(cfg);
EXPECT_TRUE(backend);
// empty db does not have ledger range
@@ -129,7 +129,7 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend)
}
{
auto backend = make_Backend(ctx, cfg);
auto backend = make_Backend(cfg);
EXPECT_TRUE(backend);
auto const range = backend->fetchLedgerRange();
@@ -155,7 +155,7 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackendReadOnlyWithEmpt
}})",
contactPoints,
keyspace))};
EXPECT_THROW(make_Backend(ctx, cfg), std::runtime_error);
EXPECT_THROW(make_Backend(cfg), std::runtime_error);
}
TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackendReadOnlyWithDBReady)
@@ -192,6 +192,6 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackendReadOnlyWithDBRe
contactPoints,
keyspace))};
EXPECT_TRUE(make_Backend(ctx, cfgWrite));
EXPECT_TRUE(make_Backend(ctx, cfgReadOnly));
EXPECT_TRUE(make_Backend(cfgWrite));
EXPECT_TRUE(make_Backend(cfgReadOnly));
}