From bf3b24867cf72d97b2c650e04b13f8a08a0414ff Mon Sep 17 00:00:00 2001 From: Alex Kremer Date: Tue, 15 Aug 2023 15:20:50 +0100 Subject: [PATCH] Implement sanitizer support via CMake (#822) Fixes #302 --- CMake/CheckCompiler.cmake | 51 ++++++-- CMake/ClioVersion.cmake | 44 +++---- CMake/Coverage.cmake | 82 ++++++------- CMake/Docs.cmake | 10 +- CMake/Settings.cmake | 2 +- CMake/SourceLocation.cmake | 18 +-- CMake/deps/Boost.cmake | 6 +- CMake/deps/OpenSSL.cmake | 4 +- CMake/deps/Threads.cmake | 4 +- CMake/deps/cassandra.cmake | 2 +- CMake/deps/gtest.cmake | 6 +- CMake/deps/libfmt.cmake | 2 +- CMake/deps/libxrpl.cmake | 2 +- CMake/install/install.cmake | 20 ++- CMakeLists.txt | 142 ++++++++++++---------- src/data/BackendFactory.h | 3 +- src/data/CassandraBackend.h | 4 +- src/etl/ETLService.cpp | 2 +- src/etl/Source.cpp | 6 +- src/etl/impl/CacheLoader.h | 19 ++- src/main/Main.cpp | 4 +- src/rpc/handlers/NFTOffersCommon.cpp | 2 +- src/web/DOSGuard.h | 64 +--------- src/web/IntervalSweepHandler.cpp | 64 ++++++++++ src/web/{impl => }/IntervalSweepHandler.h | 59 +++------ src/web/WhitelistHandler.h | 20 +-- unittests/data/BackendFactoryTests.cpp | 14 +-- unittests/web/SweepHandlerTests.cpp | 2 +- 28 files changed, 340 insertions(+), 318 deletions(-) create mode 100644 src/web/IntervalSweepHandler.cpp rename src/web/{impl => }/IntervalSweepHandler.h (58%) diff --git a/CMake/CheckCompiler.cmake b/CMake/CheckCompiler.cmake index f737d95f..7b4a2e94 100644 --- a/CMake/CheckCompiler.cmake +++ b/CMake/CheckCompiler.cmake @@ -1,15 +1,42 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14) - message(FATAL_ERROR "Clang 14+ required for building clio") - endif() + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14) + message (FATAL_ERROR "Clang 14+ required for building clio") + endif () + set (is_clang TRUE) elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14) - message(FATAL_ERROR "AppleClang 14+ required for building clio") - endif() + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14) + message (FATAL_ERROR "AppleClang 14+ required for building clio") + endif () + set (is_appleclang TRUE) elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11) - message(FATAL_ERROR "GCC 11+ required for building clio") - endif() -else() - message(FATAL_ERROR "Supported compilers: AppleClang 14+, Clang 14+, GCC 11+") -endif() + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11) + message (FATAL_ERROR "GCC 11+ required for building clio") + endif () + set (is_gcc TRUE) +else () + message (FATAL_ERROR "Supported compilers: AppleClang 14+, Clang 14+, GCC 11+") +endif () + +if (san) + string (TOLOWER ${san} san) + set (SAN_FLAG "-fsanitize=${san}") + set (SAN_LIB "") + if (is_gcc) + if (san STREQUAL "address") + set (SAN_LIB "asan") + elseif (san STREQUAL "thread") + set (SAN_LIB "tsan") + elseif (san STREQUAL "memory") + set (SAN_LIB "msan") + elseif (san STREQUAL "undefined") + set (SAN_LIB "ubsan") + endif () + endif () + set (_saved_CRL ${CMAKE_REQUIRED_LIBRARIES}) + set (CMAKE_REQUIRED_LIBRARIES "${SAN_FLAG};${SAN_LIB}") + CHECK_CXX_COMPILER_FLAG (${SAN_FLAG} COMPILER_SUPPORTS_SAN) + set (CMAKE_REQUIRED_LIBRARIES ${_saved_CRL}) + if (NOT COMPILER_SUPPORTS_SAN) + message (FATAL_ERROR "${san} sanitizer does not seem to be supported by your compiler") + endif () +endif () diff --git a/CMake/ClioVersion.cmake b/CMake/ClioVersion.cmake index 177ed33e..591fcee9 100644 --- a/CMake/ClioVersion.cmake +++ b/CMake/ClioVersion.cmake @@ -2,32 +2,32 @@ write version to source #]===================================================================] -find_package(Git REQUIRED) +find_package (Git REQUIRED) -set(GIT_COMMAND rev-parse --short HEAD) -execute_process(COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE REV OUTPUT_STRIP_TRAILING_WHITESPACE) +set (GIT_COMMAND rev-parse --short HEAD) +execute_process (COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE REV OUTPUT_STRIP_TRAILING_WHITESPACE) -set(GIT_COMMAND branch --show-current) -execute_process(COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE BRANCH OUTPUT_STRIP_TRAILING_WHITESPACE) +set (GIT_COMMAND branch --show-current) +execute_process (COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE BRANCH OUTPUT_STRIP_TRAILING_WHITESPACE) -if(BRANCH STREQUAL "") - set(BRANCH "dev") -endif() +if (BRANCH STREQUAL "") + set (BRANCH "dev") +endif () -if(NOT (BRANCH MATCHES master OR BRANCH MATCHES release/*)) # for develop and any other branch name YYYYMMDDHMS-- - execute_process(COMMAND date +%Y%m%d%H%M%S OUTPUT_VARIABLE DATE OUTPUT_STRIP_TRAILING_WHITESPACE) - set(VERSION "${DATE}-${BRANCH}-${REV}") -else() - set(GIT_COMMAND describe --tags) - execute_process(COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE TAG_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) - set(VERSION "${TAG_VERSION}-${REV}") -endif() +if (NOT (BRANCH MATCHES master OR BRANCH MATCHES release/*)) # for develop and any other branch name YYYYMMDDHMS-- + execute_process (COMMAND date +%Y%m%d%H%M%S OUTPUT_VARIABLE DATE OUTPUT_STRIP_TRAILING_WHITESPACE) + set (VERSION "${DATE}-${BRANCH}-${REV}") +else () + set (GIT_COMMAND describe --tags) + execute_process (COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} OUTPUT_VARIABLE TAG_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) + set (VERSION "${TAG_VERSION}-${REV}") +endif () -if(CMAKE_BUILD_TYPE MATCHES Debug) - set(VERSION "${VERSION}+DEBUG") -endif() +if (CMAKE_BUILD_TYPE MATCHES Debug) + set (VERSION "${VERSION}+DEBUG") +endif () -message(STATUS "Build version: ${VERSION}") -set(clio_version "${VERSION}") +message (STATUS "Build version: ${VERSION}") +set (clio_version "${VERSION}") -configure_file(CMake/Build.cpp.in ${CMAKE_SOURCE_DIR}/src/main/impl/Build.cpp) +configure_file (CMake/Build.cpp.in ${CMAKE_SOURCE_DIR}/src/main/impl/Build.cpp) diff --git a/CMake/Coverage.cmake b/CMake/Coverage.cmake index b2204eff..da6931ff 100644 --- a/CMake/Coverage.cmake +++ b/CMake/Coverage.cmake @@ -1,48 +1,45 @@ # call add_coverage(module_name) to add coverage targets for the given module -function(add_coverage module) - if("${CMAKE_C_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang" +function (add_coverage module) + if ("${CMAKE_C_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang") - message("[Coverage] Building with llvm Code Coverage Tools") + message ("[Coverage] Building with llvm Code Coverage Tools") # Using llvm gcov ; llvm install by xcode - set(LLVM_COV_PATH /Library/Developer/CommandLineTools/usr/bin) - if(NOT EXISTS ${LLVM_COV_PATH}/llvm-cov) - message(FATAL_ERROR "llvm-cov not found! Aborting.") - endif() + set (LLVM_COV_PATH /Library/Developer/CommandLineTools/usr/bin) + if (NOT EXISTS ${LLVM_COV_PATH}/llvm-cov) + message (FATAL_ERROR "llvm-cov not found! Aborting.") + endif () # set Flags - target_compile_options(${module} PRIVATE + target_compile_options (${module} PRIVATE -fprofile-instr-generate -fcoverage-mapping) - target_link_options(${module} PUBLIC + target_link_options (${module} PUBLIC -fprofile-instr-generate -fcoverage-mapping) - target_compile_options(clio PRIVATE + target_compile_options (clio PRIVATE -fprofile-instr-generate -fcoverage-mapping) - target_link_options(clio PUBLIC + target_link_options (clio PUBLIC -fprofile-instr-generate -fcoverage-mapping) # llvm-cov - add_custom_target( - ${module}-ccov-preprocessing + add_custom_target (${module}-ccov-preprocessing COMMAND LLVM_PROFILE_FILE=${module}.profraw $ COMMAND ${LLVM_COV_PATH}/llvm-profdata merge -sparse ${module}.profraw -o ${module}.profdata DEPENDS ${module}) - add_custom_target( - ${module}-ccov-show + add_custom_target (${module}-ccov-show COMMAND ${LLVM_COV_PATH}/llvm-cov show $ -instr-profile=${module}.profdata -show-line-counts-or-regions DEPENDS ${module}-ccov-preprocessing) # add summary for CI parse - add_custom_target( - ${module}-ccov-report + add_custom_target (${module}-ccov-report COMMAND ${LLVM_COV_PATH}/llvm-cov report $ -instr-profile=${module}.profdata @@ -51,8 +48,7 @@ function(add_coverage module) DEPENDS ${module}-ccov-preprocessing) # exclude libs and unittests self - add_custom_target( - ${module}-ccov + add_custom_target (${module}-ccov COMMAND ${LLVM_COV_PATH}/llvm-cov show $ -instr-profile=${module}.profdata -show-line-counts-or-regions @@ -60,37 +56,36 @@ function(add_coverage module) -ignore-filename-regex=".*_makefiles|.*unittests|.*_deps" > /dev/null 2>&1 DEPENDS ${module}-ccov-preprocessing) - add_custom_command( + add_custom_command ( TARGET ${module}-ccov POST_BUILD COMMENT "Open ${module}-llvm-cov/index.html in your browser to view the coverage report." ) - elseif("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") - message("[Coverage] Building with Gcc Code Coverage Tools") + elseif ("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") + message ("[Coverage] Building with Gcc Code Coverage Tools") - find_program(GCOV_PATH gcov) - if(NOT GCOV_PATH) - message(FATAL_ERROR "gcov not found! Aborting...") - endif() # NOT GCOV_PATH - find_program(GCOVR_PATH gcovr) - if(NOT GCOVR_PATH) - message(FATAL_ERROR "gcovr not found! Aborting...") - endif() # NOT GCOVR_PATH + find_program (GCOV_PATH gcov) + if (NOT GCOV_PATH) + message (FATAL_ERROR "gcov not found! Aborting...") + endif () # NOT GCOV_PATH + find_program (GCOVR_PATH gcovr) + if (NOT GCOVR_PATH) + message (FATAL_ERROR "gcovr not found! Aborting...") + endif () # NOT GCOVR_PATH - set(COV_OUTPUT_PATH ${module}-gcc-cov) - target_compile_options(${module} PRIVATE -fprofile-arcs -ftest-coverage + set (COV_OUTPUT_PATH ${module}-gcc-cov) + target_compile_options (${module} PRIVATE -fprofile-arcs -ftest-coverage -fPIC) - target_link_libraries(${module} PRIVATE gcov) + target_link_libraries (${module} PRIVATE gcov) - target_compile_options(clio PRIVATE -fprofile-arcs -ftest-coverage + target_compile_options (clio PRIVATE -fprofile-arcs -ftest-coverage -fPIC) - target_link_libraries(clio PRIVATE gcov) + target_link_libraries (clio PRIVATE gcov) # this target is used for CI as well generate the summary out.xml will send # to github action to generate markdown, we can paste it to comments or # readme - add_custom_target( - ${module}-ccov + add_custom_target (${module}-ccov COMMAND ${module} ${TEST_PARAMETER} COMMAND rm -rf ${COV_OUTPUT_PATH} COMMAND mkdir ${COV_OUTPUT_PATH} @@ -107,8 +102,7 @@ function(add_coverage module) COMMENT "Running gcovr to produce Cobertura code coverage report.") # generate the detail report - add_custom_target( - ${module}-ccov-report + add_custom_target (${module}-ccov-report COMMAND ${module} ${TEST_PARAMETER} COMMAND rm -rf ${COV_OUTPUT_PATH} COMMAND mkdir ${COV_OUTPUT_PATH} @@ -119,13 +113,13 @@ function(add_coverage module) --exclude='${PROJECT_BINARY_DIR}/' WORKING_DIRECTORY ${PROJECT_BINARY_DIR} COMMENT "Running gcovr to produce Cobertura code coverage report.") - add_custom_command( + add_custom_command ( TARGET ${module}-ccov-report POST_BUILD COMMENT "Open ${COV_OUTPUT_PATH}/index.html in your browser to view the coverage report." ) - else() - message(FATAL_ERROR "Complier not support yet") - endif() -endfunction() + else () + message (FATAL_ERROR "Complier not support yet") + endif () +endfunction () diff --git a/CMake/Docs.cmake b/CMake/Docs.cmake index 4f2fbb00..475011e1 100644 --- a/CMake/Docs.cmake +++ b/CMake/Docs.cmake @@ -1,10 +1,10 @@ -find_package(Doxygen REQUIRED) +find_package (Doxygen REQUIRED) -set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile) -set(DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile) +set (DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile) +set (DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile) -configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY) -add_custom_target(docs +configure_file (${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY) +add_custom_target (docs COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating API documentation with Doxygen" diff --git a/CMake/Settings.cmake b/CMake/Settings.cmake index cbf977e0..0489bd5d 100644 --- a/CMake/Settings.cmake +++ b/CMake/Settings.cmake @@ -1,4 +1,4 @@ -target_compile_options(clio PUBLIC +target_compile_options (clio PUBLIC -Wall -Werror -Wno-narrowing diff --git a/CMake/SourceLocation.cmake b/CMake/SourceLocation.cmake index afc9835b..7fd896bb 100644 --- a/CMake/SourceLocation.cmake +++ b/CMake/SourceLocation.cmake @@ -1,11 +1,11 @@ -include(CheckIncludeFileCXX) +include (CheckIncludeFileCXX) -check_include_file_cxx("source_location" SOURCE_LOCATION_AVAILABLE) -if(SOURCE_LOCATION_AVAILABLE) - target_compile_definitions(clio PUBLIC "HAS_SOURCE_LOCATION") -endif() +check_include_file_cxx ("source_location" SOURCE_LOCATION_AVAILABLE) +if (SOURCE_LOCATION_AVAILABLE) + target_compile_definitions (clio PUBLIC "HAS_SOURCE_LOCATION") +endif () -check_include_file_cxx("experimental/source_location" EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE) -if(EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE) - target_compile_definitions(clio PUBLIC "HAS_EXPERIMENTAL_SOURCE_LOCATION") -endif() +check_include_file_cxx ("experimental/source_location" EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE) +if (EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE) + target_compile_definitions (clio PUBLIC "HAS_EXPERIMENTAL_SOURCE_LOCATION") +endif () diff --git a/CMake/deps/Boost.cmake b/CMake/deps/Boost.cmake index b67f12d9..e7bee139 100644 --- a/CMake/deps/Boost.cmake +++ b/CMake/deps/Boost.cmake @@ -1,7 +1,7 @@ -set(Boost_USE_STATIC_LIBS ON) -set(Boost_USE_STATIC_RUNTIME ON) +set (Boost_USE_STATIC_LIBS ON) +set (Boost_USE_STATIC_RUNTIME ON) -find_package(Boost 1.82 REQUIRED +find_package (Boost 1.82 REQUIRED COMPONENTS program_options coroutine diff --git a/CMake/deps/OpenSSL.cmake b/CMake/deps/OpenSSL.cmake index 3241b727..bea8aa35 100644 --- a/CMake/deps/OpenSSL.cmake +++ b/CMake/deps/OpenSSL.cmake @@ -1,5 +1,5 @@ -find_package(OpenSSL 1.1.1 REQUIRED) +find_package (OpenSSL 1.1.1 REQUIRED) -set_target_properties(OpenSSL::SSL PROPERTIES +set_target_properties (OpenSSL::SSL PROPERTIES INTERFACE_COMPILE_DEFINITIONS OPENSSL_NO_SSL2 ) diff --git a/CMake/deps/Threads.cmake b/CMake/deps/Threads.cmake index bc68a9eb..6904fe75 100644 --- a/CMake/deps/Threads.cmake +++ b/CMake/deps/Threads.cmake @@ -1,2 +1,2 @@ -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(Threads) +set (THREADS_PREFER_PTHREAD_FLAG ON) +find_package (Threads) diff --git a/CMake/deps/cassandra.cmake b/CMake/deps/cassandra.cmake index 16914797..c6e6dbb9 100644 --- a/CMake/deps/cassandra.cmake +++ b/CMake/deps/cassandra.cmake @@ -1 +1 @@ -find_package(cassandra-cpp-driver REQUIRED) +find_package (cassandra-cpp-driver REQUIRED) diff --git a/CMake/deps/gtest.cmake b/CMake/deps/gtest.cmake index 600e8833..eb6605fb 100644 --- a/CMake/deps/gtest.cmake +++ b/CMake/deps/gtest.cmake @@ -1,4 +1,4 @@ -find_package(GTest REQUIRED) +find_package (GTest REQUIRED) -enable_testing() -include(GoogleTest) +enable_testing () +include (GoogleTest) diff --git a/CMake/deps/libfmt.cmake b/CMake/deps/libfmt.cmake index c6d3a73d..006f3603 100644 --- a/CMake/deps/libfmt.cmake +++ b/CMake/deps/libfmt.cmake @@ -1 +1 @@ -find_package(fmt REQUIRED) +find_package (fmt REQUIRED) diff --git a/CMake/deps/libxrpl.cmake b/CMake/deps/libxrpl.cmake index 852d763f..efc17b10 100644 --- a/CMake/deps/libxrpl.cmake +++ b/CMake/deps/libxrpl.cmake @@ -1 +1 @@ -find_package(xrpl REQUIRED) +find_package (xrpl REQUIRED) diff --git a/CMake/install/install.cmake b/CMake/install/install.cmake index f4f8a5b1..319fea89 100644 --- a/CMake/install/install.cmake +++ b/CMake/install/install.cmake @@ -1,16 +1,14 @@ -set(CLIO_INSTALL_DIR "/opt/clio") -set(CMAKE_INSTALL_PREFIX ${CLIO_INSTALL_DIR}) +set (CLIO_INSTALL_DIR "/opt/clio") +set (CMAKE_INSTALL_PREFIX ${CLIO_INSTALL_DIR}) -install(TARGETS clio_server DESTINATION bin) -# install(TARGETS clio_tests DESTINATION bin) # NOTE: Do we want to install the tests? +install (TARGETS clio_server DESTINATION bin) -#install(FILES example-config.json DESTINATION etc RENAME config.json) -file(READ example-config.json config) -string(REGEX REPLACE "./clio_log" "/var/log/clio/" config "${config}") -file(WRITE ${CMAKE_BINARY_DIR}/install-config.json "${config}") -install(FILES ${CMAKE_BINARY_DIR}/install-config.json DESTINATION etc RENAME config.json) +file (READ example-config.json config) +string (REGEX REPLACE "./clio_log" "/var/log/clio/" config "${config}") +file (WRITE ${CMAKE_BINARY_DIR}/install-config.json "${config}") +install (FILES ${CMAKE_BINARY_DIR}/install-config.json DESTINATION etc RENAME config.json) -configure_file("${CMAKE_SOURCE_DIR}/CMake/install/clio.service.in" "${CMAKE_BINARY_DIR}/clio.service") +configure_file ("${CMAKE_SOURCE_DIR}/CMake/install/clio.service.in" "${CMAKE_BINARY_DIR}/clio.service") -install(FILES "${CMAKE_BINARY_DIR}/clio.service" DESTINATION /lib/systemd/system) +install (FILES "${CMAKE_BINARY_DIR}/clio.service" DESTINATION /lib/systemd/system) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a375316..941acfde 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,45 +1,51 @@ cmake_minimum_required(VERSION 3.16.3) project(clio) -# ==================================================== # -# Options # -# ==================================================== # -option(verbose "Verbose build" FALSE) -option(tests "Build tests" FALSE) -option(docs "Generate doxygen docs" FALSE) -option(coverage "Build test coverage report" FALSE) -option(packaging "Create distribution packages" FALSE) -# ==================================================== # +# ========================================================================== # +# Options # +# ========================================================================== # +option (verbose "Verbose build" FALSE) +option (tests "Build tests" FALSE) +option (docs "Generate doxygen docs" FALSE) +option (coverage "Build test coverage report" FALSE) +option (packaging "Create distribution packages" FALSE) +# ========================================================================== # +set (san "" CACHE STRING "Add sanitizer instrumentation") +set_property (CACHE san PROPERTY STRINGS ";undefined;memory;address;thread") +# ========================================================================== # -if(verbose) - set(CMAKE_VERBOSE_MAKEFILE TRUE) -endif() +# Include required modules +include (CheckCXXCompilerFlag) -if(packaging) - add_definitions(-DPKG=1) -endif() +if (verbose) + set (CMAKE_VERBOSE_MAKEFILE TRUE) +endif () -add_library(clio) +if (packaging) + add_definitions (-DPKG=1) +endif () + +add_library (clio) # Clio tweaks and checks -include(CMake/CheckCompiler.cmake) -include(CMake/Settings.cmake) -include(CMake/ClioVersion.cmake) -include(CMake/SourceLocation.cmake) +include (CMake/CheckCompiler.cmake) +include (CMake/Settings.cmake) +include (CMake/ClioVersion.cmake) +include (CMake/SourceLocation.cmake) # Clio deps -include(CMake/deps/libxrpl.cmake) -include(CMake/deps/Boost.cmake) -include(CMake/deps/OpenSSL.cmake) -include(CMake/deps/Threads.cmake) -include(CMake/deps/libfmt.cmake) -include(CMake/deps/cassandra.cmake) +include (CMake/deps/libxrpl.cmake) +include (CMake/deps/Boost.cmake) +include (CMake/deps/OpenSSL.cmake) +include (CMake/deps/Threads.cmake) +include (CMake/deps/libfmt.cmake) +include (CMake/deps/cassandra.cmake) # TODO: Include directory will be wrong when installed. -target_include_directories(clio PUBLIC src) -target_compile_features(clio PUBLIC cxx_std_20) +target_include_directories (clio PUBLIC src) +target_compile_features (clio PUBLIC cxx_std_20) -target_link_libraries(clio +target_link_libraries (clio PUBLIC Boost::boost PUBLIC Boost::coroutine PUBLIC Boost::program_options @@ -55,12 +61,12 @@ target_link_libraries(clio INTERFACE Threads::Threads ) -if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") +if (is_gcc) # FIXME: needed on gcc for now - target_compile_definitions(clio PUBLIC BOOST_ASIO_DISABLE_CONCEPTS) -endif() + target_compile_definitions (clio PUBLIC BOOST_ASIO_DISABLE_CONCEPTS) +endif () -target_sources(clio PRIVATE +target_sources (clio PRIVATE ## Main src/main/impl/Build.cpp ## Backend @@ -83,6 +89,8 @@ target_sources(clio PRIVATE src/etl/impl/ForwardCache.cpp ## Feed src/feed/SubscriptionManager.cpp + ## Web + src/web/IntervalSweepHandler.cpp ## RPC src/rpc/Errors.cpp src/rpc/Factories.cpp @@ -126,13 +134,13 @@ target_sources(clio PRIVATE src/util/Taggable.cpp) # Clio server -add_executable(clio_server src/main/Main.cpp) -target_link_libraries(clio_server PUBLIC clio) +add_executable (clio_server src/main/Main.cpp) +target_link_libraries (clio_server PUBLIC clio) # Unittesting -if(tests) - set(TEST_TARGET clio_tests) - add_executable(${TEST_TARGET} +if (tests) + set (TEST_TARGET clio_tests) + add_executable (${TEST_TARGET} # Common unittests/Main.cpp unittests/Playground.cpp @@ -206,38 +214,50 @@ if(tests) unittests/web/WhitelistHandlerTests.cpp unittests/web/SweepHandlerTests.cpp) - include(CMake/deps/gtest.cmake) + include (CMake/deps/gtest.cmake) # See https://github.com/google/googletest/issues/3475 - gtest_discover_tests(clio_tests DISCOVERY_TIMEOUT 10) + gtest_discover_tests (clio_tests DISCOVERY_TIMEOUT 10) # Fix for dwarf5 bug on ci - target_compile_options(clio PUBLIC -gdwarf-4) + target_compile_options (clio PUBLIC -gdwarf-4) - # TODO: support sanitizers properly - # Tmp: uncomment for TSAN - # target_compile_options(${TEST_TARGET} PRIVATE -fsanitize=thread) - # target_link_options(${TEST_TARGET} PRIVATE -fsanitize=thread) - - target_compile_definitions(${TEST_TARGET} PUBLIC UNITTEST_BUILD) - target_include_directories(${TEST_TARGET} PRIVATE unittests) - target_link_libraries(${TEST_TARGET} PUBLIC clio gtest::gtest) + target_compile_definitions (${TEST_TARGET} PUBLIC UNITTEST_BUILD) + target_include_directories (${TEST_TARGET} PRIVATE unittests) + target_link_libraries (${TEST_TARGET} PUBLIC clio gtest::gtest) # Generate `clio_tests-ccov` if coverage is enabled # Note: use `make clio_tests-ccov` to generate report - if(coverage) - include(CMake/Coverage.cmake) - add_coverage(${TEST_TARGET}) - endif() -endif() + if (coverage) + include (CMake/Coverage.cmake) + add_coverage (${TEST_TARGET}) + endif () +endif () + +# Enable selected sanitizer if enabled via `san` +if (san) + target_compile_options (clio + PUBLIC + # Sanitizers recommend minimum of -O1 for reasonable performance + $<$:-O1> + ${SAN_FLAG} + -fno-omit-frame-pointer) + target_compile_definitions (clio + PUBLIC + $<$:SANITIZER=ASAN> + $<$:SANITIZER=TSAN> + $<$:SANITIZER=MSAN> + $<$:SANITIZER=UBSAN>) + target_link_libraries (clio INTERFACE ${SAN_FLAG} ${SAN_LIB}) +endif () # Generate `docs` target for doxygen documentation if enabled # Note: use `make docs` to generate the documentation -if(docs) - include(CMake/Docs.cmake) -endif() +if (docs) + include (CMake/Docs.cmake) +endif () -include(CMake/install/install.cmake) -if(packaging) - include(CMake/packaging.cmake) # This file exists only in build runner -endif() +include (CMake/install/install.cmake) +if (packaging) + include (CMake/packaging.cmake) # This file exists only in build runner +endif () diff --git a/src/data/BackendFactory.h b/src/data/BackendFactory.h index 6301ae77..ac3e192e 100644 --- a/src/data/BackendFactory.h +++ b/src/data/BackendFactory.h @@ -31,12 +31,11 @@ namespace data { /** * @brief A factory function that creates the backend based on a config. * - * @param ioc The boost::asio::io_context to use * @param config The clio config to use * @return A shared_ptr with the selected implementation */ std::shared_ptr -make_Backend(boost::asio::io_context& ioc, util::Config const& config) +make_Backend(util::Config const& config) { static util::Logger log{"Backend"}; LOG(log.info()) << "Constructing BackendInterface"; diff --git a/src/data/CassandraBackend.h b/src/data/CassandraBackend.h index 067e8a4c..c9ab5272 100644 --- a/src/data/CassandraBackend.h +++ b/src/data/CassandraBackend.h @@ -568,7 +568,7 @@ public: std::vector statements; statements.reserve(numHashes); - auto const timeDiff = util::timed([this, &yield, &results, &hashes, &statements]() { + auto const timeDiff = util::timed([this, yield, &results, &hashes, &statements]() { // TODO: seems like a job for "hash IN (list of hashes)" instead? std::transform( std::cbegin(hashes), std::cend(hashes), std::back_inserter(statements), [this](auto const& hash) { @@ -634,7 +634,7 @@ public: std::vector fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const override { - auto const [keys, timeDiff] = util::timed([this, &ledgerSequence, &yield]() -> std::vector { + auto const [keys, timeDiff] = util::timed([this, &ledgerSequence, yield]() -> std::vector { auto const res = executor_.read(yield, schema_->selectDiff, ledgerSequence); if (not res) { diff --git a/src/etl/ETLService.cpp b/src/etl/ETLService.cpp index 94008ffe..289e4c47 100644 --- a/src/etl/ETLService.cpp +++ b/src/etl/ETLService.cpp @@ -236,7 +236,7 @@ void ETLService::doWork() { worker_ = std::thread([this]() { - beast::setCurrentThreadName("rippled: ETLService worker"); + beast::setCurrentThreadName("ETLService worker"); if (state_.isReadOnly) monitorReadOnly(); diff --git a/src/etl/Source.cpp b/src/etl/Source.cpp index 08531dfd..c4acfa5c 100644 --- a/src/etl/Source.cpp +++ b/src/etl/Source.cpp @@ -59,8 +59,7 @@ PlainSource::close(bool startAgain) derived().ws().async_close(boost::beast::websocket::close_code::normal, [this, startAgain](auto ec) { if (ec) { - LOG(log_.error()) << " async_close : " - << "error code = " << ec << " - " << toString(); + LOG(log_.error()) << "async_close: error code = " << ec << " - " << toString(); } closing_ = false; if (startAgain) @@ -94,8 +93,7 @@ SslSource::close(bool startAgain) derived().ws().async_close(boost::beast::websocket::close_code::normal, [this, startAgain](auto ec) { if (ec) { - LOG(log_.error()) << " async_close : " - << "error code = " << ec << " - " << toString(); + LOG(log_.error()) << "async_close: error code = " << ec << " - " << toString(); } closing_ = false; if (startAgain) diff --git a/src/etl/impl/CacheLoader.h b/src/etl/impl/CacheLoader.h index 943b9ca1..ecd320db 100644 --- a/src/etl/impl/CacheLoader.h +++ b/src/etl/impl/CacheLoader.h @@ -68,7 +68,6 @@ class CacheLoader std::vector clioPeers_; - std::thread thread_; std::atomic_bool stopping_ = false; public: @@ -116,8 +115,6 @@ public: ~CacheLoader() { stop(); - if (thread_.joinable()) - thread_.join(); } /** @@ -357,28 +354,28 @@ private: diff.erase(std::unique(diff.begin(), diff.end(), [](auto a, auto b) { return a.key == b.key; }), diff.end()); cursors.push_back({}); - for (auto& obj : diff) + for (auto const& obj : diff) if (obj.blob.size()) cursors.push_back({obj.key}); cursors.push_back({}); std::stringstream cursorStr; - for (auto& c : cursors) + for (auto const& c : cursors) if (c) cursorStr << ripple::strHex(*c) << ", "; LOG(log_.info()) << "Loading cache. num cursors = " << cursors.size() - 1; LOG(log_.trace()) << "cursors = " << cursorStr.str(); - thread_ = std::thread{[this, seq, cursors]() { + boost::asio::post(ioContext_.get(), [this, seq, cursors = std::move(cursors)]() { auto startTime = std::chrono::system_clock::now(); auto markers = std::make_shared(0); auto numRemaining = std::make_shared(cursors.size() - 1); for (size_t i = 0; i < cursors.size() - 1; ++i) { - auto const start = cursors[i]; - auto const end = cursors[i + 1]; + auto const start = cursors.at(i); + auto const end = cursors.at(i + 1); markers->wait(numCacheMarkers_); ++(*markers); @@ -386,14 +383,14 @@ private: boost::asio::spawn( ioContext_.get(), [this, seq, start, end, numRemaining, startTime, markers](boost::asio::yield_context yield) { - std::optional cursor = start; + auto cursor = start; std::string cursorStr = cursor.has_value() ? ripple::strHex(cursor.value()) : ripple::strHex(data::firstKey); LOG(log_.debug()) << "Starting a cursor: " << cursorStr << " markers = " << *markers; while (not stopping_) { - auto res = data::retryOnTimeout([this, seq, &cursor, &yield]() { + auto res = data::retryOnTimeout([this, seq, &cursor, yield]() { return backend_->fetchLedgerPage(cursor, seq, cachePageFetchSize_, false, yield); }); @@ -428,7 +425,7 @@ private: } }); } - }}; + }); } }; diff --git a/src/main/Main.cpp b/src/main/Main.cpp index 667df246..e4008ee5 100644 --- a/src/main/Main.cpp +++ b/src/main/Main.cpp @@ -184,7 +184,7 @@ try auto dosGuard = web::DOSGuard{config, whitelistHandler, sweepHandler}; // Interface to the database - auto backend = data::make_Backend(ioc, config); + auto backend = data::make_Backend(config); // Manages clients subscribed to streams auto subscriptions = feed::SubscriptionManager::make_SubscriptionManager(config, backend); @@ -208,7 +208,7 @@ try auto const rpcEngine = rpc::RPCEngine::make_RPCEngine( config, backend, subscriptions, balancer, etl, dosGuard, workQueue, counters, handlerProvider); - // init the web server + // Init the web server auto handler = std::make_shared>( config, backend, rpcEngine, etl, subscriptions); auto ctx = parseCerts(config); diff --git a/src/rpc/handlers/NFTOffersCommon.cpp b/src/rpc/handlers/NFTOffersCommon.cpp index 20fac538..9eb1a801 100644 --- a/src/rpc/handlers/NFTOffersCommon.cpp +++ b/src/rpc/handlers/NFTOffersCommon.cpp @@ -84,7 +84,7 @@ NFTOffersHandlerBase::iterateOfferDirectory( cursor = uint256(input.marker->c_str()); // We have a start point. Use limit - 1 from the result and use the very last one for the resume. - auto const sle = [this, &cursor, &lgrInfo, &yield]() -> std::shared_ptr { + auto const sle = [this, &cursor, &lgrInfo, yield]() -> std::shared_ptr { auto const key = keylet::nftoffer(cursor).key; if (auto const blob = sharedPtrBackend_->fetchLedgerObject(key, lgrInfo.seq, yield); blob) diff --git a/src/web/DOSGuard.h b/src/web/DOSGuard.h index 60360096..888043b0 100644 --- a/src/web/DOSGuard.h +++ b/src/web/DOSGuard.h @@ -20,6 +20,8 @@ #pragma once #include +#include +#include #include #include @@ -253,66 +255,6 @@ private: } }; -/** - * @brief Sweep handler using a steady_timer and boost::asio::io_context. - */ -class IntervalSweepHandler -{ - std::chrono::milliseconds sweepInterval_; - std::reference_wrapper ctx_; - boost::asio::steady_timer timer_; - - BaseDOSGuard* dosGuard_ = nullptr; - -public: - /** - * @brief Construct a new interval-based sweep handler. - * - * @param config Clio config - * @param ctx The boost::asio::io_context - */ - IntervalSweepHandler(util::Config const& config, boost::asio::io_context& ctx) - : sweepInterval_{std::max(1u, static_cast(config.valueOr("dos_guard.sweep_interval", 1.0) * 1000.0))} - , ctx_{std::ref(ctx)} - , timer_{ctx.get_executor()} - { - } - - ~IntervalSweepHandler() - { - timer_.cancel(); - } - - /** - * @brief This setup member function is called by @ref BasicDOSGuard during its initialization. - * - * @param guard Pointer to the dos guard - */ - void - setup(BaseDOSGuard* guard) - { - assert(dosGuard_ == nullptr); - dosGuard_ = guard; - assert(dosGuard_ != nullptr); - - createTimer(); - } - -private: - void - createTimer() - { - timer_.expires_after(sweepInterval_); - timer_.async_wait([this](boost::system::error_code const& error) { - if (error == boost::asio::error::operation_aborted) - return; - - dosGuard_->clear(); - boost::asio::post(ctx_.get().get_executor(), [this] { createTimer(); }); - }); - } -}; - -using DOSGuard = BasicDOSGuard; +using DOSGuard = BasicDOSGuard; } // namespace web diff --git a/src/web/IntervalSweepHandler.cpp b/src/web/IntervalSweepHandler.cpp new file mode 100644 index 00000000..2afe7e9a --- /dev/null +++ b/src/web/IntervalSweepHandler.cpp @@ -0,0 +1,64 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2023, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +#include + +#include +#include + +namespace web { + +IntervalSweepHandler::IntervalSweepHandler(util::Config const& config, boost::asio::io_context& ctx) + : sweepInterval_{std::max(1u, static_cast(config.valueOr("dos_guard.sweep_interval", 1.0) * 1000.0))} + , ctx_{std::ref(ctx)} + , timer_{ctx.get_executor()} +{ +} + +IntervalSweepHandler::~IntervalSweepHandler() +{ + boost::asio::post(ctx_.get(), [this]() { timer_.cancel(); }); +} + +void +IntervalSweepHandler::setup(web::BaseDOSGuard* guard) +{ + assert(dosGuard_ == nullptr); + dosGuard_ = guard; + assert(dosGuard_ != nullptr); + + createTimer(); +} + +void +IntervalSweepHandler::createTimer() +{ + timer_.expires_after(sweepInterval_); + timer_.async_wait([this](boost::system::error_code const& error) { + if (error == boost::asio::error::operation_aborted) + return; + + dosGuard_->clear(); + boost::asio::post(ctx_.get(), [this] { createTimer(); }); + }); +} + +} // namespace web diff --git a/src/web/impl/IntervalSweepHandler.h b/src/web/IntervalSweepHandler.h similarity index 58% rename from src/web/impl/IntervalSweepHandler.h rename to src/web/IntervalSweepHandler.h index 6e8da34c..d9a0f215 100644 --- a/src/web/impl/IntervalSweepHandler.h +++ b/src/web/IntervalSweepHandler.h @@ -19,15 +19,15 @@ #pragma once +#include + #include -#include -#include -#include #include -#include -namespace web::detail { +namespace web { + +class BaseDOSGuard; /** * @brief Sweep handler using a steady_timer and boost::asio::io_context. @@ -42,52 +42,29 @@ class IntervalSweepHandler public: /** - * @brief Construct a new interval-based sweep handler + * @brief Construct a new interval-based sweep handler. * - * @param config Clio config - * @param ctx The boost::asio::io_context + * @param config Clio config to use + * @param ctx The boost::asio::io_context to use */ - IntervalSweepHandler(util::Config const& config, boost::asio::io_context& ctx) - : sweepInterval_{std::max(1u, static_cast(config.valueOr("dos_guard.sweep_interval", 1.0) * 1000.0))} - , ctx_{std::ref(ctx)} - , timer_{ctx.get_executor()} - { - } - - ~IntervalSweepHandler() - { - timer_.cancel(); - } + IntervalSweepHandler(util::Config const& config, boost::asio::io_context& ctx); /** - * @brief This setup member function is called by @ref BasicDOSGuard during - * its initialization. + * @brief Cancels the sweep timer. + */ + ~IntervalSweepHandler(); + + /** + * @brief This setup member function is called by @ref BasicDOSGuard during its initialization. * * @param guard Pointer to the dos guard */ void - setup(web::BaseDOSGuard* guard) - { - assert(dosGuard_ == nullptr); - dosGuard_ = guard; - assert(dosGuard_ != nullptr); - - createTimer(); - } + setup(web::BaseDOSGuard* guard); private: void - createTimer() - { - timer_.expires_after(sweepInterval_); - timer_.async_wait([this](boost::system::error_code const& error) { - if (error == boost::asio::error::operation_aborted) - return; - - dosGuard_->clear(); - boost::asio::post(ctx_.get().get_executor(), [this] { createTimer(); }); - }); - } + createTimer(); }; -} // namespace web::detail \ No newline at end of file +} // namespace web diff --git a/src/web/WhitelistHandler.h b/src/web/WhitelistHandler.h index 53d71506..fed7f3bf 100644 --- a/src/web/WhitelistHandler.h +++ b/src/web/WhitelistHandler.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -30,8 +31,7 @@ namespace web { /** - * @brief A whitelist to remove rate limits of certain IP addresses - * + * @brief A whitelist to remove rate limits of certain IP addresses. */ class Whitelist { @@ -41,7 +41,7 @@ class Whitelist public: /** - * @brief Add network address to whitelist + * @brief Add network address to whitelist. * * @param net Network part of the ip address * @throws std::runtime::error when the network address is not valid @@ -66,7 +66,7 @@ public: } /** - * @brief Checks to see if ip address is whitelisted + * @brief Checks to see if ip address is whitelisted. * * @param ip IP address * @throws std::runtime::error when the network address is not valid @@ -130,15 +130,18 @@ private: }; /** - * @brief A simple handler to add/check elements in a whitelist - * - * @param arr map of net addresses to add to whitelist + * @brief A simple handler to add/check elements in a whitelist. */ class WhitelistHandler { Whitelist whitelist_; public: + /** + * @brief Adds all whitelisted IPs and masks from the given config. + * + * @param config The Clio config to use + */ WhitelistHandler(util::Config const& config) { std::unordered_set arr = getWhitelist(config); @@ -146,6 +149,9 @@ public: whitelist_.add(net); } + /** + * @return true if the given IP is whitelisted; false otherwise + */ bool isWhiteListed(std::string_view ip) const { diff --git a/unittests/data/BackendFactoryTests.cpp b/unittests/data/BackendFactoryTests.cpp index 92e5ddc1..4f5c638e 100644 --- a/unittests/data/BackendFactoryTests.cpp +++ b/unittests/data/BackendFactoryTests.cpp @@ -74,7 +74,7 @@ TEST_F(BackendCassandraFactoryTest, NoSuchBackend) "type":"unknown" } })")}; - EXPECT_THROW(make_Backend(ctx, cfg), std::runtime_error); + EXPECT_THROW(make_Backend(cfg), std::runtime_error); } TEST_F(BackendCassandraFactoryTest, CreateCassandraBackendDBDisconnect) @@ -94,7 +94,7 @@ TEST_F(BackendCassandraFactoryTest, CreateCassandraBackendDBDisconnect) }})", "127.0.0.2", keyspace))}; - EXPECT_THROW(make_Backend(ctx, cfg), std::runtime_error); + EXPECT_THROW(make_Backend(cfg), std::runtime_error); } TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend) @@ -115,7 +115,7 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend) keyspace))}; { - auto backend = make_Backend(ctx, cfg); + auto backend = make_Backend(cfg); EXPECT_TRUE(backend); // empty db does not have ledger range @@ -129,7 +129,7 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend) } { - auto backend = make_Backend(ctx, cfg); + auto backend = make_Backend(cfg); EXPECT_TRUE(backend); auto const range = backend->fetchLedgerRange(); @@ -155,7 +155,7 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackendReadOnlyWithEmpt }})", contactPoints, keyspace))}; - EXPECT_THROW(make_Backend(ctx, cfg), std::runtime_error); + EXPECT_THROW(make_Backend(cfg), std::runtime_error); } TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackendReadOnlyWithDBReady) @@ -192,6 +192,6 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackendReadOnlyWithDBRe contactPoints, keyspace))}; - EXPECT_TRUE(make_Backend(ctx, cfgWrite)); - EXPECT_TRUE(make_Backend(ctx, cfgReadOnly)); + EXPECT_TRUE(make_Backend(cfgWrite)); + EXPECT_TRUE(make_Backend(cfgReadOnly)); } diff --git a/unittests/web/SweepHandlerTests.cpp b/unittests/web/SweepHandlerTests.cpp index a30f4330..b3e80289 100644 --- a/unittests/web/SweepHandlerTests.cpp +++ b/unittests/web/SweepHandlerTests.cpp @@ -52,4 +52,4 @@ TEST_F(DOSGuardIntervalSweepHandlerTest, SweepAfterInterval) { EXPECT_CALL(guard, clear()).Times(AtLeast(2)); ctx.run_for(std::chrono::milliseconds(400)); -} \ No newline at end of file +}