From 4315913a5d6002b5055df9746ab0254bdf7c3629 Mon Sep 17 00:00:00 2001 From: Mike Ellery Date: Wed, 29 Jan 2020 08:36:00 -0800 Subject: [PATCH] Update RocksDB to 6.5: * update EP and find package requirements * minor protobuf/libarchive build fixes * change travis release builds to nounity to ameliorate vm memory exhaustion. FIXES: #3223, #3232 --- .travis.yml | 4 +- Builds/CMake/FindRocksDB.cmake | 62 +++++++++++++++++++ Builds/CMake/deps/Libarchive.cmake | 11 +++- Builds/CMake/deps/Protobuf.cmake | 17 +++--- Builds/CMake/deps/Rocksdb.cmake | 96 ++++++++++++++++++++++-------- Builds/CMake/deps/Snappy.cmake | 5 +- Builds/CMake/rocks_thirdparty.inc | 11 ++-- 7 files changed, 161 insertions(+), 45 deletions(-) create mode 100644 Builds/CMake/FindRocksDB.cmake diff --git a/.travis.yml b/.travis.yml index 5e2284750..12ff07195 100644 --- a/.travis.yml +++ b/.travis.yml @@ -115,7 +115,7 @@ matrix: env: - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" - BUILD_TYPE=Release - - CMAKE_ADD="-Dassert=ON" + - CMAKE_ADD="-Dassert=ON -Dunity=OFF" - MANUAL_TESTS=true # release builds - <<: *linux @@ -125,7 +125,7 @@ matrix: env: - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" - BUILD_TYPE=Release - - CMAKE_ADD="-Dassert=ON" + - CMAKE_ADD="-Dassert=ON -Dunity=OFF" - <<: *linux if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_release/ compiler: clang-8 diff --git a/Builds/CMake/FindRocksDB.cmake b/Builds/CMake/FindRocksDB.cmake new file mode 100644 index 000000000..e0d35706b --- /dev/null +++ b/Builds/CMake/FindRocksDB.cmake @@ -0,0 +1,62 @@ +set (RocksDB_DIR "" CACHE PATH "Root directory of RocksDB distribution") + +find_path (RocksDB_INCLUDE_DIR + rocksdb/db.h + PATHS ${RocksDB_DIR}) + +set (RocksDB_VERSION "") +find_file (RocksDB_VERSION_FILE + rocksdb/version.h + PATHS ${RocksDB_DIR}) +if (RocksDB_VERSION_FILE) + file (READ ${RocksDB_VERSION_FILE} _verfile) + if ("${_verfile}" MATCHES "#define[ \\t]+ROCKSDB_MAJOR[ \\t]+([0-9]+)") + string (APPEND RocksDB_VERSION "${CMAKE_MATCH_1}") + else () + string (APPEND RocksDB_VERSION "0") + endif() + if ("${_verfile}" MATCHES "#define[ \\t]+ROCKSDB_MINOR[ \\t]+([0-9]+)") + string (APPEND RocksDB_VERSION ".${CMAKE_MATCH_1}") + else () + string (APPEND RocksDB_VERSION ".0") + endif() + if ("${_verfile}" MATCHES "#define[ \\t]+ROCKSDB_PATCH[ \\t]+([0-9]+)") + string (APPEND RocksDB_VERSION ".${CMAKE_MATCH_1}") + else () + string (APPEND RocksDB_VERSION ".0") + endif() +endif () + +if (RocksDB_USE_STATIC) + list (APPEND RocksDB_NAMES + "${CMAKE_STATIC_LIBRARY_PREFIX}rocksdb${CMAKE_STATIC_LIBRARY_SUFFIX}" + "${CMAKE_STATIC_LIBRARY_PREFIX}rocksdblib${CMAKE_STATIC_LIBRARY_SUFFIX}") +endif () + +list (APPEND RocksDB_NAMES rocksdb) + +find_library (RocksDB_LIBRARY NAMES ${RocksDB_NAMES} + PATHS + ${RocksDB_DIR} + ${RocksDB_DIR}/bin/Release + ${RocksDB_DIR}/bin64_vs2013/Release + PATH_SUFFIXES lib lib64) + +foreach (_n RocksDB_NAMES) + list (APPEND RocksDB_NAMES_DBG "${_n}_d" "${_n}d") +endforeach () +find_library (RocksDB_LIBRARY_DEBUG NAMES ${RocksDB_NAMES_DBG} + PATHS + ${RocksDB_DIR} + ${RocksDB_DIR}/bin/Debug + ${RocksDB_DIR}/bin64_vs2013/Debug + PATH_SUFFIXES lib lib64) + +include (FindPackageHandleStandardArgs) +find_package_handle_standard_args (RocksDB + REQUIRED_VARS RocksDB_LIBRARY RocksDB_INCLUDE_DIR + VERSION_VAR RocksDB_VERSION) + +mark_as_advanced (RocksDB_INCLUDE_DIR RocksDB_LIBRARY) +set (RocksDB_INCLUDE_DIRS ${RocksDB_INCLUDE_DIR}) +set (RocksDB_LIBRARIES ${RocksDB_LIBRARY}) diff --git a/Builds/CMake/deps/Libarchive.cmake b/Builds/CMake/deps/Libarchive.cmake index a3ee532e3..db2716f64 100644 --- a/Builds/CMake/deps/Libarchive.cmake +++ b/Builds/CMake/deps/Libarchive.cmake @@ -3,7 +3,7 @@ #]===================================================================] option (local_libarchive "use local build of libarchive." OFF) -add_library (archive_lib STATIC IMPORTED GLOBAL) +add_library (archive_lib UNKNOWN IMPORTED GLOBAL) if (NOT local_libarchive) if (NOT WIN32) @@ -18,6 +18,15 @@ if (NOT local_libarchive) ${archive} INTERFACE_INCLUDE_DIRECTORIES ${LIBARCHIVE_INCLUDE_DIR}) + # pkg-config can return extra info for static lib linking + # this is probably needed/useful generally, but apply + # to APPLE for now (mostly for homebrew) + if (APPLE AND static AND libarchive_PC_STATIC_LIBRARIES) + message(STATUS "NOTE: libarchive static libs: ${libarchive_PC_STATIC_LIBRARIES}") + # also, APPLE seems to need iconv...maybe linux does too (TBD) + target_link_libraries (archive_lib + INTERFACE iconv ${libarchive_PC_STATIC_LIBRARIES}) + endif () else () ## now try searching using the minimal find module that cmake provides find_package(LibArchive 3.3.3 QUIET) diff --git a/Builds/CMake/deps/Protobuf.cmake b/Builds/CMake/deps/Protobuf.cmake index cf54c1fb7..002074e71 100644 --- a/Builds/CMake/deps/Protobuf.cmake +++ b/Builds/CMake/deps/Protobuf.cmake @@ -10,6 +10,7 @@ if (static) endif () find_package (Protobuf 3.8) if (local_protobuf OR NOT Protobuf_FOUND) + include (GNUInstallDirs) message (STATUS "using local protobuf build.") if (WIN32) # protobuf prepends lib even on windows @@ -56,10 +57,10 @@ if (local_protobuf OR NOT Protobuf_FOUND) INSTALL_COMMAND ${CMAKE_COMMAND} -E env --unset=DESTDIR ${CMAKE_COMMAND} --build . --config $ --target install BUILD_BYPRODUCTS - /_installed_/lib/${pbuf_lib_pre}protobuf${ep_lib_suffix} - /_installed_/lib/${pbuf_lib_pre}protobuf_d${ep_lib_suffix} - /_installed_/lib/${pbuf_lib_pre}protoc${ep_lib_suffix} - /_installed_/lib/${pbuf_lib_pre}protoc_d${ep_lib_suffix} + /_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protobuf${ep_lib_suffix} + /_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protobuf_d${ep_lib_suffix} + /_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protoc${ep_lib_suffix} + /_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protoc_d${ep_lib_suffix} /_installed_/bin/protoc${CMAKE_EXECUTABLE_SUFFIX} ) ExternalProject_Get_Property (protobuf_src BINARY_DIR) @@ -75,9 +76,9 @@ if (local_protobuf OR NOT Protobuf_FOUND) file (MAKE_DIRECTORY ${BINARY_DIR}/_installed_/include) set_target_properties (protobuf::libprotobuf PROPERTIES IMPORTED_LOCATION_DEBUG - ${BINARY_DIR}/_installed_/lib/${pbuf_lib_pre}protobuf_d${ep_lib_suffix} + ${BINARY_DIR}/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protobuf_d${ep_lib_suffix} IMPORTED_LOCATION_RELEASE - ${BINARY_DIR}/_installed_/lib/${pbuf_lib_pre}protobuf${ep_lib_suffix} + ${BINARY_DIR}/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protobuf${ep_lib_suffix} INTERFACE_INCLUDE_DIRECTORIES ${BINARY_DIR}/_installed_/include) add_dependencies (protobuf::libprotobuf protobuf_src) @@ -88,9 +89,9 @@ if (local_protobuf OR NOT Protobuf_FOUND) endif () set_target_properties (protobuf::libprotoc PROPERTIES IMPORTED_LOCATION_DEBUG - ${BINARY_DIR}/_installed_/lib/${pbuf_lib_pre}protoc_d${ep_lib_suffix} + ${BINARY_DIR}/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protoc_d${ep_lib_suffix} IMPORTED_LOCATION_RELEASE - ${BINARY_DIR}/_installed_/lib/${pbuf_lib_pre}protoc${ep_lib_suffix} + ${BINARY_DIR}/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protoc${ep_lib_suffix} INTERFACE_INCLUDE_DIRECTORIES ${BINARY_DIR}/_installed_/include) add_dependencies (protobuf::libprotoc protobuf_src) diff --git a/Builds/CMake/deps/Rocksdb.cmake b/Builds/CMake/deps/Rocksdb.cmake index a8190ffe0..029bc0ece 100644 --- a/Builds/CMake/deps/Rocksdb.cmake +++ b/Builds/CMake/deps/Rocksdb.cmake @@ -2,23 +2,65 @@ NIH dep: rocksdb #]===================================================================] -find_package (RocksDB CONFIG QUIET) -if (TARGET RocksDB::rocksdb) - get_target_property (_rockslib_l RocksDB::rocksdb IMPORTED_LOCATION_DEBUG) - if (NOT _rockslib_l) +add_library (rocksdb_lib UNKNOWN IMPORTED GLOBAL) +set_target_properties (rocksdb_lib + PROPERTIES INTERFACE_COMPILE_DEFINITIONS RIPPLE_ROCKSDB_AVAILABLE=1) + +option (local_rocksdb "use local build of rocksdb." OFF) +if (NOT local_rocksdb) + find_package (RocksDB 6.5 QUIET CONFIG) + if (TARGET RocksDB::rocksdb) + message (STATUS "Found RocksDB using config.") + get_target_property (_rockslib_l RocksDB::rocksdb IMPORTED_LOCATION_DEBUG) + if (_rockslib_l) + set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION_DEBUG ${_rockslib_l}) + endif () get_target_property (_rockslib_l RocksDB::rocksdb IMPORTED_LOCATION_RELEASE) - endif () - if (NOT _rockslib_l) + if (_rockslib_l) + set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION_RELEASE ${_rockslib_l}) + endif () get_target_property (_rockslib_l RocksDB::rocksdb IMPORTED_LOCATION) + if (_rockslib_l) + set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION ${_rockslib_l}) + endif () + get_target_property (_rockslib_i RocksDB::rocksdb INTERFACE_INCLUDE_DIRECTORIES) + if (_rockslib_i) + set_target_properties (rocksdb_lib PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${_rockslib_i}) + endif () + target_link_libraries (ripple_libs INTERFACE RocksDB::rocksdb) + else () + # using a find module with rocksdb is difficult because + # you have no idea how it was configured (transitive dependencies). + # the code below will generally find rocksdb using the module, but + # will then result in linker errors for static linkage since the + # transitive dependencies are unknown. force local build here for now, but leave the code as + # a placeholder for future investigation. + if (static) + set (local_rocksdb ON CACHE BOOL "" FORCE) + # TBD if there is some way to extract transitive deps..then: + #set (RocksDB_USE_STATIC ON) + else () + find_package (RocksDB 6.5 MODULE) + if (ROCKSDB_FOUND) + if (RocksDB_LIBRARY_DEBUG) + set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION_DEBUG ${RocksDB_LIBRARY_DEBUG}) + endif () + set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION_RELEASE ${RocksDB_LIBRARIES}) + set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION ${RocksDB_LIBRARIES}) + set_target_properties (rocksdb_lib PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${RocksDB_INCLUDE_DIRS}) + else () + set (local_rocksdb ON CACHE BOOL "" FORCE) + endif () + endif () endif () - message (STATUS "Found RocksDB using module/config. Using ${_rockslib_l}.") - target_link_libraries (ripple_libs INTERFACE RocksDB::rocksdb) -else () - add_library (rocksdb_lib STATIC IMPORTED GLOBAL) +endif () + +if (local_rocksdb) + message (STATUS "Using local build of RocksDB.") ExternalProject_Add (rocksdb PREFIX ${nih_cache_path} GIT_REPOSITORY https://github.com/facebook/rocksdb.git - GIT_TAG v5.17.2 + GIT_TAG v6.5.3 PATCH_COMMAND # only used by windows build ${CMAKE_COMMAND} -E copy @@ -47,13 +89,15 @@ else () -DWITH_GFLAGS=OFF -DWITH_BZ2=OFF -ULZ4_* - -DLZ4_INCLUDE_DIR=$,::> - -DLZ4_LIBRARIES=$,$,$> - -DLZ4_FOUND=ON + -Ulz4_* + -Dlz4_INCLUDE_DIRS=$,::> + -Dlz4_LIBRARIES=$,$,$> + -Dlz4_FOUND=ON -USNAPPY_* - -DSNAPPY_INCLUDE_DIR=$,::> - -DSNAPPY_LIBRARIES=$,$,$> - -DSNAPPY_FOUND=ON + -Usnappy_* + -Dsnappy_INCLUDE_DIRS=$,::> + -Dsnappy_LIBRARIES=$,$,$> + -Dsnappy_FOUND=ON -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=$,ON,OFF> -DFAIL_ON_WARNINGS=OFF @@ -114,15 +158,15 @@ else () IMPORTED_LOCATION_RELEASE ${BINARY_DIR}/${ep_lib_prefix}rocksdb${ep_lib_suffix} INTERFACE_INCLUDE_DIRECTORIES - ${SOURCE_DIR}/include - INTERFACE_COMPILE_DEFINITIONS - RIPPLE_ROCKSDB_AVAILABLE=1) + ${SOURCE_DIR}/include) add_dependencies (rocksdb_lib rocksdb) - target_link_libraries (rocksdb_lib INTERFACE snappy_lib lz4_lib) - if (MSVC) - target_link_libraries (rocksdb_lib INTERFACE rpcrt4) - endif () - target_link_libraries (ripple_libs INTERFACE rocksdb_lib) exclude_if_included (rocksdb) - exclude_if_included (rocksdb_lib) endif () + +target_link_libraries (rocksdb_lib + INTERFACE + snappy_lib + lz4_lib + $<$:rpcrt4>) +exclude_if_included (rocksdb_lib) +target_link_libraries (ripple_libs INTERFACE rocksdb_lib) diff --git a/Builds/CMake/deps/Snappy.cmake b/Builds/CMake/deps/Snappy.cmake index e31bd28f9..9792bec1b 100644 --- a/Builds/CMake/deps/Snappy.cmake +++ b/Builds/CMake/deps/Snappy.cmake @@ -50,7 +50,8 @@ else() > TEST_COMMAND "" - INSTALL_COMMAND "" + INSTALL_COMMAND + ${CMAKE_COMMAND} -E copy_if_different /config.h /snappy-stubs-public.h BUILD_BYPRODUCTS /${ep_lib_prefix}snappy${ep_lib_suffix} /${ep_lib_prefix}snappy_d${ep_lib_suffix} @@ -67,7 +68,7 @@ else() IMPORTED_LOCATION_RELEASE ${BINARY_DIR}/${ep_lib_prefix}snappy${ep_lib_suffix} INTERFACE_INCLUDE_DIRECTORIES - "${SOURCE_DIR};${BINARY_DIR}") + ${SOURCE_DIR}) endif() add_dependencies (snappy_lib snappy) diff --git a/Builds/CMake/rocks_thirdparty.inc b/Builds/CMake/rocks_thirdparty.inc index 068f98f55..cba23df71 100644 --- a/Builds/CMake/rocks_thirdparty.inc +++ b/Builds/CMake/rocks_thirdparty.inc @@ -1,16 +1,15 @@ set (THIRDPARTY_LIBS "") + if(WITH_SNAPPY) - find_package(snappy REQUIRED) add_definitions(-DSNAPPY) - include_directories(${SNAPPY_INCLUDE_DIR}) - list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES}) + include_directories(${snappy_INCLUDE_DIRS}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${snappy_LIBRARIES}) endif() if(WITH_LZ4) - find_package(lz4 REQUIRED) add_definitions(-DLZ4) - include_directories(${LZ4_INCLUDE_DIR}) - list(APPEND THIRDPARTY_LIBS ${LZ4_LIBRARIES}) + include_directories(${lz4_INCLUDE_DIRS}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${lz4_LIBRARIES}) endif()