Update RocksDB to 6.5:

* update EP and find package requirements
* minor protobuf/libarchive build fixes
* change travis release builds to nounity to
  ameliorate vm memory exhaustion.

FIXES: #3223, #3232

.travis.yml

@@ -115,7 +115,7 @@ matrix:
       env:
         - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
         - BUILD_TYPE=Release
-        - CMAKE_ADD="-Dassert=ON"
+        - CMAKE_ADD="-Dassert=ON -Dunity=OFF"
         - MANUAL_TESTS=true
     # release builds
     - <<: *linux
@@ -125,7 +125,7 @@ matrix:
       env:
         - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
         - BUILD_TYPE=Release
-        - CMAKE_ADD="-Dassert=ON"
+        - CMAKE_ADD="-Dassert=ON -Dunity=OFF"
     - <<: *linux
       if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_release/
       compiler: clang-8

Builds/CMake/FindRocksDB.cmake

@@ -0,0 +1,62 @@
+set (RocksDB_DIR "" CACHE PATH "Root directory of RocksDB distribution")
+
+find_path (RocksDB_INCLUDE_DIR
+    rocksdb/db.h
+    PATHS ${RocksDB_DIR})
+
+set (RocksDB_VERSION "")
+find_file (RocksDB_VERSION_FILE
+    rocksdb/version.h
+    PATHS ${RocksDB_DIR})
+if (RocksDB_VERSION_FILE)
+    file (READ ${RocksDB_VERSION_FILE} _verfile)
+    if ("${_verfile}" MATCHES "#define[ \\t]+ROCKSDB_MAJOR[ \\t]+([0-9]+)")
+        string (APPEND RocksDB_VERSION "${CMAKE_MATCH_1}")
+    else ()
+        string (APPEND RocksDB_VERSION "0")
+    endif()
+    if ("${_verfile}" MATCHES "#define[ \\t]+ROCKSDB_MINOR[ \\t]+([0-9]+)")
+        string (APPEND RocksDB_VERSION ".${CMAKE_MATCH_1}")
+    else ()
+        string (APPEND RocksDB_VERSION ".0")
+    endif()
+    if ("${_verfile}" MATCHES "#define[ \\t]+ROCKSDB_PATCH[ \\t]+([0-9]+)")
+        string (APPEND RocksDB_VERSION ".${CMAKE_MATCH_1}")
+    else ()
+        string (APPEND RocksDB_VERSION ".0")
+    endif()
+endif ()
+
+if (RocksDB_USE_STATIC)
+    list (APPEND RocksDB_NAMES
+        "${CMAKE_STATIC_LIBRARY_PREFIX}rocksdb${CMAKE_STATIC_LIBRARY_SUFFIX}"
+        "${CMAKE_STATIC_LIBRARY_PREFIX}rocksdblib${CMAKE_STATIC_LIBRARY_SUFFIX}")
+endif ()
+
+list (APPEND RocksDB_NAMES rocksdb)
+
+find_library (RocksDB_LIBRARY NAMES ${RocksDB_NAMES}
+    PATHS
+    ${RocksDB_DIR}
+    ${RocksDB_DIR}/bin/Release
+    ${RocksDB_DIR}/bin64_vs2013/Release
+    PATH_SUFFIXES lib lib64)
+
+foreach (_n RocksDB_NAMES)
+    list (APPEND RocksDB_NAMES_DBG "${_n}_d" "${_n}d")
+endforeach ()
+find_library (RocksDB_LIBRARY_DEBUG NAMES ${RocksDB_NAMES_DBG}
+    PATHS
+    ${RocksDB_DIR}
+    ${RocksDB_DIR}/bin/Debug
+    ${RocksDB_DIR}/bin64_vs2013/Debug
+    PATH_SUFFIXES lib lib64)
+
+include (FindPackageHandleStandardArgs)
+find_package_handle_standard_args (RocksDB
+    REQUIRED_VARS RocksDB_LIBRARY RocksDB_INCLUDE_DIR
+    VERSION_VAR RocksDB_VERSION)
+
+mark_as_advanced (RocksDB_INCLUDE_DIR RocksDB_LIBRARY)
+set (RocksDB_INCLUDE_DIRS ${RocksDB_INCLUDE_DIR})
+set (RocksDB_LIBRARIES ${RocksDB_LIBRARY})

Builds/CMake/deps/Libarchive.cmake

@@ -3,7 +3,7 @@
 #]===================================================================]
 
 option (local_libarchive "use local build of libarchive." OFF)
-add_library (archive_lib STATIC IMPORTED GLOBAL)
+add_library (archive_lib UNKNOWN IMPORTED GLOBAL)
 
 if (NOT local_libarchive)
   if (NOT WIN32)
@@ -18,6 +18,15 @@ if (NOT local_libarchive)
         ${archive}
       INTERFACE_INCLUDE_DIRECTORIES
         ${LIBARCHIVE_INCLUDE_DIR})
+      # pkg-config can return extra info for static lib linking
+      # this is probably needed/useful generally, but apply
+      # to APPLE for now (mostly for homebrew)
+      if (APPLE AND static AND libarchive_PC_STATIC_LIBRARIES)
+        message(STATUS "NOTE: libarchive static libs: ${libarchive_PC_STATIC_LIBRARIES}")
+        # also, APPLE seems to need iconv...maybe linux does too (TBD)
+        target_link_libraries (archive_lib
+          INTERFACE iconv ${libarchive_PC_STATIC_LIBRARIES})
+      endif ()
   else ()
     ## now try searching using the minimal find module that cmake provides
     find_package(LibArchive 3.3.3 QUIET)

Builds/CMake/deps/Protobuf.cmake

@@ -10,6 +10,7 @@ if (static)
 endif ()
 find_package (Protobuf 3.8)
 if (local_protobuf OR NOT Protobuf_FOUND)
+  include (GNUInstallDirs)
   message (STATUS "using local protobuf build.")
   if (WIN32)
     # protobuf prepends lib even on windows
@@ -56,10 +57,10 @@ if (local_protobuf OR NOT Protobuf_FOUND)
     INSTALL_COMMAND
       ${CMAKE_COMMAND} -E env --unset=DESTDIR ${CMAKE_COMMAND} --build . --config $<CONFIG> --target install
     BUILD_BYPRODUCTS
-      <BINARY_DIR>/_installed_/lib/${pbuf_lib_pre}protobuf${ep_lib_suffix}
+      <BINARY_DIR>/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protobuf${ep_lib_suffix}
-      <BINARY_DIR>/_installed_/lib/${pbuf_lib_pre}protobuf_d${ep_lib_suffix}
+      <BINARY_DIR>/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protobuf_d${ep_lib_suffix}
-      <BINARY_DIR>/_installed_/lib/${pbuf_lib_pre}protoc${ep_lib_suffix}
+      <BINARY_DIR>/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protoc${ep_lib_suffix}
-      <BINARY_DIR>/_installed_/lib/${pbuf_lib_pre}protoc_d${ep_lib_suffix}
+      <BINARY_DIR>/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protoc_d${ep_lib_suffix}
       <BINARY_DIR>/_installed_/bin/protoc${CMAKE_EXECUTABLE_SUFFIX}
   )
   ExternalProject_Get_Property (protobuf_src BINARY_DIR)
@@ -75,9 +76,9 @@ if (local_protobuf OR NOT Protobuf_FOUND)
   file (MAKE_DIRECTORY ${BINARY_DIR}/_installed_/include)
   set_target_properties (protobuf::libprotobuf PROPERTIES
     IMPORTED_LOCATION_DEBUG
-      ${BINARY_DIR}/_installed_/lib/${pbuf_lib_pre}protobuf_d${ep_lib_suffix}
+      ${BINARY_DIR}/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protobuf_d${ep_lib_suffix}
     IMPORTED_LOCATION_RELEASE
-      ${BINARY_DIR}/_installed_/lib/${pbuf_lib_pre}protobuf${ep_lib_suffix}
+      ${BINARY_DIR}/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protobuf${ep_lib_suffix}
     INTERFACE_INCLUDE_DIRECTORIES
       ${BINARY_DIR}/_installed_/include)
   add_dependencies (protobuf::libprotobuf protobuf_src)
@@ -88,9 +89,9 @@ if (local_protobuf OR NOT Protobuf_FOUND)
   endif ()
   set_target_properties (protobuf::libprotoc PROPERTIES
     IMPORTED_LOCATION_DEBUG
-      ${BINARY_DIR}/_installed_/lib/${pbuf_lib_pre}protoc_d${ep_lib_suffix}
+      ${BINARY_DIR}/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protoc_d${ep_lib_suffix}
     IMPORTED_LOCATION_RELEASE
-      ${BINARY_DIR}/_installed_/lib/${pbuf_lib_pre}protoc${ep_lib_suffix}
+      ${BINARY_DIR}/_installed_/${CMAKE_INSTALL_LIBDIR}/${pbuf_lib_pre}protoc${ep_lib_suffix}
     INTERFACE_INCLUDE_DIRECTORIES
       ${BINARY_DIR}/_installed_/include)
   add_dependencies (protobuf::libprotoc protobuf_src)

Builds/CMake/deps/Rocksdb.cmake

@@ -2,23 +2,65 @@
    NIH dep: rocksdb
 #]===================================================================]
 
-find_package (RocksDB CONFIG QUIET)
+add_library (rocksdb_lib UNKNOWN IMPORTED GLOBAL)
-if (TARGET RocksDB::rocksdb)
+set_target_properties (rocksdb_lib
-  get_target_property (_rockslib_l RocksDB::rocksdb IMPORTED_LOCATION_DEBUG)
+  PROPERTIES INTERFACE_COMPILE_DEFINITIONS RIPPLE_ROCKSDB_AVAILABLE=1)
-  if (NOT _rockslib_l)
+
+option (local_rocksdb "use local build of rocksdb." OFF)
+if (NOT local_rocksdb)
+  find_package (RocksDB 6.5 QUIET CONFIG)
+  if (TARGET RocksDB::rocksdb)
+    message (STATUS "Found RocksDB using config.")
+    get_target_property (_rockslib_l RocksDB::rocksdb IMPORTED_LOCATION_DEBUG)
+    if (_rockslib_l)
+      set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION_DEBUG ${_rockslib_l})
+    endif ()
     get_target_property (_rockslib_l RocksDB::rocksdb IMPORTED_LOCATION_RELEASE)
-  endif ()
+    if (_rockslib_l)
-  if (NOT _rockslib_l)
+      set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION_RELEASE ${_rockslib_l})
+    endif ()
     get_target_property (_rockslib_l RocksDB::rocksdb IMPORTED_LOCATION)
+    if (_rockslib_l)
+      set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION ${_rockslib_l})
+    endif ()
+    get_target_property (_rockslib_i RocksDB::rocksdb INTERFACE_INCLUDE_DIRECTORIES)
+    if (_rockslib_i)
+      set_target_properties (rocksdb_lib PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${_rockslib_i})
+    endif ()
+    target_link_libraries (ripple_libs INTERFACE RocksDB::rocksdb)
+  else ()
+    # using a find module with rocksdb is difficult because
+    # you have no idea how it was configured (transitive dependencies).
+    # the code below will generally find rocksdb using the module, but
+    # will then result in linker errors for static linkage since the
+    # transitive dependencies are unknown. force local build here for now, but leave the code as
+    # a placeholder for future investigation.
+    if (static)
+      set (local_rocksdb ON CACHE BOOL "" FORCE)
+      # TBD if there is some way to extract transitive deps..then:
+      #set (RocksDB_USE_STATIC ON)
+    else ()
+      find_package (RocksDB 6.5 MODULE)
+      if (ROCKSDB_FOUND)
+        if (RocksDB_LIBRARY_DEBUG)
+          set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION_DEBUG ${RocksDB_LIBRARY_DEBUG})
+        endif ()
+        set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION_RELEASE ${RocksDB_LIBRARIES})
+        set_target_properties (rocksdb_lib PROPERTIES IMPORTED_LOCATION ${RocksDB_LIBRARIES})
+        set_target_properties (rocksdb_lib PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${RocksDB_INCLUDE_DIRS})
+      else ()
+        set (local_rocksdb ON CACHE BOOL "" FORCE)
+      endif ()
+    endif ()
   endif ()
-  message (STATUS "Found RocksDB using module/config. Using ${_rockslib_l}.")
+endif ()
-  target_link_libraries (ripple_libs INTERFACE RocksDB::rocksdb)
+
-else ()
+if (local_rocksdb)
-  add_library (rocksdb_lib STATIC IMPORTED GLOBAL)
+  message (STATUS "Using local build of RocksDB.")
   ExternalProject_Add (rocksdb
     PREFIX ${nih_cache_path}
     GIT_REPOSITORY https://github.com/facebook/rocksdb.git
-    GIT_TAG v5.17.2
+    GIT_TAG v6.5.3
     PATCH_COMMAND
       # only used by windows build
       ${CMAKE_COMMAND} -E copy
@@ -47,13 +89,15 @@ else ()
       -DWITH_GFLAGS=OFF
       -DWITH_BZ2=OFF
       -ULZ4_*
-      -DLZ4_INCLUDE_DIR=$<JOIN:$<TARGET_PROPERTY:lz4_lib,INTERFACE_INCLUDE_DIRECTORIES>,::>
+      -Ulz4_*
-      -DLZ4_LIBRARIES=$<IF:$<CONFIG:Debug>,$<TARGET_PROPERTY:lz4_lib,IMPORTED_LOCATION_DEBUG>,$<TARGET_PROPERTY:lz4_lib,IMPORTED_LOCATION_RELEASE>>
+      -Dlz4_INCLUDE_DIRS=$<JOIN:$<TARGET_PROPERTY:lz4_lib,INTERFACE_INCLUDE_DIRECTORIES>,::>
-      -DLZ4_FOUND=ON
+      -Dlz4_LIBRARIES=$<IF:$<CONFIG:Debug>,$<TARGET_PROPERTY:lz4_lib,IMPORTED_LOCATION_DEBUG>,$<TARGET_PROPERTY:lz4_lib,IMPORTED_LOCATION_RELEASE>>
+      -Dlz4_FOUND=ON
       -USNAPPY_*
-      -DSNAPPY_INCLUDE_DIR=$<JOIN:$<TARGET_PROPERTY:snappy_lib,INTERFACE_INCLUDE_DIRECTORIES>,::>
+      -Usnappy_*
-      -DSNAPPY_LIBRARIES=$<IF:$<CONFIG:Debug>,$<TARGET_PROPERTY:snappy_lib,IMPORTED_LOCATION_DEBUG>,$<TARGET_PROPERTY:snappy_lib,IMPORTED_LOCATION_RELEASE>>
+      -Dsnappy_INCLUDE_DIRS=$<JOIN:$<TARGET_PROPERTY:snappy_lib,INTERFACE_INCLUDE_DIRECTORIES>,::>
-      -DSNAPPY_FOUND=ON
+      -Dsnappy_LIBRARIES=$<IF:$<CONFIG:Debug>,$<TARGET_PROPERTY:snappy_lib,IMPORTED_LOCATION_DEBUG>,$<TARGET_PROPERTY:snappy_lib,IMPORTED_LOCATION_RELEASE>>
+      -Dsnappy_FOUND=ON
       -DWITH_MD_LIBRARY=OFF
       -DWITH_RUNTIME_DEBUG=$<IF:$<CONFIG:Debug>,ON,OFF>
       -DFAIL_ON_WARNINGS=OFF
@@ -114,15 +158,15 @@ else ()
     IMPORTED_LOCATION_RELEASE
       ${BINARY_DIR}/${ep_lib_prefix}rocksdb${ep_lib_suffix}
     INTERFACE_INCLUDE_DIRECTORIES
-      ${SOURCE_DIR}/include
+      ${SOURCE_DIR}/include)
-    INTERFACE_COMPILE_DEFINITIONS
-      RIPPLE_ROCKSDB_AVAILABLE=1)
   add_dependencies (rocksdb_lib rocksdb)
-  target_link_libraries (rocksdb_lib INTERFACE snappy_lib lz4_lib)
-  if (MSVC)
-    target_link_libraries (rocksdb_lib INTERFACE rpcrt4)
-  endif ()
-  target_link_libraries (ripple_libs INTERFACE rocksdb_lib)
   exclude_if_included (rocksdb)
-  exclude_if_included (rocksdb_lib)
 endif ()
+
+target_link_libraries (rocksdb_lib
+  INTERFACE
+    snappy_lib
+    lz4_lib
+    $<$<BOOL:${MSVC}>:rpcrt4>)
+exclude_if_included (rocksdb_lib)
+target_link_libraries (ripple_libs INTERFACE rocksdb_lib)

Builds/CMake/deps/Snappy.cmake

@@ -50,7 +50,8 @@ else()
           <BINARY_DIR>
         >
     TEST_COMMAND ""
-    INSTALL_COMMAND ""
+    INSTALL_COMMAND
+      ${CMAKE_COMMAND} -E copy_if_different <BINARY_DIR>/config.h <BINARY_DIR>/snappy-stubs-public.h <SOURCE_DIR>
     BUILD_BYPRODUCTS
       <BINARY_DIR>/${ep_lib_prefix}snappy${ep_lib_suffix}
       <BINARY_DIR>/${ep_lib_prefix}snappy_d${ep_lib_suffix}
@@ -67,7 +68,7 @@ else()
     IMPORTED_LOCATION_RELEASE
       ${BINARY_DIR}/${ep_lib_prefix}snappy${ep_lib_suffix}
     INTERFACE_INCLUDE_DIRECTORIES
-      "${SOURCE_DIR};${BINARY_DIR}")
+      ${SOURCE_DIR})
 endif()
 
 add_dependencies (snappy_lib snappy)

Builds/CMake/rocks_thirdparty.inc

@@ -1,16 +1,15 @@
 set (THIRDPARTY_LIBS "")
+
 if(WITH_SNAPPY)
-  find_package(snappy REQUIRED)
   add_definitions(-DSNAPPY)
-  include_directories(${SNAPPY_INCLUDE_DIR})
+  include_directories(${snappy_INCLUDE_DIRS})
-  list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES})
+  set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${snappy_LIBRARIES})
 endif()
 
 if(WITH_LZ4)
-  find_package(lz4 REQUIRED)
   add_definitions(-DLZ4)
-  include_directories(${LZ4_INCLUDE_DIR})
+  include_directories(${lz4_INCLUDE_DIRS})
-  list(APPEND THIRDPARTY_LIBS ${LZ4_LIBRARIES})
+  set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${lz4_LIBRARIES})
 endif()