From 7f2ddf431b1cc354d03a61a95bd1d78cd79abd66 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Tue, 12 Mar 2024 16:01:15 -0500 Subject: [PATCH] Embed patched recipe for RocksDB 6.29.5 (#4947) --- BUILD.md | 7 + external/rocksdb/conandata.yml | 27 ++ external/rocksdb/conanfile.py | 336 ++++++++++-------- ...-0001-add-include-cstdint-for-gcc-13.patch | 30 ++ .../6.29.5-0002-exclude-thirdparty.patch | 16 + 5 files changed, 268 insertions(+), 148 deletions(-) create mode 100644 external/rocksdb/conandata.yml create mode 100644 external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch create mode 100644 external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch diff --git a/BUILD.md b/BUILD.md index c84daf8bc..b562184f3 100644 --- a/BUILD.md +++ b/BUILD.md @@ -147,6 +147,13 @@ which allows you to statically link it with GCC, if you want. conan export external/snappy snappy/1.1.9@ ``` +Export our [Conan recipe for RocksDB](./external/rocksdb). +It does not override paths to dependencies when building with Visual Studio. + + ``` + conan export external/rocksdb rocksdb/6.29.5@ + ``` + Export our [Conan recipe for SOCI](./external/soci). It patches their CMake to correctly import its dependencies. diff --git a/external/rocksdb/conandata.yml b/external/rocksdb/conandata.yml new file mode 100644 index 000000000..86b42f79f --- /dev/null +++ b/external/rocksdb/conandata.yml @@ -0,0 +1,27 @@ +sources: + "6.29.5": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.29.5.tar.gz" + sha256: "ddbf84791f0980c0bbce3902feb93a2c7006f6f53bfd798926143e31d4d756f0" + "6.27.3": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.27.3.tar.gz" + sha256: "ee29901749b9132692b26f0a6c1d693f47d1a9ed8e3771e60556afe80282bf58" + "6.20.3": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.20.3.tar.gz" + sha256: "c6502c7aae641b7e20fafa6c2b92273d935d2b7b2707135ebd9a67b092169dca" + "8.8.1": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v8.8.1.tar.gz" + sha256: "056c7e21ad8ae36b026ac3b94b9d6e0fcc60e1d937fc80330921e4181be5c36e" +patches: + "6.29.5": + - patch_file: "patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch" + patch_description: "Fix build with gcc 13 by including cstdint" + patch_type: "portability" + patch_source: "https://github.com/facebook/rocksdb/pull/11118" + - patch_file: "patches/6.29.5-0002-exclude-thirdparty.patch" + patch_description: "Do not include thirdparty.inc" + patch_type: "portability" + "6.27.3": + - patch_file: "patches/6.27.3-0001-add-include-cstdint-for-gcc-13.patch" + patch_description: "Fix build with gcc 13 by including cstdint" + patch_type: "portability" + patch_source: "https://github.com/facebook/rocksdb/pull/11118" diff --git a/external/rocksdb/conanfile.py b/external/rocksdb/conanfile.py index a219f4f55..09425b9f8 100644 --- a/external/rocksdb/conanfile.py +++ b/external/rocksdb/conanfile.py @@ -1,193 +1,233 @@ import os +import glob import shutil -from conans import ConanFile, CMake -from conan.tools import microsoft as ms -class RocksDB(ConanFile): - name = 'rocksdb' - version = '6.27.3' +from conan import ConanFile +from conan.errors import ConanInvalidConfiguration +from conan.tools.build import check_min_cppstd +from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout +from conan.tools.files import apply_conandata_patches, collect_libs, copy, export_conandata_patches, get, rm, rmdir +from conan.tools.microsoft import check_min_vs, is_msvc, is_msvc_static_runtime +from conan.tools.scm import Version - license = ('GPL-2.0-only', 'Apache-2.0') - url = 'https://github.com/conan-io/conan-center-index' - description = 'A library that provides an embeddable, persistent key-value store for fast storage' - topics = ('rocksdb', 'database', 'leveldb', 'facebook', 'key-value') +required_conan_version = ">=1.53.0" - settings = 'os', 'compiler', 'build_type', 'arch' + +class RocksDBConan(ConanFile): + name = "rocksdb" + homepage = "https://github.com/facebook/rocksdb" + license = ("GPL-2.0-only", "Apache-2.0") + url = "https://github.com/conan-io/conan-center-index" + description = "A library that provides an embeddable, persistent key-value store for fast storage" + topics = ("database", "leveldb", "facebook", "key-value") + package_type = "library" + settings = "os", "arch", "compiler", "build_type" options = { - 'enable_sse': [False, 'sse42', 'avx2'], - 'fPIC': [True, False], - 'lite': [True, False], - 'shared': [True, False], - 'use_rtti': [True, False], - 'with_gflags': [True, False], - 'with_jemalloc': [True, False], - 'with_lz4': [True, False], - 'with_snappy': [True, False], - 'with_tbb': [True, False], - 'with_zlib': [True, False], - 'with_zstd': [True, False], + "shared": [True, False], + "fPIC": [True, False], + "lite": [True, False], + "with_gflags": [True, False], + "with_snappy": [True, False], + "with_lz4": [True, False], + "with_zlib": [True, False], + "with_zstd": [True, False], + "with_tbb": [True, False], + "with_jemalloc": [True, False], + "enable_sse": [False, "sse42", "avx2"], + "use_rtti": [True, False], } default_options = { - 'enable_sse': False, - 'fPIC': True, - 'lite': False, - 'shared': False, - 'use_rtti': False, - 'with_gflags': False, - 'with_jemalloc': False, - 'with_lz4': False, - 'with_snappy': False, - 'with_tbb': False, - 'with_zlib': False, - 'with_zstd': False, + "shared": False, + "fPIC": True, + "lite": False, + "with_snappy": False, + "with_lz4": False, + "with_zlib": False, + "with_zstd": False, + "with_gflags": False, + "with_tbb": False, + "with_jemalloc": False, + "enable_sse": False, + "use_rtti": False, } - def requirements(self): - if self.options.with_gflags: - self.requires('gflags/2.2.2') - if self.options.with_jemalloc: - self.requires('jemalloc/5.2.1') - if self.options.with_lz4: - self.requires('lz4/1.9.3') - if self.options.with_snappy: - self.requires('snappy/1.1.9') - if self.options.with_tbb: - self.requires('onetbb/2020.3') - if self.options.with_zlib: - self.requires('zlib/1.2.11') - if self.options.with_zstd: - self.requires('zstd/1.5.2') + @property + def _min_cppstd(self): + return "11" if Version(self.version) < "8.8.1" else "17" + + @property + def _compilers_minimum_version(self): + return {} if self._min_cppstd == "11" else { + "apple-clang": "10", + "clang": "7", + "gcc": "7", + "msvc": "191", + "Visual Studio": "15", + } + + def export_sources(self): + export_conandata_patches(self) def config_options(self): - if self.settings.os == 'Windows': + if self.settings.os == "Windows": del self.options.fPIC + if self.settings.arch != "x86_64": + del self.options.with_tbb + if self.settings.build_type == "Debug": + self.options.use_rtti = True # Rtti are used in asserts for debug mode... def configure(self): if self.options.shared: - del self.options.fPIC + self.options.rm_safe("fPIC") - generators = 'cmake', 'cmake_find_package' + def layout(self): + cmake_layout(self, src_folder="src") - scm = { - 'type': 'git', - 'url': 'https://github.com/facebook/rocksdb.git', - 'revision': 'v6.27.3', - } + def requirements(self): + if self.options.with_gflags: + self.requires("gflags/2.2.2") + if self.options.with_snappy: + self.requires("snappy/1.1.10") + if self.options.with_lz4: + self.requires("lz4/1.9.4") + if self.options.with_zlib: + self.requires("zlib/[>=1.2.11 <2]") + if self.options.with_zstd: + self.requires("zstd/1.5.5") + if self.options.get_safe("with_tbb"): + self.requires("onetbb/2021.10.0") + if self.options.with_jemalloc: + self.requires("jemalloc/5.3.0") - exports_sources = 'thirdparty.inc' - # For out-of-source build. - no_copy_source = True + def validate(self): + if self.settings.compiler.get_safe("cppstd"): + check_min_cppstd(self, self._min_cppstd) - _cmake = None - - def _configure_cmake(self): - if self._cmake: - return - - self._cmake = CMake(self) - - self._cmake.definitions['CMAKE_POSITION_INDEPENDENT_CODE'] = True - - self._cmake.definitions['DISABLE_STALL_NOTIF'] = False - self._cmake.definitions['FAIL_ON_WARNINGS'] = False - self._cmake.definitions['OPTDBG'] = True - self._cmake.definitions['WITH_TESTS'] = False - self._cmake.definitions['WITH_TOOLS'] = False - - self._cmake.definitions['WITH_GFLAGS'] = self.options.with_gflags - self._cmake.definitions['WITH_JEMALLOC'] = self.options.with_jemalloc - self._cmake.definitions['WITH_LZ4'] = self.options.with_lz4 - self._cmake.definitions['WITH_SNAPPY'] = self.options.with_snappy - self._cmake.definitions['WITH_TBB'] = self.options.with_tbb - self._cmake.definitions['WITH_ZLIB'] = self.options.with_zlib - self._cmake.definitions['WITH_ZSTD'] = self.options.with_zstd - - self._cmake.definitions['USE_RTTI'] = self.options.use_rtti - self._cmake.definitions['ROCKSDB_LITE'] = self.options.lite - self._cmake.definitions['ROCKSDB_INSTALL_ON_WINDOWS'] = ( - self.settings.os == 'Windows' - ) - - if not self.options.enable_sse: - self._cmake.definitions['PORTABLE'] = True - self._cmake.definitions['FORCE_SSE42'] = False - elif self.options.enable_sse == 'sse42': - self._cmake.definitions['PORTABLE'] = True - self._cmake.definitions['FORCE_SSE42'] = True - elif self.options.enable_sse == 'avx2': - self._cmake.definitions['PORTABLE'] = False - self._cmake.definitions['FORCE_SSE42'] = False - - self._cmake.definitions['WITH_ASAN'] = False - self._cmake.definitions['WITH_BZ2'] = False - self._cmake.definitions['WITH_JNI'] = False - self._cmake.definitions['WITH_LIBRADOS'] = False - if ms.is_msvc(self): - self._cmake.definitions['WITH_MD_LIBRARY'] = ( - ms.msvc_runtime_flag(self).startswith('MD') + minimum_version = self._compilers_minimum_version.get(str(self.settings.compiler), False) + if minimum_version and Version(self.settings.compiler.version) < minimum_version: + raise ConanInvalidConfiguration( + f"{self.ref} requires C++{self._min_cppstd}, which your compiler does not support." ) - self._cmake.definitions['WITH_RUNTIME_DEBUG'] = ( - ms.msvc_runtime_flag(self).endswith('d') - ) - self._cmake.definitions['WITH_NUMA'] = False - self._cmake.definitions['WITH_TSAN'] = False - self._cmake.definitions['WITH_UBSAN'] = False - self._cmake.definitions['WITH_WINDOWS_UTF8_FILENAMES'] = False - self._cmake.definitions['WITH_XPRESS'] = False - self._cmake.definitions['WITH_FALLOCATE'] = True + if self.settings.arch not in ["x86_64", "ppc64le", "ppc64", "mips64", "armv8"]: + raise ConanInvalidConfiguration("Rocksdb requires 64 bits") + + check_min_vs(self, "191") + + if self.version == "6.20.3" and \ + self.settings.os == "Linux" and \ + self.settings.compiler == "gcc" and \ + Version(self.settings.compiler.version) < "5": + raise ConanInvalidConfiguration("Rocksdb 6.20.3 is not compilable with gcc <5.") # See https://github.com/facebook/rocksdb/issues/3522 + + def source(self): + get(self, **self.conan_data["sources"][self.version], strip_root=True) + + def generate(self): + tc = CMakeToolchain(self) + tc.variables["FAIL_ON_WARNINGS"] = False + tc.variables["WITH_TESTS"] = False + tc.variables["WITH_TOOLS"] = False + tc.variables["WITH_CORE_TOOLS"] = False + tc.variables["WITH_BENCHMARK_TOOLS"] = False + tc.variables["WITH_FOLLY_DISTRIBUTED_MUTEX"] = False + if is_msvc(self): + tc.variables["WITH_MD_LIBRARY"] = not is_msvc_static_runtime(self) + tc.variables["ROCKSDB_INSTALL_ON_WINDOWS"] = self.settings.os == "Windows" + tc.variables["ROCKSDB_LITE"] = self.options.lite + tc.variables["WITH_GFLAGS"] = self.options.with_gflags + tc.variables["WITH_SNAPPY"] = self.options.with_snappy + tc.variables["WITH_LZ4"] = self.options.with_lz4 + tc.variables["WITH_ZLIB"] = self.options.with_zlib + tc.variables["WITH_ZSTD"] = self.options.with_zstd + tc.variables["WITH_TBB"] = self.options.get_safe("with_tbb", False) + tc.variables["WITH_JEMALLOC"] = self.options.with_jemalloc + tc.variables["ROCKSDB_BUILD_SHARED"] = self.options.shared + tc.variables["ROCKSDB_LIBRARY_EXPORTS"] = self.settings.os == "Windows" and self.options.shared + tc.variables["ROCKSDB_DLL" ] = self.settings.os == "Windows" and self.options.shared + tc.variables["USE_RTTI"] = self.options.use_rtti + if not bool(self.options.enable_sse): + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = False + elif self.options.enable_sse == "sse42": + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = True + elif self.options.enable_sse == "avx2": + tc.variables["PORTABLE"] = False + tc.variables["FORCE_SSE42"] = False + # not available yet in CCI + tc.variables["WITH_NUMA"] = False + tc.generate() + + deps = CMakeDeps(self) + if self.options.with_jemalloc: + deps.set_property("jemalloc", "cmake_file_name", "JeMalloc") + deps.set_property("jemalloc", "cmake_target_name", "JeMalloc::JeMalloc") + deps.generate() def build(self): - if ms.is_msvc(self): - file = os.path.join( - self.recipe_folder, '..', 'export_source', 'thirdparty.inc' - ) - shutil.copy(file, self.build_folder) - self._configure_cmake() - self._cmake.configure() - self._cmake.build() + apply_conandata_patches(self) + cmake = CMake(self) + cmake.configure() + cmake.build() + + def _remove_static_libraries(self): + rm(self, "rocksdb.lib", os.path.join(self.package_folder, "lib")) + for lib in glob.glob(os.path.join(self.package_folder, "lib", "*.a")): + if not lib.endswith(".dll.a"): + os.remove(lib) + + def _remove_cpp_headers(self): + for path in glob.glob(os.path.join(self.package_folder, "include", "rocksdb", "*")): + if path != os.path.join(self.package_folder, "include", "rocksdb", "c.h"): + if os.path.isfile(path): + os.remove(path) + else: + shutil.rmtree(path) def package(self): - self._configure_cmake() - self._cmake.install() + copy(self, "COPYING", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + copy(self, "LICENSE*", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + cmake = CMake(self) + cmake.install() + if self.options.shared: + self._remove_static_libraries() + self._remove_cpp_headers() # Force stable ABI for shared libraries + rmdir(self, os.path.join(self.package_folder, "lib", "cmake")) + rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig")) def package_info(self): - self.cpp_info.filenames['cmake_find_package'] = 'RocksDB' - self.cpp_info.filenames['cmake_find_package_multi'] = 'RocksDB' - self.cpp_info.set_property('cmake_file_name', 'RocksDB') - - self.cpp_info.names['cmake_find_package'] = 'RocksDB' - self.cpp_info.names['cmake_find_package_multi'] = 'RocksDB' - - self.cpp_info.components['librocksdb'].names['cmake_find_package'] = 'rocksdb' - self.cpp_info.components['librocksdb'].names['cmake_find_package_multi'] = 'rocksdb' - self.cpp_info.components['librocksdb'].set_property( - 'cmake_target_name', 'RocksDB::rocksdb' - ) - - self.cpp_info.components['librocksdb'].libs = ['rocksdb'] - + cmake_target = "rocksdb-shared" if self.options.shared else "rocksdb" + self.cpp_info.set_property("cmake_file_name", "RocksDB") + self.cpp_info.set_property("cmake_target_name", f"RocksDB::{cmake_target}") + # TODO: back to global scope in conan v2 once cmake_find_package* generators removed + self.cpp_info.components["librocksdb"].libs = collect_libs(self) if self.settings.os == "Windows": self.cpp_info.components["librocksdb"].system_libs = ["shlwapi", "rpcrt4"] if self.options.shared: self.cpp_info.components["librocksdb"].defines = ["ROCKSDB_DLL"] elif self.settings.os in ["Linux", "FreeBSD"]: self.cpp_info.components["librocksdb"].system_libs = ["pthread", "m"] - if self.options.lite: self.cpp_info.components["librocksdb"].defines.append("ROCKSDB_LITE") + # TODO: to remove in conan v2 once cmake_find_package* generators removed + self.cpp_info.names["cmake_find_package"] = "RocksDB" + self.cpp_info.names["cmake_find_package_multi"] = "RocksDB" + self.cpp_info.components["librocksdb"].names["cmake_find_package"] = cmake_target + self.cpp_info.components["librocksdb"].names["cmake_find_package_multi"] = cmake_target + self.cpp_info.components["librocksdb"].set_property("cmake_target_name", f"RocksDB::{cmake_target}") if self.options.with_gflags: self.cpp_info.components["librocksdb"].requires.append("gflags::gflags") - if self.options.with_jemalloc: - self.cpp_info.components["librocksdb"].requires.append("jemalloc::jemalloc") - if self.options.with_lz4: - self.cpp_info.components["librocksdb"].requires.append("lz4::lz4") if self.options.with_snappy: self.cpp_info.components["librocksdb"].requires.append("snappy::snappy") - if self.options.with_tbb: - self.cpp_info.components["librocksdb"].requires.append("onetbb::onetbb") + if self.options.with_lz4: + self.cpp_info.components["librocksdb"].requires.append("lz4::lz4") if self.options.with_zlib: self.cpp_info.components["librocksdb"].requires.append("zlib::zlib") if self.options.with_zstd: self.cpp_info.components["librocksdb"].requires.append("zstd::zstd") + if self.options.get_safe("with_tbb"): + self.cpp_info.components["librocksdb"].requires.append("onetbb::onetbb") + if self.options.with_jemalloc: + self.cpp_info.components["librocksdb"].requires.append("jemalloc::jemalloc") diff --git a/external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch b/external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch new file mode 100644 index 000000000..05725bf2c --- /dev/null +++ b/external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch @@ -0,0 +1,30 @@ +--- a/include/rocksdb/utilities/checkpoint.h ++++ b/include/rocksdb/utilities/checkpoint.h +@@ -8,6 +8,7 @@ + #pragma once + #ifndef ROCKSDB_LITE + ++#include + #include + #include + #include "rocksdb/status.h" +--- a/table/block_based/data_block_hash_index.h ++++ b/table/block_based/data_block_hash_index.h +@@ -5,6 +5,7 @@ + + #pragma once + ++#include + #include + #include + +--- a/util/string_util.h ++++ b/util/string_util.h +@@ -6,6 +6,7 @@ + + #pragma once + ++#include + #include + #include + #include diff --git a/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch b/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch new file mode 100644 index 000000000..fb0dd0c46 --- /dev/null +++ b/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch @@ -0,0 +1,16 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index ec59d4491..35577c998 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -101 +100,0 @@ if(MSVC) +- option(WITH_GFLAGS "build with GFlags" OFF) +@@ -103,2 +102,2 @@ if(MSVC) +- include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc) +-else() ++endif() ++ +@@ -117 +116 @@ else() +- if(MINGW) ++ if(MINGW OR MSVC) +@@ -183 +181,0 @@ else() +-endif()