mirror of
https://github.com/XRPLF/rippled.git
synced 2025-11-18 18:15:50 +00:00
Merge branch 'develop' into bthomee/proto
This commit is contained in:
Bart Thomee
3
.github/actions/dependencies/action.yml
vendored
3
.github/actions/dependencies/action.yml
vendored
@@ -11,7 +11,6 @@ runs:
|
||||
run: |
|
||||
conan export --version 6.30.1 external/protobuf
|
||||
conan export --version 1.1.10 external/snappy
|
||||
conan export --version 9.7.3 external/rocksdb
|
||||
conan export --version 4.0.3 external/soci
|
||||
- name: add Ripple Conan remote
|
||||
if: env.CONAN_URL != ''
|
||||
@@ -23,7 +22,6 @@ runs:
|
||||
fi
|
||||
conan remote add --index 0 ripple "${CONAN_URL}"
|
||||
echo "Added conan remote ripple at ${CONAN_URL}"
|
||||
|
||||
- name: try to authenticate to Ripple Conan remote
|
||||
if: env.CONAN_URL != '' && env.CONAN_LOGIN_USERNAME_RIPPLE != '' && env.CONAN_PASSWORD_RIPPLE != ''
|
||||
id: remote
|
||||
@@ -32,7 +30,6 @@ runs:
|
||||
echo "Authenticating to ripple remote..."
|
||||
conan remote auth ripple --force
|
||||
conan remote list-users
|
||||
|
||||
- name: list missing binaries
|
||||
id: binaries
|
||||
shell: bash
|
||||
|
||||
1
.github/workflows/macos.yml
vendored
1
.github/workflows/macos.yml
vendored
@@ -95,7 +95,6 @@ jobs:
|
||||
run: |
|
||||
conan export --version 6.30.1 external/protobuf
|
||||
conan export --version 1.1.10 external/snappy
|
||||
conan export --version 9.7.3 external/rocksdb
|
||||
conan export --version 4.0.3 external/soci
|
||||
- name: add Ripple Conan remote
|
||||
if: env.CONAN_URL != ''
|
||||
|
||||
1
.github/workflows/nix.yml
vendored
1
.github/workflows/nix.yml
vendored
@@ -99,7 +99,6 @@ jobs:
|
||||
run: tar -czf conan.tar.gz -C ${CONAN_HOME} .
|
||||
- name: build dependencies
|
||||
uses: ./.github/actions/dependencies
|
||||
|
||||
with:
|
||||
configuration: ${{ matrix.configuration }}
|
||||
- name: upload archive
|
||||
|
||||
1
.github/workflows/windows.yml
vendored
1
.github/workflows/windows.yml
vendored
@@ -90,7 +90,6 @@ jobs:
|
||||
run: |
|
||||
conan export --version 6.30.1 external/protobuf
|
||||
conan export --version 1.1.10 external/snappy
|
||||
conan export --version 9.7.3 external/rocksdb
|
||||
conan export --version 4.0.3 external/soci
|
||||
- name: add Ripple Conan remote
|
||||
if: env.CONAN_URL != ''
|
||||
|
||||
8
BUILD.md
8
BUILD.md
@@ -182,14 +182,6 @@ which allows you to statically link it with GCC, if you want.
|
||||
conan export --version 1.1.10 external/snappy
|
||||
```
|
||||
|
||||
Export our [Conan recipe for RocksDB](./external/rocksdb).
|
||||
It does not override paths to dependencies when building with Visual Studio.
|
||||
|
||||
```
|
||||
# Conan 2.x
|
||||
conan export --version 9.7.3 external/rocksdb
|
||||
```
|
||||
|
||||
Export our [Conan recipe for SOCI](./external/soci).
|
||||
It patches their CMake to correctly import its dependencies.
|
||||
|
||||
|
||||
@@ -121,7 +121,7 @@ class Xrpl(ConanFile):
|
||||
if self.options.jemalloc:
|
||||
self.requires('jemalloc/5.3.0')
|
||||
if self.options.rocksdb:
|
||||
self.requires('rocksdb/9.7.3')
|
||||
self.requires('rocksdb/10.0.1')
|
||||
self.requires('xxhash/0.8.3', **transitive_headers_opt)
|
||||
|
||||
exports_sources = (
|
||||
|
||||
12
external/rocksdb/conandata.yml
vendored
12
external/rocksdb/conandata.yml
vendored
@@ -1,12 +0,0 @@
|
||||
sources:
|
||||
"9.7.3":
|
||||
url: "https://github.com/facebook/rocksdb/archive/refs/tags/v9.7.3.tar.gz"
|
||||
sha256: "acfabb989cbfb5b5c4d23214819b059638193ec33dad2d88373c46448d16d38b"
|
||||
patches:
|
||||
"9.7.3":
|
||||
- patch_file: "patches/9.x.x-0001-exclude-thirdparty.patch"
|
||||
patch_description: "Do not include thirdparty.inc"
|
||||
patch_type: "portability"
|
||||
- patch_file: "patches/9.7.3-0001-memory-leak.patch"
|
||||
patch_description: "Fix a leak of obsolete blob files left open until DB::Close()"
|
||||
patch_type: "portability"
|
||||
235
external/rocksdb/conanfile.py
vendored
235
external/rocksdb/conanfile.py
vendored
@@ -1,235 +0,0 @@
|
||||
import os
|
||||
import glob
|
||||
import shutil
|
||||
|
||||
from conan import ConanFile
|
||||
from conan.errors import ConanInvalidConfiguration
|
||||
from conan.tools.build import check_min_cppstd
|
||||
from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout
|
||||
from conan.tools.files import apply_conandata_patches, collect_libs, copy, export_conandata_patches, get, rm, rmdir
|
||||
from conan.tools.microsoft import check_min_vs, is_msvc, is_msvc_static_runtime
|
||||
from conan.tools.scm import Version
|
||||
|
||||
required_conan_version = ">=1.53.0"
|
||||
|
||||
|
||||
class RocksDBConan(ConanFile):
|
||||
name = "rocksdb"
|
||||
description = "A library that provides an embeddable, persistent key-value store for fast storage"
|
||||
license = ("GPL-2.0-only", "Apache-2.0")
|
||||
url = "https://github.com/conan-io/conan-center-index"
|
||||
homepage = "https://github.com/facebook/rocksdb"
|
||||
topics = ("database", "leveldb", "facebook", "key-value")
|
||||
package_type = "library"
|
||||
settings = "os", "arch", "compiler", "build_type"
|
||||
options = {
|
||||
"shared": [True, False],
|
||||
"fPIC": [True, False],
|
||||
"lite": [True, False],
|
||||
"with_gflags": [True, False],
|
||||
"with_snappy": [True, False],
|
||||
"with_lz4": [True, False],
|
||||
"with_zlib": [True, False],
|
||||
"with_zstd": [True, False],
|
||||
"with_tbb": [True, False],
|
||||
"with_jemalloc": [True, False],
|
||||
"enable_sse": [False, "sse42", "avx2"],
|
||||
"use_rtti": [True, False],
|
||||
}
|
||||
default_options = {
|
||||
"shared": False,
|
||||
"fPIC": True,
|
||||
"lite": False,
|
||||
"with_snappy": False,
|
||||
"with_lz4": False,
|
||||
"with_zlib": False,
|
||||
"with_zstd": False,
|
||||
"with_gflags": False,
|
||||
"with_tbb": False,
|
||||
"with_jemalloc": False,
|
||||
"enable_sse": False,
|
||||
"use_rtti": False,
|
||||
}
|
||||
|
||||
@property
|
||||
def _min_cppstd(self):
|
||||
return "11" if Version(self.version) < "8.8.1" else "17"
|
||||
|
||||
@property
|
||||
def _compilers_minimum_version(self):
|
||||
return {} if self._min_cppstd == "11" else {
|
||||
"apple-clang": "10",
|
||||
"clang": "7",
|
||||
"gcc": "7",
|
||||
"msvc": "191",
|
||||
"Visual Studio": "15",
|
||||
}
|
||||
|
||||
def export_sources(self):
|
||||
export_conandata_patches(self)
|
||||
|
||||
def config_options(self):
|
||||
if self.settings.os == "Windows":
|
||||
del self.options.fPIC
|
||||
if self.settings.arch != "x86_64":
|
||||
del self.options.with_tbb
|
||||
if self.settings.build_type == "Debug":
|
||||
self.options.use_rtti = True # Rtti are used in asserts for debug mode...
|
||||
|
||||
def configure(self):
|
||||
if self.options.shared:
|
||||
self.options.rm_safe("fPIC")
|
||||
|
||||
def layout(self):
|
||||
cmake_layout(self, src_folder="src")
|
||||
|
||||
def requirements(self):
|
||||
if self.options.with_gflags:
|
||||
self.requires("gflags/2.2.2")
|
||||
if self.options.with_snappy:
|
||||
self.requires("snappy/1.1.10")
|
||||
if self.options.with_lz4:
|
||||
self.requires("lz4/1.10.0")
|
||||
if self.options.with_zlib:
|
||||
self.requires("zlib/[>=1.2.11 <2]")
|
||||
if self.options.with_zstd:
|
||||
self.requires("zstd/1.5.6")
|
||||
if self.options.get_safe("with_tbb"):
|
||||
self.requires("onetbb/2021.12.0")
|
||||
if self.options.with_jemalloc:
|
||||
self.requires("jemalloc/5.3.0")
|
||||
|
||||
def validate(self):
|
||||
if self.settings.compiler.get_safe("cppstd"):
|
||||
check_min_cppstd(self, self._min_cppstd)
|
||||
|
||||
minimum_version = self._compilers_minimum_version.get(str(self.settings.compiler), False)
|
||||
if minimum_version and Version(self.settings.compiler.version) < minimum_version:
|
||||
raise ConanInvalidConfiguration(
|
||||
f"{self.ref} requires C++{self._min_cppstd}, which your compiler does not support."
|
||||
)
|
||||
|
||||
if self.settings.arch not in ["x86_64", "ppc64le", "ppc64", "mips64", "armv8"]:
|
||||
raise ConanInvalidConfiguration("Rocksdb requires 64 bits")
|
||||
|
||||
check_min_vs(self, "191")
|
||||
|
||||
if self.version == "6.20.3" and \
|
||||
self.settings.os == "Linux" and \
|
||||
self.settings.compiler == "gcc" and \
|
||||
Version(self.settings.compiler.version) < "5":
|
||||
raise ConanInvalidConfiguration("Rocksdb 6.20.3 is not compilable with gcc <5.") # See https://github.com/facebook/rocksdb/issues/3522
|
||||
|
||||
def source(self):
|
||||
get(self, **self.conan_data["sources"][self.version], strip_root=True)
|
||||
|
||||
def generate(self):
|
||||
tc = CMakeToolchain(self)
|
||||
tc.variables["FAIL_ON_WARNINGS"] = False
|
||||
tc.variables["WITH_TESTS"] = False
|
||||
tc.variables["WITH_TOOLS"] = False
|
||||
tc.variables["WITH_CORE_TOOLS"] = False
|
||||
tc.variables["WITH_BENCHMARK_TOOLS"] = False
|
||||
tc.variables["WITH_FOLLY_DISTRIBUTED_MUTEX"] = False
|
||||
if is_msvc(self):
|
||||
tc.variables["WITH_MD_LIBRARY"] = not is_msvc_static_runtime(self)
|
||||
tc.variables["ROCKSDB_INSTALL_ON_WINDOWS"] = self.settings.os == "Windows"
|
||||
tc.variables["ROCKSDB_LITE"] = self.options.lite
|
||||
tc.variables["WITH_GFLAGS"] = self.options.with_gflags
|
||||
tc.variables["WITH_SNAPPY"] = self.options.with_snappy
|
||||
tc.variables["WITH_LZ4"] = self.options.with_lz4
|
||||
tc.variables["WITH_ZLIB"] = self.options.with_zlib
|
||||
tc.variables["WITH_ZSTD"] = self.options.with_zstd
|
||||
tc.variables["WITH_TBB"] = self.options.get_safe("with_tbb", False)
|
||||
tc.variables["WITH_JEMALLOC"] = self.options.with_jemalloc
|
||||
tc.variables["ROCKSDB_BUILD_SHARED"] = self.options.shared
|
||||
tc.variables["ROCKSDB_LIBRARY_EXPORTS"] = self.settings.os == "Windows" and self.options.shared
|
||||
tc.variables["ROCKSDB_DLL" ] = self.settings.os == "Windows" and self.options.shared
|
||||
tc.variables["USE_RTTI"] = self.options.use_rtti
|
||||
if not bool(self.options.enable_sse):
|
||||
tc.variables["PORTABLE"] = True
|
||||
tc.variables["FORCE_SSE42"] = False
|
||||
elif self.options.enable_sse == "sse42":
|
||||
tc.variables["PORTABLE"] = True
|
||||
tc.variables["FORCE_SSE42"] = True
|
||||
elif self.options.enable_sse == "avx2":
|
||||
tc.variables["PORTABLE"] = False
|
||||
tc.variables["FORCE_SSE42"] = False
|
||||
# not available yet in CCI
|
||||
tc.variables["WITH_NUMA"] = False
|
||||
tc.generate()
|
||||
|
||||
deps = CMakeDeps(self)
|
||||
if self.options.with_jemalloc:
|
||||
deps.set_property("jemalloc", "cmake_file_name", "JeMalloc")
|
||||
deps.set_property("jemalloc", "cmake_target_name", "JeMalloc::JeMalloc")
|
||||
if self.options.with_zstd:
|
||||
deps.set_property("zstd", "cmake_target_name", "zstd::zstd")
|
||||
deps.generate()
|
||||
|
||||
def build(self):
|
||||
apply_conandata_patches(self)
|
||||
cmake = CMake(self)
|
||||
cmake.configure()
|
||||
cmake.build()
|
||||
|
||||
def _remove_static_libraries(self):
|
||||
rm(self, "rocksdb.lib", os.path.join(self.package_folder, "lib"))
|
||||
for lib in glob.glob(os.path.join(self.package_folder, "lib", "*.a")):
|
||||
if not lib.endswith(".dll.a"):
|
||||
os.remove(lib)
|
||||
|
||||
def _remove_cpp_headers(self):
|
||||
for path in glob.glob(os.path.join(self.package_folder, "include", "rocksdb", "*")):
|
||||
if path != os.path.join(self.package_folder, "include", "rocksdb", "c.h"):
|
||||
if os.path.isfile(path):
|
||||
os.remove(path)
|
||||
else:
|
||||
shutil.rmtree(path)
|
||||
|
||||
def package(self):
|
||||
copy(self, "COPYING", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses"))
|
||||
copy(self, "LICENSE*", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses"))
|
||||
cmake = CMake(self)
|
||||
cmake.install()
|
||||
if self.options.shared:
|
||||
self._remove_static_libraries()
|
||||
self._remove_cpp_headers() # Force stable ABI for shared libraries
|
||||
rmdir(self, os.path.join(self.package_folder, "lib", "cmake"))
|
||||
rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig"))
|
||||
|
||||
def package_info(self):
|
||||
cmake_target = "rocksdb-shared" if self.options.shared else "rocksdb"
|
||||
self.cpp_info.set_property("cmake_file_name", "RocksDB")
|
||||
self.cpp_info.set_property("cmake_target_name", f"RocksDB::{cmake_target}")
|
||||
# TODO: back to global scope in conan v2 once cmake_find_package* generators removed
|
||||
self.cpp_info.components["librocksdb"].libs = collect_libs(self)
|
||||
if self.settings.os == "Windows":
|
||||
self.cpp_info.components["librocksdb"].system_libs = ["shlwapi", "rpcrt4"]
|
||||
if self.options.shared:
|
||||
self.cpp_info.components["librocksdb"].defines = ["ROCKSDB_DLL"]
|
||||
elif self.settings.os in ["Linux", "FreeBSD"]:
|
||||
self.cpp_info.components["librocksdb"].system_libs = ["pthread", "m"]
|
||||
if self.options.lite:
|
||||
self.cpp_info.components["librocksdb"].defines.append("ROCKSDB_LITE")
|
||||
|
||||
# TODO: to remove in conan v2 once cmake_find_package* generators removed
|
||||
self.cpp_info.names["cmake_find_package"] = "RocksDB"
|
||||
self.cpp_info.names["cmake_find_package_multi"] = "RocksDB"
|
||||
self.cpp_info.components["librocksdb"].names["cmake_find_package"] = cmake_target
|
||||
self.cpp_info.components["librocksdb"].names["cmake_find_package_multi"] = cmake_target
|
||||
self.cpp_info.components["librocksdb"].set_property("cmake_target_name", f"RocksDB::{cmake_target}")
|
||||
if self.options.with_gflags:
|
||||
self.cpp_info.components["librocksdb"].requires.append("gflags::gflags")
|
||||
if self.options.with_snappy:
|
||||
self.cpp_info.components["librocksdb"].requires.append("snappy::snappy")
|
||||
if self.options.with_lz4:
|
||||
self.cpp_info.components["librocksdb"].requires.append("lz4::lz4")
|
||||
if self.options.with_zlib:
|
||||
self.cpp_info.components["librocksdb"].requires.append("zlib::zlib")
|
||||
if self.options.with_zstd:
|
||||
self.cpp_info.components["librocksdb"].requires.append("zstd::zstd")
|
||||
if self.options.get_safe("with_tbb"):
|
||||
self.cpp_info.components["librocksdb"].requires.append("onetbb::onetbb")
|
||||
if self.options.with_jemalloc:
|
||||
self.cpp_info.components["librocksdb"].requires.append("jemalloc::jemalloc")
|
||||
@@ -1,319 +0,0 @@
|
||||
diff --git a/HISTORY.md b/HISTORY.md
|
||||
index 36d472229..05ad1a202 100644
|
||||
--- a/HISTORY.md
|
||||
+++ b/HISTORY.md
|
||||
@@ -1,6 +1,10 @@
|
||||
# Rocksdb Change Log
|
||||
> NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`
|
||||
|
||||
+## 9.7.4 (10/31/2024)
|
||||
+### Bug Fixes
|
||||
+* Fix a leak of obsolete blob files left open until DB::Close(). This bug was introduced in version 9.4.0.
|
||||
+
|
||||
## 9.7.3 (10/16/2024)
|
||||
### Behavior Changes
|
||||
* OPTIONS file to be loaded by remote worker is now preserved so that it does not get purged by the primary host. A similar technique as how we are preserving new SST files from getting purged is used for this. min_options_file_numbers_ is tracked like pending_outputs_ is tracked.
|
||||
diff --git a/db/blob/blob_file_cache.cc b/db/blob/blob_file_cache.cc
|
||||
index 5f340aadf..1b9faa238 100644
|
||||
--- a/db/blob/blob_file_cache.cc
|
||||
+++ b/db/blob/blob_file_cache.cc
|
||||
@@ -42,6 +42,7 @@ Status BlobFileCache::GetBlobFileReader(
|
||||
assert(blob_file_reader);
|
||||
assert(blob_file_reader->IsEmpty());
|
||||
|
||||
+ // NOTE: sharing same Cache with table_cache
|
||||
const Slice key = GetSliceForKey(&blob_file_number);
|
||||
|
||||
assert(cache_);
|
||||
@@ -98,4 +99,13 @@ Status BlobFileCache::GetBlobFileReader(
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
+void BlobFileCache::Evict(uint64_t blob_file_number) {
|
||||
+ // NOTE: sharing same Cache with table_cache
|
||||
+ const Slice key = GetSliceForKey(&blob_file_number);
|
||||
+
|
||||
+ assert(cache_);
|
||||
+
|
||||
+ cache_.get()->Erase(key);
|
||||
+}
|
||||
+
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
diff --git a/db/blob/blob_file_cache.h b/db/blob/blob_file_cache.h
|
||||
index 740e67ada..6858d012b 100644
|
||||
--- a/db/blob/blob_file_cache.h
|
||||
+++ b/db/blob/blob_file_cache.h
|
||||
@@ -36,6 +36,15 @@ class BlobFileCache {
|
||||
uint64_t blob_file_number,
|
||||
CacheHandleGuard<BlobFileReader>* blob_file_reader);
|
||||
|
||||
+ // Called when a blob file is obsolete to ensure it is removed from the cache
|
||||
+ // to avoid effectively leaking the open file and assicated memory
|
||||
+ void Evict(uint64_t blob_file_number);
|
||||
+
|
||||
+ // Used to identify cache entries for blob files (not normally useful)
|
||||
+ static const Cache::CacheItemHelper* GetHelper() {
|
||||
+ return CacheInterface::GetBasicHelper();
|
||||
+ }
|
||||
+
|
||||
private:
|
||||
using CacheInterface =
|
||||
BasicTypedCacheInterface<BlobFileReader, CacheEntryRole::kMisc>;
|
||||
diff --git a/db/column_family.h b/db/column_family.h
|
||||
index e4b7adde8..86637736a 100644
|
||||
--- a/db/column_family.h
|
||||
+++ b/db/column_family.h
|
||||
@@ -401,6 +401,7 @@ class ColumnFamilyData {
|
||||
SequenceNumber earliest_seq);
|
||||
|
||||
TableCache* table_cache() const { return table_cache_.get(); }
|
||||
+ BlobFileCache* blob_file_cache() const { return blob_file_cache_.get(); }
|
||||
BlobSource* blob_source() const { return blob_source_.get(); }
|
||||
|
||||
// See documentation in compaction_picker.h
|
||||
diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc
|
||||
index 261593423..06573ac2e 100644
|
||||
--- a/db/db_impl/db_impl.cc
|
||||
+++ b/db/db_impl/db_impl.cc
|
||||
@@ -659,8 +659,9 @@ Status DBImpl::CloseHelper() {
|
||||
// We need to release them before the block cache is destroyed. The block
|
||||
// cache may be destroyed inside versions_.reset(), when column family data
|
||||
// list is destroyed, so leaving handles in table cache after
|
||||
- // versions_.reset() may cause issues.
|
||||
- // Here we clean all unreferenced handles in table cache.
|
||||
+ // versions_.reset() may cause issues. Here we clean all unreferenced handles
|
||||
+ // in table cache, and (for certain builds/conditions) assert that no obsolete
|
||||
+ // files are hanging around unreferenced (leak) in the table/blob file cache.
|
||||
// Now we assume all user queries have finished, so only version set itself
|
||||
// can possibly hold the blocks from block cache. After releasing unreferenced
|
||||
// handles here, only handles held by version set left and inside
|
||||
@@ -668,6 +669,9 @@ Status DBImpl::CloseHelper() {
|
||||
// time a handle is released, we erase it from the cache too. By doing that,
|
||||
// we can guarantee that after versions_.reset(), table cache is empty
|
||||
// so the cache can be safely destroyed.
|
||||
+#ifndef NDEBUG
|
||||
+ TEST_VerifyNoObsoleteFilesCached(/*db_mutex_already_held=*/true);
|
||||
+#endif // !NDEBUG
|
||||
table_cache_->EraseUnRefEntries();
|
||||
|
||||
for (auto& txn_entry : recovered_transactions_) {
|
||||
@@ -3227,6 +3231,8 @@ Status DBImpl::MultiGetImpl(
|
||||
s = Status::Aborted();
|
||||
break;
|
||||
}
|
||||
+ // This could be a long-running operation
|
||||
+ ROCKSDB_THREAD_YIELD_HOOK();
|
||||
}
|
||||
|
||||
// Post processing (decrement reference counts and record statistics)
|
||||
diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h
|
||||
index 5e4fa310b..ccc0abfa7 100644
|
||||
--- a/db/db_impl/db_impl.h
|
||||
+++ b/db/db_impl/db_impl.h
|
||||
@@ -1241,9 +1241,14 @@ class DBImpl : public DB {
|
||||
static Status TEST_ValidateOptions(const DBOptions& db_options) {
|
||||
return ValidateOptions(db_options);
|
||||
}
|
||||
-
|
||||
#endif // NDEBUG
|
||||
|
||||
+ // In certain configurations, verify that the table/blob file cache only
|
||||
+ // contains entries for live files, to check for effective leaks of open
|
||||
+ // files. This can only be called when purging of obsolete files has
|
||||
+ // "settled," such as during parts of DB Close().
|
||||
+ void TEST_VerifyNoObsoleteFilesCached(bool db_mutex_already_held) const;
|
||||
+
|
||||
// persist stats to column family "_persistent_stats"
|
||||
void PersistStats();
|
||||
|
||||
diff --git a/db/db_impl/db_impl_debug.cc b/db/db_impl/db_impl_debug.cc
|
||||
index 790a50d7a..67f5b4aaf 100644
|
||||
--- a/db/db_impl/db_impl_debug.cc
|
||||
+++ b/db/db_impl/db_impl_debug.cc
|
||||
@@ -9,6 +9,7 @@
|
||||
|
||||
#ifndef NDEBUG
|
||||
|
||||
+#include "db/blob/blob_file_cache.h"
|
||||
#include "db/column_family.h"
|
||||
#include "db/db_impl/db_impl.h"
|
||||
#include "db/error_handler.h"
|
||||
@@ -328,5 +329,49 @@ size_t DBImpl::TEST_EstimateInMemoryStatsHistorySize() const {
|
||||
InstrumentedMutexLock l(&const_cast<DBImpl*>(this)->stats_history_mutex_);
|
||||
return EstimateInMemoryStatsHistorySize();
|
||||
}
|
||||
+
|
||||
+void DBImpl::TEST_VerifyNoObsoleteFilesCached(
|
||||
+ bool db_mutex_already_held) const {
|
||||
+ // This check is somewhat expensive and obscure to make a part of every
|
||||
+ // unit test in every build variety. Thus, we only enable it for ASAN builds.
|
||||
+ if (!kMustFreeHeapAllocations) {
|
||||
+ return;
|
||||
+ }
|
||||
+
|
||||
+ std::optional<InstrumentedMutexLock> l;
|
||||
+ if (db_mutex_already_held) {
|
||||
+ mutex_.AssertHeld();
|
||||
+ } else {
|
||||
+ l.emplace(&mutex_);
|
||||
+ }
|
||||
+
|
||||
+ std::vector<uint64_t> live_files;
|
||||
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
|
||||
+ if (cfd->IsDropped()) {
|
||||
+ continue;
|
||||
+ }
|
||||
+ // Sneakily add both SST and blob files to the same list
|
||||
+ cfd->current()->AddLiveFiles(&live_files, &live_files);
|
||||
+ }
|
||||
+ std::sort(live_files.begin(), live_files.end());
|
||||
+
|
||||
+ auto fn = [&live_files](const Slice& key, Cache::ObjectPtr, size_t,
|
||||
+ const Cache::CacheItemHelper* helper) {
|
||||
+ if (helper != BlobFileCache::GetHelper()) {
|
||||
+ // Skip non-blob files for now
|
||||
+ // FIXME: diagnose and fix the leaks of obsolete SST files revealed in
|
||||
+ // unit tests.
|
||||
+ return;
|
||||
+ }
|
||||
+ // See TableCache and BlobFileCache
|
||||
+ assert(key.size() == sizeof(uint64_t));
|
||||
+ uint64_t file_number;
|
||||
+ GetUnaligned(reinterpret_cast<const uint64_t*>(key.data()), &file_number);
|
||||
+ // Assert file is in sorted live_files
|
||||
+ assert(
|
||||
+ std::binary_search(live_files.begin(), live_files.end(), file_number));
|
||||
+ };
|
||||
+ table_cache_->ApplyToAllEntries(fn, {});
|
||||
+}
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
#endif // NDEBUG
|
||||
diff --git a/db/db_iter.cc b/db/db_iter.cc
|
||||
index e02586377..bf4749eb9 100644
|
||||
--- a/db/db_iter.cc
|
||||
+++ b/db/db_iter.cc
|
||||
@@ -540,6 +540,8 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
|
||||
} else {
|
||||
iter_.Next();
|
||||
}
|
||||
+ // This could be a long-running operation due to tombstones, etc.
|
||||
+ ROCKSDB_THREAD_YIELD_HOOK();
|
||||
} while (iter_.Valid());
|
||||
|
||||
valid_ = false;
|
||||
diff --git a/db/table_cache.cc b/db/table_cache.cc
|
||||
index 71fc29c32..8a5be75e8 100644
|
||||
--- a/db/table_cache.cc
|
||||
+++ b/db/table_cache.cc
|
||||
@@ -164,6 +164,7 @@ Status TableCache::GetTableReader(
|
||||
}
|
||||
|
||||
Cache::Handle* TableCache::Lookup(Cache* cache, uint64_t file_number) {
|
||||
+ // NOTE: sharing same Cache with BlobFileCache
|
||||
Slice key = GetSliceForFileNumber(&file_number);
|
||||
return cache->Lookup(key);
|
||||
}
|
||||
@@ -179,6 +180,7 @@ Status TableCache::FindTable(
|
||||
size_t max_file_size_for_l0_meta_pin, Temperature file_temperature) {
|
||||
PERF_TIMER_GUARD_WITH_CLOCK(find_table_nanos, ioptions_.clock);
|
||||
uint64_t number = file_meta.fd.GetNumber();
|
||||
+ // NOTE: sharing same Cache with BlobFileCache
|
||||
Slice key = GetSliceForFileNumber(&number);
|
||||
*handle = cache_.Lookup(key);
|
||||
TEST_SYNC_POINT_CALLBACK("TableCache::FindTable:0",
|
||||
diff --git a/db/version_builder.cc b/db/version_builder.cc
|
||||
index ed8ab8214..c98f53f42 100644
|
||||
--- a/db/version_builder.cc
|
||||
+++ b/db/version_builder.cc
|
||||
@@ -24,6 +24,7 @@
|
||||
#include <vector>
|
||||
|
||||
#include "cache/cache_reservation_manager.h"
|
||||
+#include "db/blob/blob_file_cache.h"
|
||||
#include "db/blob/blob_file_meta.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/internal_stats.h"
|
||||
@@ -744,12 +745,9 @@ class VersionBuilder::Rep {
|
||||
return Status::Corruption("VersionBuilder", oss.str());
|
||||
}
|
||||
|
||||
- // Note: we use C++11 for now but in C++14, this could be done in a more
|
||||
- // elegant way using generalized lambda capture.
|
||||
- VersionSet* const vs = version_set_;
|
||||
- const ImmutableCFOptions* const ioptions = ioptions_;
|
||||
-
|
||||
- auto deleter = [vs, ioptions](SharedBlobFileMetaData* shared_meta) {
|
||||
+ auto deleter = [vs = version_set_, ioptions = ioptions_,
|
||||
+ bc = cfd_ ? cfd_->blob_file_cache()
|
||||
+ : nullptr](SharedBlobFileMetaData* shared_meta) {
|
||||
if (vs) {
|
||||
assert(ioptions);
|
||||
assert(!ioptions->cf_paths.empty());
|
||||
@@ -758,6 +756,9 @@ class VersionBuilder::Rep {
|
||||
vs->AddObsoleteBlobFile(shared_meta->GetBlobFileNumber(),
|
||||
ioptions->cf_paths.front().path);
|
||||
}
|
||||
+ if (bc) {
|
||||
+ bc->Evict(shared_meta->GetBlobFileNumber());
|
||||
+ }
|
||||
|
||||
delete shared_meta;
|
||||
};
|
||||
@@ -766,7 +767,7 @@ class VersionBuilder::Rep {
|
||||
blob_file_number, blob_file_addition.GetTotalBlobCount(),
|
||||
blob_file_addition.GetTotalBlobBytes(),
|
||||
blob_file_addition.GetChecksumMethod(),
|
||||
- blob_file_addition.GetChecksumValue(), deleter);
|
||||
+ blob_file_addition.GetChecksumValue(), std::move(deleter));
|
||||
|
||||
mutable_blob_file_metas_.emplace(
|
||||
blob_file_number, MutableBlobFileMetaData(std::move(shared_meta)));
|
||||
diff --git a/db/version_set.h b/db/version_set.h
|
||||
index 9336782b1..024f869e7 100644
|
||||
--- a/db/version_set.h
|
||||
+++ b/db/version_set.h
|
||||
@@ -1514,7 +1514,6 @@ class VersionSet {
|
||||
void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
|
||||
|
||||
void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) {
|
||||
- // TODO: Erase file from BlobFileCache?
|
||||
obsolete_blob_files_.emplace_back(blob_file_number, std::move(path));
|
||||
}
|
||||
|
||||
diff --git a/include/rocksdb/version.h b/include/rocksdb/version.h
|
||||
index 2a19796b8..0afa2cab1 100644
|
||||
--- a/include/rocksdb/version.h
|
||||
+++ b/include/rocksdb/version.h
|
||||
@@ -13,7 +13,7 @@
|
||||
// minor or major version number planned for release.
|
||||
#define ROCKSDB_MAJOR 9
|
||||
#define ROCKSDB_MINOR 7
|
||||
-#define ROCKSDB_PATCH 3
|
||||
+#define ROCKSDB_PATCH 4
|
||||
|
||||
// Do not use these. We made the mistake of declaring macros starting with
|
||||
// double underscore. Now we have to live with our choice. We'll deprecate these
|
||||
diff --git a/port/port.h b/port/port.h
|
||||
index 13aa56d47..141716e5b 100644
|
||||
--- a/port/port.h
|
||||
+++ b/port/port.h
|
||||
@@ -19,3 +19,19 @@
|
||||
#elif defined(OS_WIN)
|
||||
#include "port/win/port_win.h"
|
||||
#endif
|
||||
+
|
||||
+#ifdef OS_LINUX
|
||||
+// A temporary hook into long-running RocksDB threads to support modifying their
|
||||
+// priority etc. This should become a public API hook once the requirements
|
||||
+// are better understood.
|
||||
+extern "C" void RocksDbThreadYield() __attribute__((__weak__));
|
||||
+#define ROCKSDB_THREAD_YIELD_HOOK() \
|
||||
+ { \
|
||||
+ if (RocksDbThreadYield) { \
|
||||
+ RocksDbThreadYield(); \
|
||||
+ } \
|
||||
+ }
|
||||
+#else
|
||||
+#define ROCKSDB_THREAD_YIELD_HOOK() \
|
||||
+ {}
|
||||
+#endif
|
||||
@@ -1,30 +0,0 @@
|
||||
diff --git a/CMakeLists.txt b/CMakeLists.txt
|
||||
index 93b884d..b715cb6 100644
|
||||
--- a/CMakeLists.txt
|
||||
+++ b/CMakeLists.txt
|
||||
@@ -106,14 +106,9 @@ endif()
|
||||
include(CMakeDependentOption)
|
||||
|
||||
if(MSVC)
|
||||
- option(WITH_GFLAGS "build with GFlags" OFF)
|
||||
option(WITH_XPRESS "build with windows built in compression" OFF)
|
||||
- option(ROCKSDB_SKIP_THIRDPARTY "skip thirdparty.inc" OFF)
|
||||
-
|
||||
- if(NOT ROCKSDB_SKIP_THIRDPARTY)
|
||||
- include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
|
||||
- endif()
|
||||
-else()
|
||||
+endif()
|
||||
+if(TRUE)
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
|
||||
# FreeBSD has jemalloc as default malloc
|
||||
# but it does not have all the jemalloc files in include/...
|
||||
@@ -126,7 +121,7 @@ else()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
- if(MINGW)
|
||||
+ if(MSVC OR MINGW)
|
||||
option(WITH_GFLAGS "build with GFlags" OFF)
|
||||
else()
|
||||
option(WITH_GFLAGS "build with GFlags" ON)
|
||||
Reference in New Issue
Block a user