mirror of
https://github.com/Xahau/xahaud.git
synced 2025-12-06 17:27:52 +00:00
25888ae Merge pull request #329 from fyrz/master 89833e5 Fixed signed-unsigned comparison warning in db_test.cc fcac705 Fixed compile warning on Mac caused by unused variables. b3343fd resolution for java build problem introduced by 5ec53f3edf62bec1b690ce12fb21a6c52203f3c8 187b299 ForwardIterator: update prev_key_ only if prefix hasn't changed 5ec53f3 make compaction related options changeable d122e7b Update INSTALL.md 986dad0 Merge pull request #324 from dalgaaf/wip-da-SCA-20140930 8ee75dc db/memtable.cc: remove unused variable merge_result 0fd8bbc db/db_impl.cc: reduce scope of prefix_initialized 676ff7b compaction_picker.cc: remove check for >=0 for unsigned e55aea5 document_db.cc: fix assert d517c83 in_table_factory.cc: use correct format specifier b140375 ttl/ttl_test.cc: prefer prefix ++operator for non-primitive types 43c789c spatialdb/spatial_db.cc: use !empty() instead of 'size() > 0' 0de452e document_db.cc: pass const parameter by reference 4cc8643 util/ldb_cmd.cc: prefer prefix ++operator for non-primitive types af8c2b2 util/signal_test.cc: suppress intentional null pointer deref 33580fa db/db_impl.cc: fix object handling, remove double lines 873f135 db_ttl_impl.h: pass func parameter by reference 8558457 ldb_cmd_execute_result.h: perform init in initialization list 063471b table/table_test.cc: pass func parameter by reference 93548ce table/cuckoo_table_reader.cc: pass func parameter by ref b8b7117 db/version_set.cc: use !empty() instead of 'size() > 0' 8ce050b table/bloom_block.*: pass func parameter by reference 53910dd db_test.cc: pass parameter by reference 68ca534 corruption_test.cc: pass parameter by reference 7506198 cuckoo_table_db_test.cc: add flush after delete 1f96330 Print MB per second compaction throughput separately for reads and writes ffe3d49 Add an instruction about SSE in INSTALL.md ee1f3cc Package generation for Ubuntu and CentOS f0f7955 Fixing comile errors on OS X 99fb613 remove 2 space linter b2d64a4 Fix linters, second try 747523d Print per column family metrics in db_bench 56ebd40 Fix arc lint (should fix #238) 637f891 Merge pull request #321 from eonnen/master 827e31c Make test use a compatible type in the size checks. fd5d80d CompactedDB: log using the correct info_log 2faf49d use GetContext to replace callback function pointer 983d2de Add AUTHORS file. Fix #203 abd70c5 Merge pull request #316 from fyrz/ReverseBytewiseComparator 2dc6f62 handle kDelete type in cuckoo builder 8b8011a Changed name of ReverseBytewiseComparator based on review comment 389edb6 universal compaction picker: use double for potential overflow 5340484 Built-in comparator(s) in RocksJava d439451 delay initialization of cuckoo table iterator 94997ea reduce memory usage of cuckoo table builder c627595 improve memory efficiency of cuckoo reader 581442d option to choose module when calculating CuckooTable hash fbd2daf CompactedDBImpl::MultiGet() for better CuckooTable performance 3c68006 CompactedDBImpl f7375f3 Fix double deletes 21ddcf6 Remove allow_thread_local fb4a492 Merge pull request #311 from ankgup87/master 611e286 Merge branch 'master' of https://github.com/facebook/rocksdb 0103b44 Merge branch 'master' of ssh://github.com/ankgup87/rocksdb 1dfb7bb Add block based table config options cdaf44f Enlarge log size cap when printing file summary 7cc1ed7 Merge pull request #309 from naveenatceg/staticbuild ba6d660 Resolving merge conflict 51eeaf6 Addressing review comments fd7d3fe Addressing review comments (adding a env variable to override temp directory) cf7ace8 Addressing review comments 0a29ce5 re-enable BlockBasedTable::SetupForCompaction() 55af370 Remove TODO for checking index checksums 3d74f09 Fix compile 53b0039 Fix release compile d0de413 WriteBatchWithIndex to allow different Comparators for different column families 57a32f1 change target_file_size_base to uint64_t 5e6aee4 dont create backup_input if compaction filter v2 is not used 49b5f94 Merge pull request #306 from Liuchang0812/fix_cast 787cb4d remove cast, replace %llu with % PRIu64 a7574d4 Update logging.cc 7e0dcb9 Update logging.cc 57fa3cc Merge pull request #304 from Liuchang0812/fix-check cd44522 Merge pull request #305 from Liuchang0812/fix-logging 6a031b6 remove unused variable 4436f17 fixed #303: replace %ld with % PRId64 7a1bd05 Merge pull request #302 from ankgup87/master 423e52c Merge branch 'master' of https://github.com/facebook/rocksdb bfeef94 Add rate limiter 32f2532 Print compression_size_percent as a signed int 976caca Skip AllocateTest if fallocate() is not supported in the file system 3b897cd Enable no-fbcode RocksDB build f445947 RocksDB: Format uint64 using PRIu64 in db_impl.cc e17bc65 Merge pull request #299 from ankgup87/master b93797a Fix build adae3ca [Java] Fix JNI link error caused by the removal of options.db_stats_log_interval 90b8c07 Fix unit tests errors 51af7c3 CuckooTable: add one option to allow identity function for the first hash function 0350435 Fixed a signed-unsigned comparison in spatial_db.cc -- issue #293 2fb1fea Fix syncronization issues ff76895 Remove some unnecessary constructors feadb9d fix cuckoo table builder test 3c232e1 Fix mac compile 54cada9 Run make format on PR #249 27b22f1 Merge pull request #249 from tdfischer/decompression-refactoring fb6456b Replace naked calls to operator new and delete (Fixes #222) 5600c8f cuckoo table: return estimated size - 1 a062e1f SetOptions() for memtable related options e4eca6a Options conversion function for convenience a7c2094 Merge pull request #292 from saghmrossi/master 4d05234 Merge branch 'master' of github.com:saghmrossi/rocksdb 60a4aa1 Test use_mmap_reads 94e43a1 [Java] Fixed 32-bit overflowing issue when converting jlong to size_t f9eaaa6 added include for inttypes.h to fix nonworking printf statements f090575 Replaced "built on on earlier work" by "built on earlier work" in README.md faad439 Fix #284 49aacd8 Fix make install acb9348 [Java] Include WriteBatch into RocksDBSample.java, fix how DbBenchmark.java handles WriteBatch. 4a27a2f Don't sync manifest when disableDataSync = true 9b8480d Merge pull request #287 from yinqiwen/rate-limiter-crash-fix 28be16b fix rate limiter crash #286 04ce1b2 Fix #284 add22e3 standardize scripts to run RocksDB benchmarks dee91c2 WriteThread 540a257 Fix WAL synced 24f034b Merge pull request #282 from Chilledheart/develop 49fe329 Fix build issue under macosx ebb5c65 Add make install 0352a9f add_wrapped_bloom_test 9c0e66c Don't run background jobs (flush, compactions) when bg_error_ is set a9639bd Fix valgrind test d1f24dc Relax FlushSchedule test 3d9e6f7 Push model for flushing memtables 059e584 [unit test] CompactRange should fail if we don't have space dd641b2 fix RocksDB java build 53404d9 add_qps_info_in cache bench a52cecb Fix Mac compile 092f97e Fix comments and typos 6cc1286 Added a few statistics for BackupableDB 0a42295 Fix SimpleWriteTimeoutTest 06d9862 Always pass MergeContext as pointer, not reference d343c3f Improve db recovery 6bb7e3e Merger test 88841bd Explicitly cast char to signed char in Hash() 5231146 MemTableOptions 1d284db Addressing review comments 55114e7 Some updates for SpatialDB 171d4ff remove TailingIterator reference in db_impl.h 9b0f7ff rename version_set options_ to db_options_ to avoid confusion 2d57828 Check stop level trigger-0 before slowdown level-0 trigger 659d2d5 move compaction_filter to immutable_options 048560a reduce references to cfd->options() in DBImpl 011241b DB::Flush() Do not wait for background threads when there is nothing in mem table a2bb7c3 Push- instead of pull-model for managing Write stalls 0af157f Implement full filter for block based table. 9360cc6 Fix valgrind issue 02d5bff Merge pull request #277 from wankai/master 88a2f44 fix comments 7c16e39 Merge pull request #276 from wankai/master 8237738 replace hard-coded number with named variable db8ca52 Merge pull request #273 from nbougalis/static-analysis b7b031f Merge pull request #274 from wankai/master 4c2b1f0 Merge remote-tracking branch 'upstream/master' a5d2863 typo improvement 9f8aa09 Don't leak data returned by opendir d1cfb71 Remove unused member(s) bfee319 sizeof(int*) where sizeof(int) was intended d40c1f7 Add missing break statement 2e97c38 Avoid off-by-one error when using readlink 40ddc3d add cache bench 9f1c80b Drop column family from write thread 8de151b Add db_bench with lots of column families to regression tests c9e419c rename options_ to db_options_ in DBImpl to avoid confusion 5cd0576 Fix compaction bug in Cuckoo Table Builder. Use kvs_.size() instead of num_entries in FileSize() method. 0fbb3fa fixed memory leak in unit test DBIteratorBoundTest adcd253 fix asan check 4092b7a Merge pull request #272 from project-zerus/patch-1 bb6ae0f fix more compile warnings 6d31441 Merge pull request #271 from nbougalis/cleanups 0cd0ec4 Plug memory leak during index creation 4329d74 Fix swapped variable names to accurately reflect usage 45a5e3e Remove path with arena==nullptr from NewInternalIterator 5665e5e introduce ImmutableOptions e0b99d4 created a new ReadOptions parameter 'iterate_upper_bound' 51ea889 Fix travis builds a481626 Relax backupable rate limiting test f7f973d Merge pull request #269 from huahang/patch-2 ef5b384 fix a few compile warnings 2fd3806 Merge pull request #263 from wankai/master 1785114 delete unused Comparator 1b1d961 update HISTORY.md 703c3ea comments about the BlockBasedTableOptions migration in Options 4b5ad88 Merge pull request #260 from wankai/master 19cc588 change to filter_block std::unique_ptr support RAII 9b976e3 Merge pull request #259 from wankai/master 5d25a46 Merge remote-tracking branch 'upstream/master' 9b58c73 call SanitizeDBOptionsByCFOptions() in the right place a84234a Ignore missing column families 8ed70fc add assert to db Put in db_stress test 7f19bb9 Merge pull request #242 from tdfischer/perf-timer-destructors 8438a19 fix dropping column family bug 6614a48 Refactor PerfStepTimer to stop on destruct 076bd01 Fix compile 990df99 Fix ios compile 7dcadb1 Don't let flush preempt compaction in certain cases dff2b1a typo improvement 985a31c Merge pull request #251 from nbougalis/master f09329c Fix candidate file comparison when using path ids 7e9f28c limit max bytes that can be read/written per pread/write syscall d20b8cf Improve Cuckoo Table Reader performance. Inlined hash function and number of buckets a power of two. 0f9c43e ForwardIterator: reset incomplete iterators on Seek() 722d80c reduce recordTick overhead in compaction loop 22a0a60 Merge pull request #250 from wankai/master be25ee4 delete unused struct Options 0c26e76 Merge pull request #237 from tdfischer/tdfischer/faster-timeout-test 1d23b5c remove_internal_filter_policy 2a8faf7 Compact SpatialDB as we go, not at the end 7f71448 Implementing a cache friendly version of Cuckoo Hash d977e55 Don't let other compactions run when manual compaction runs d5bd6c7 Fix ios compile 6b46f78 Merge pull request #248 from wankai/master 528a11c Update block_builder.h 536e997 Remove assert in vector rep 4142a3e Adding a user comparator for comparing Uint64 slices. 1913ce2 more concurrent flushes in SpatialDB 808e809 Adjust SpatialDB column family options 0c39f54 Use Vector memtable when bulk loading SpatialDB b6fd781 Don't do memtable lookup in db_impl_readonly if memtables are empty while opening db. 9dcb75b Add is-file-deletions-enabled property 1755581 improve OptimizeForPointLookup() d9c0785 Fix assertion in PosixRandomAccessFile bda6f33 fix valgrind error in c_test caused by BlockBasedTableOptions 0db6b02 Update timeout to 50ms instead of 3. ff6ec0e Optimize SpatialDB 2386185 ReadOptions.total_order_seek to allow total order seek for block-based table when hash index is enabled a98badf print table options 66f62e5 JNI changes corresponding to BlockBasedTableOptions migration 3844001 move block based table related options BlockBasedTableOptions 17b54ae Merge pull request #243 from andybons/patch-1 0508691 Add missing include to use std::unique_ptr 42ea795 Fix concurrency issue in CompactionPicker bb530c0 Merge pull request #240 from ShaoYuZhang/master f76eda7 Fix compilation issue on OSX 08be7f5 Implement Prepare method in CuckooTableReader 47b452c Fix the error of c_test.c 562b7a1 Add missing implementaiton of SanitizeDBOptions in simple_table_db_test.cc 63a2215 Improve Options sanitization and add MmapReadRequired() to TableFactory e173bf9 Eliminate VersionSet memory leak 10720a5 Revert the unintended change that DestroyDB() doesn't clean up info logs. 01cbdd2 Optimize storage parameters for spatialDB 045575a Add CuckooHash table format to table_reader_bench 7c5173d test: db: fix test to have a smaller timeout for when it runs on faster hardware 6929b08 Remove BitStream* tests 50b790c Removing BitStream* functions 162b815 Adding Column Family support in db_bench. 28b5c76 WriteBatchWithIndex: a wrapper of WriteBatch, with a searchable index 5585e00 Update release note of 3.4 343e98a Reverting import change ddb8039 RocksDB static build Make file changes to download and build the dependencies .Load the shared library when RocksDB is initialized 68eed8c Bump up version 36e759d Adding Cuckoo Table SST option to db_bench a6fd14c Fix valgrind error in c_test c703715 attempt to fix auto_roll_logger_test c8ecfae Merge pull request #230 from cockroachdb/spencerkimball/send-user-keys-to-v2-filter 570ba5a Avoid retrying to read property block from a table when it does not exist. 625b9ef Merge pull request #234 from bbiao/master 59a2763 Fix typo huage => huge f611935 Fix autovector iterator increment/decrement comments 58b0f9d Support purging logs from separate log directory 2da53b1 [Java] Add purgeOldBackups API 6c4c159 fix_sst_dump_for_old_sst_format 8dfe2fd fix compile error under Mac OS X 58c4946 Allow env_posix to lower background thread IO priority 6a2be31 fix_valgrind_error_caused_in_db_info_dummper e91ebf1 print compaction_filter name in Options.Dump 5a5953b Add histogram for DB_SEEK 5e64240 log db path info before open 0c9dc9f Remove malloc from FormatFileNumber bcefede Update HISTORY.md 4808177 Revert "Include candidate files under options.db_log_dir in FindObsoleteFiles()" 0138b8e Fixed compile errors (signed / unsigned comparison) in cuckoo_table_db_test on Mac 1562653 Fixed a signed-unsigned comparison error in db_test 218857b remove tailing_iter.h/cc 5d0074c set bytes_per_sync to 1MB if rate limiter is enabled 3fcf7b2 Pass parsed user key to prefix extractor in V2 compaction 2fa6434 Add scope guard 06a52bd Flush only one column family 9674c11 Integrating Cuckoo Hash SST Table format into RocksDB git-subtree-dir: src/rocksdb2 git-subtree-split: 25888ae0068c9b8e3d9421ea8c78a7be339298d8
805 lines
29 KiB
C++
805 lines
29 KiB
C++
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under the BSD-style license found in the
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
//
|
|
// An Env is an interface used by the rocksdb implementation to access
|
|
// operating system functionality like the filesystem etc. Callers
|
|
// may wish to provide a custom Env object when opening a database to
|
|
// get fine gain control; e.g., to rate limit file system operations.
|
|
//
|
|
// All Env implementations are safe for concurrent access from
|
|
// multiple threads without any external synchronization.
|
|
|
|
#ifndef STORAGE_ROCKSDB_INCLUDE_ENV_H_
|
|
#define STORAGE_ROCKSDB_INCLUDE_ENV_H_
|
|
|
|
#include <cstdarg>
|
|
#include <string>
|
|
#include <memory>
|
|
#include <vector>
|
|
#include <stdint.h>
|
|
#include "rocksdb/status.h"
|
|
|
|
namespace rocksdb {
|
|
|
|
class FileLock;
|
|
class Logger;
|
|
class RandomAccessFile;
|
|
class SequentialFile;
|
|
class Slice;
|
|
class WritableFile;
|
|
class RandomRWFile;
|
|
class Directory;
|
|
struct DBOptions;
|
|
class RateLimiter;
|
|
|
|
using std::unique_ptr;
|
|
using std::shared_ptr;
|
|
|
|
|
|
// Options while opening a file to read/write
|
|
struct EnvOptions {
|
|
|
|
// construct with default Options
|
|
EnvOptions();
|
|
|
|
// construct from Options
|
|
explicit EnvOptions(const DBOptions& options);
|
|
|
|
// If true, then allow caching of data in environment buffers
|
|
bool use_os_buffer = true;
|
|
|
|
// If true, then use mmap to read data
|
|
bool use_mmap_reads = false;
|
|
|
|
// If true, then use mmap to write data
|
|
bool use_mmap_writes = true;
|
|
|
|
// If true, set the FD_CLOEXEC on open fd.
|
|
bool set_fd_cloexec = true;
|
|
|
|
// Allows OS to incrementally sync files to disk while they are being
|
|
// written, in the background. Issue one request for every bytes_per_sync
|
|
// written. 0 turns it off.
|
|
// Default: 0
|
|
uint64_t bytes_per_sync = 0;
|
|
|
|
// If true, we will preallocate the file with FALLOC_FL_KEEP_SIZE flag, which
|
|
// means that file size won't change as part of preallocation.
|
|
// If false, preallocation will also change the file size. This option will
|
|
// improve the performance in workloads where you sync the data on every
|
|
// write. By default, we set it to true for MANIFEST writes and false for
|
|
// WAL writes
|
|
bool fallocate_with_keep_size = true;
|
|
|
|
// If not nullptr, write rate limiting is enabled for flush and compaction
|
|
RateLimiter* rate_limiter = nullptr;
|
|
};
|
|
|
|
class Env {
|
|
public:
|
|
Env() { }
|
|
virtual ~Env();
|
|
|
|
// Return a default environment suitable for the current operating
|
|
// system. Sophisticated users may wish to provide their own Env
|
|
// implementation instead of relying on this default environment.
|
|
//
|
|
// The result of Default() belongs to rocksdb and must never be deleted.
|
|
static Env* Default();
|
|
|
|
// Create a brand new sequentially-readable file with the specified name.
|
|
// On success, stores a pointer to the new file in *result and returns OK.
|
|
// On failure stores nullptr in *result and returns non-OK. If the file does
|
|
// not exist, returns a non-OK status.
|
|
//
|
|
// The returned file will only be accessed by one thread at a time.
|
|
virtual Status NewSequentialFile(const std::string& fname,
|
|
unique_ptr<SequentialFile>* result,
|
|
const EnvOptions& options)
|
|
= 0;
|
|
|
|
// Create a brand new random access read-only file with the
|
|
// specified name. On success, stores a pointer to the new file in
|
|
// *result and returns OK. On failure stores nullptr in *result and
|
|
// returns non-OK. If the file does not exist, returns a non-OK
|
|
// status.
|
|
//
|
|
// The returned file may be concurrently accessed by multiple threads.
|
|
virtual Status NewRandomAccessFile(const std::string& fname,
|
|
unique_ptr<RandomAccessFile>* result,
|
|
const EnvOptions& options)
|
|
= 0;
|
|
|
|
// Create an object that writes to a new file with the specified
|
|
// name. Deletes any existing file with the same name and creates a
|
|
// new file. On success, stores a pointer to the new file in
|
|
// *result and returns OK. On failure stores nullptr in *result and
|
|
// returns non-OK.
|
|
//
|
|
// The returned file will only be accessed by one thread at a time.
|
|
virtual Status NewWritableFile(const std::string& fname,
|
|
unique_ptr<WritableFile>* result,
|
|
const EnvOptions& options) = 0;
|
|
|
|
// Create an object that both reads and writes to a file on
|
|
// specified offsets (random access). If file already exists,
|
|
// does not overwrite it. On success, stores a pointer to the
|
|
// new file in *result and returns OK. On failure stores nullptr
|
|
// in *result and returns non-OK.
|
|
virtual Status NewRandomRWFile(const std::string& fname,
|
|
unique_ptr<RandomRWFile>* result,
|
|
const EnvOptions& options) = 0;
|
|
|
|
// Create an object that represents a directory. Will fail if directory
|
|
// doesn't exist. If the directory exists, it will open the directory
|
|
// and create a new Directory object.
|
|
//
|
|
// On success, stores a pointer to the new Directory in
|
|
// *result and returns OK. On failure stores nullptr in *result and
|
|
// returns non-OK.
|
|
virtual Status NewDirectory(const std::string& name,
|
|
unique_ptr<Directory>* result) = 0;
|
|
|
|
// Returns true iff the named file exists.
|
|
virtual bool FileExists(const std::string& fname) = 0;
|
|
|
|
// Store in *result the names of the children of the specified directory.
|
|
// The names are relative to "dir".
|
|
// Original contents of *results are dropped.
|
|
virtual Status GetChildren(const std::string& dir,
|
|
std::vector<std::string>* result) = 0;
|
|
|
|
// Delete the named file.
|
|
virtual Status DeleteFile(const std::string& fname) = 0;
|
|
|
|
// Create the specified directory. Returns error if directory exists.
|
|
virtual Status CreateDir(const std::string& dirname) = 0;
|
|
|
|
// Creates directory if missing. Return Ok if it exists, or successful in
|
|
// Creating.
|
|
virtual Status CreateDirIfMissing(const std::string& dirname) = 0;
|
|
|
|
// Delete the specified directory.
|
|
virtual Status DeleteDir(const std::string& dirname) = 0;
|
|
|
|
// Store the size of fname in *file_size.
|
|
virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) = 0;
|
|
|
|
// Store the last modification time of fname in *file_mtime.
|
|
virtual Status GetFileModificationTime(const std::string& fname,
|
|
uint64_t* file_mtime) = 0;
|
|
// Rename file src to target.
|
|
virtual Status RenameFile(const std::string& src,
|
|
const std::string& target) = 0;
|
|
|
|
// Lock the specified file. Used to prevent concurrent access to
|
|
// the same db by multiple processes. On failure, stores nullptr in
|
|
// *lock and returns non-OK.
|
|
//
|
|
// On success, stores a pointer to the object that represents the
|
|
// acquired lock in *lock and returns OK. The caller should call
|
|
// UnlockFile(*lock) to release the lock. If the process exits,
|
|
// the lock will be automatically released.
|
|
//
|
|
// If somebody else already holds the lock, finishes immediately
|
|
// with a failure. I.e., this call does not wait for existing locks
|
|
// to go away.
|
|
//
|
|
// May create the named file if it does not already exist.
|
|
virtual Status LockFile(const std::string& fname, FileLock** lock) = 0;
|
|
|
|
// Release the lock acquired by a previous successful call to LockFile.
|
|
// REQUIRES: lock was returned by a successful LockFile() call
|
|
// REQUIRES: lock has not already been unlocked.
|
|
virtual Status UnlockFile(FileLock* lock) = 0;
|
|
|
|
// Priority for scheduling job in thread pool
|
|
enum Priority { LOW, HIGH, TOTAL };
|
|
|
|
// Priority for requesting bytes in rate limiter scheduler
|
|
enum IOPriority {
|
|
IO_LOW = 0,
|
|
IO_HIGH = 1,
|
|
IO_TOTAL = 2
|
|
};
|
|
|
|
// Arrange to run "(*function)(arg)" once in a background thread, in
|
|
// the thread pool specified by pri. By default, jobs go to the 'LOW'
|
|
// priority thread pool.
|
|
|
|
// "function" may run in an unspecified thread. Multiple functions
|
|
// added to the same Env may run concurrently in different threads.
|
|
// I.e., the caller may not assume that background work items are
|
|
// serialized.
|
|
virtual void Schedule(
|
|
void (*function)(void* arg),
|
|
void* arg,
|
|
Priority pri = LOW) = 0;
|
|
|
|
// Start a new thread, invoking "function(arg)" within the new thread.
|
|
// When "function(arg)" returns, the thread will be destroyed.
|
|
virtual void StartThread(void (*function)(void* arg), void* arg) = 0;
|
|
|
|
// Wait for all threads started by StartThread to terminate.
|
|
virtual void WaitForJoin() {}
|
|
|
|
// Get thread pool queue length for specific thrad pool.
|
|
virtual unsigned int GetThreadPoolQueueLen(Priority pri = LOW) const {
|
|
return 0;
|
|
}
|
|
|
|
// *path is set to a temporary directory that can be used for testing. It may
|
|
// or many not have just been created. The directory may or may not differ
|
|
// between runs of the same process, but subsequent calls will return the
|
|
// same directory.
|
|
virtual Status GetTestDirectory(std::string* path) = 0;
|
|
|
|
// Create and return a log file for storing informational messages.
|
|
virtual Status NewLogger(const std::string& fname,
|
|
shared_ptr<Logger>* result) = 0;
|
|
|
|
// Returns the number of micro-seconds since some fixed point in time. Only
|
|
// useful for computing deltas of time.
|
|
virtual uint64_t NowMicros() = 0;
|
|
|
|
// Returns the number of nano-seconds since some fixed point in time. Only
|
|
// useful for computing deltas of time in one run.
|
|
// Default implementation simply relies on NowMicros
|
|
virtual uint64_t NowNanos() {
|
|
return NowMicros() * 1000;
|
|
}
|
|
|
|
// Sleep/delay the thread for the perscribed number of micro-seconds.
|
|
virtual void SleepForMicroseconds(int micros) = 0;
|
|
|
|
// Get the current host name.
|
|
virtual Status GetHostName(char* name, uint64_t len) = 0;
|
|
|
|
// Get the number of seconds since the Epoch, 1970-01-01 00:00:00 (UTC).
|
|
virtual Status GetCurrentTime(int64_t* unix_time) = 0;
|
|
|
|
// Get full directory name for this db.
|
|
virtual Status GetAbsolutePath(const std::string& db_path,
|
|
std::string* output_path) = 0;
|
|
|
|
// The number of background worker threads of a specific thread pool
|
|
// for this environment. 'LOW' is the default pool.
|
|
// default number: 1
|
|
virtual void SetBackgroundThreads(int number, Priority pri = LOW) = 0;
|
|
|
|
// Lower IO priority for threads from the specified pool.
|
|
virtual void LowerThreadPoolIOPriority(Priority pool = LOW) {}
|
|
|
|
// Converts seconds-since-Jan-01-1970 to a printable string
|
|
virtual std::string TimeToString(uint64_t time) = 0;
|
|
|
|
// Generates a unique id that can be used to identify a db
|
|
virtual std::string GenerateUniqueId();
|
|
|
|
// OptimizeForLogWrite will create a new EnvOptions object that is a copy of
|
|
// the EnvOptions in the parameters, but is optimized for writing log files.
|
|
// Default implementation returns the copy of the same object.
|
|
virtual EnvOptions OptimizeForLogWrite(const EnvOptions& env_options) const;
|
|
// OptimizeForManifestWrite will create a new EnvOptions object that is a copy
|
|
// of the EnvOptions in the parameters, but is optimized for writing manifest
|
|
// files. Default implementation returns the copy of the same object.
|
|
virtual EnvOptions OptimizeForManifestWrite(const EnvOptions& env_options)
|
|
const;
|
|
|
|
private:
|
|
// No copying allowed
|
|
Env(const Env&);
|
|
void operator=(const Env&);
|
|
};
|
|
|
|
// A file abstraction for reading sequentially through a file
|
|
class SequentialFile {
|
|
public:
|
|
SequentialFile() { }
|
|
virtual ~SequentialFile();
|
|
|
|
// Read up to "n" bytes from the file. "scratch[0..n-1]" may be
|
|
// written by this routine. Sets "*result" to the data that was
|
|
// read (including if fewer than "n" bytes were successfully read).
|
|
// May set "*result" to point at data in "scratch[0..n-1]", so
|
|
// "scratch[0..n-1]" must be live when "*result" is used.
|
|
// If an error was encountered, returns a non-OK status.
|
|
//
|
|
// REQUIRES: External synchronization
|
|
virtual Status Read(size_t n, Slice* result, char* scratch) = 0;
|
|
|
|
// Skip "n" bytes from the file. This is guaranteed to be no
|
|
// slower that reading the same data, but may be faster.
|
|
//
|
|
// If end of file is reached, skipping will stop at the end of the
|
|
// file, and Skip will return OK.
|
|
//
|
|
// REQUIRES: External synchronization
|
|
virtual Status Skip(uint64_t n) = 0;
|
|
|
|
// Remove any kind of caching of data from the offset to offset+length
|
|
// of this file. If the length is 0, then it refers to the end of file.
|
|
// If the system is not caching the file contents, then this is a noop.
|
|
virtual Status InvalidateCache(size_t offset, size_t length) {
|
|
return Status::NotSupported("InvalidateCache not supported.");
|
|
}
|
|
};
|
|
|
|
// A file abstraction for randomly reading the contents of a file.
|
|
class RandomAccessFile {
|
|
public:
|
|
RandomAccessFile() { }
|
|
virtual ~RandomAccessFile();
|
|
|
|
// Read up to "n" bytes from the file starting at "offset".
|
|
// "scratch[0..n-1]" may be written by this routine. Sets "*result"
|
|
// to the data that was read (including if fewer than "n" bytes were
|
|
// successfully read). May set "*result" to point at data in
|
|
// "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
|
|
// "*result" is used. If an error was encountered, returns a non-OK
|
|
// status.
|
|
//
|
|
// Safe for concurrent use by multiple threads.
|
|
virtual Status Read(uint64_t offset, size_t n, Slice* result,
|
|
char* scratch) const = 0;
|
|
|
|
// Tries to get an unique ID for this file that will be the same each time
|
|
// the file is opened (and will stay the same while the file is open).
|
|
// Furthermore, it tries to make this ID at most "max_size" bytes. If such an
|
|
// ID can be created this function returns the length of the ID and places it
|
|
// in "id"; otherwise, this function returns 0, in which case "id"
|
|
// may not have been modified.
|
|
//
|
|
// This function guarantees, for IDs from a given environment, two unique ids
|
|
// cannot be made equal to eachother by adding arbitrary bytes to one of
|
|
// them. That is, no unique ID is the prefix of another.
|
|
//
|
|
// This function guarantees that the returned ID will not be interpretable as
|
|
// a single varint.
|
|
//
|
|
// Note: these IDs are only valid for the duration of the process.
|
|
virtual size_t GetUniqueId(char* id, size_t max_size) const {
|
|
return 0; // Default implementation to prevent issues with backwards
|
|
// compatibility.
|
|
};
|
|
|
|
|
|
enum AccessPattern { NORMAL, RANDOM, SEQUENTIAL, WILLNEED, DONTNEED };
|
|
|
|
virtual void Hint(AccessPattern pattern) {}
|
|
|
|
// Remove any kind of caching of data from the offset to offset+length
|
|
// of this file. If the length is 0, then it refers to the end of file.
|
|
// If the system is not caching the file contents, then this is a noop.
|
|
virtual Status InvalidateCache(size_t offset, size_t length) {
|
|
return Status::NotSupported("InvalidateCache not supported.");
|
|
}
|
|
};
|
|
|
|
// A file abstraction for sequential writing. The implementation
|
|
// must provide buffering since callers may append small fragments
|
|
// at a time to the file.
|
|
class WritableFile {
|
|
public:
|
|
WritableFile()
|
|
: last_preallocated_block_(0),
|
|
preallocation_block_size_(0),
|
|
io_priority_(Env::IO_TOTAL) {
|
|
}
|
|
virtual ~WritableFile();
|
|
|
|
virtual Status Append(const Slice& data) = 0;
|
|
virtual Status Close() = 0;
|
|
virtual Status Flush() = 0;
|
|
virtual Status Sync() = 0; // sync data
|
|
|
|
/*
|
|
* Sync data and/or metadata as well.
|
|
* By default, sync only data.
|
|
* Override this method for environments where we need to sync
|
|
* metadata as well.
|
|
*/
|
|
virtual Status Fsync() {
|
|
return Sync();
|
|
}
|
|
|
|
/*
|
|
* Change the priority in rate limiter if rate limiting is enabled.
|
|
* If rate limiting is not enabled, this call has no effect.
|
|
*/
|
|
virtual void SetIOPriority(Env::IOPriority pri) {
|
|
io_priority_ = pri;
|
|
}
|
|
|
|
/*
|
|
* Get the size of valid data in the file.
|
|
*/
|
|
virtual uint64_t GetFileSize() {
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Get and set the default pre-allocation block size for writes to
|
|
* this file. If non-zero, then Allocate will be used to extend the
|
|
* underlying storage of a file (generally via fallocate) if the Env
|
|
* instance supports it.
|
|
*/
|
|
void SetPreallocationBlockSize(size_t size) {
|
|
preallocation_block_size_ = size;
|
|
}
|
|
|
|
virtual void GetPreallocationStatus(size_t* block_size,
|
|
size_t* last_allocated_block) {
|
|
*last_allocated_block = last_preallocated_block_;
|
|
*block_size = preallocation_block_size_;
|
|
}
|
|
|
|
// For documentation, refer to RandomAccessFile::GetUniqueId()
|
|
virtual size_t GetUniqueId(char* id, size_t max_size) const {
|
|
return 0; // Default implementation to prevent issues with backwards
|
|
}
|
|
|
|
// Remove any kind of caching of data from the offset to offset+length
|
|
// of this file. If the length is 0, then it refers to the end of file.
|
|
// If the system is not caching the file contents, then this is a noop.
|
|
// This call has no effect on dirty pages in the cache.
|
|
virtual Status InvalidateCache(size_t offset, size_t length) {
|
|
return Status::NotSupported("InvalidateCache not supported.");
|
|
}
|
|
|
|
protected:
|
|
// PrepareWrite performs any necessary preparation for a write
|
|
// before the write actually occurs. This allows for pre-allocation
|
|
// of space on devices where it can result in less file
|
|
// fragmentation and/or less waste from over-zealous filesystem
|
|
// pre-allocation.
|
|
void PrepareWrite(size_t offset, size_t len) {
|
|
if (preallocation_block_size_ == 0) {
|
|
return;
|
|
}
|
|
// If this write would cross one or more preallocation blocks,
|
|
// determine what the last preallocation block necesessary to
|
|
// cover this write would be and Allocate to that point.
|
|
const auto block_size = preallocation_block_size_;
|
|
size_t new_last_preallocated_block =
|
|
(offset + len + block_size - 1) / block_size;
|
|
if (new_last_preallocated_block > last_preallocated_block_) {
|
|
size_t num_spanned_blocks =
|
|
new_last_preallocated_block - last_preallocated_block_;
|
|
Allocate(block_size * last_preallocated_block_,
|
|
block_size * num_spanned_blocks);
|
|
last_preallocated_block_ = new_last_preallocated_block;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Pre-allocate space for a file.
|
|
*/
|
|
virtual Status Allocate(off_t offset, off_t len) {
|
|
return Status::OK();
|
|
}
|
|
|
|
// Sync a file range with disk.
|
|
// offset is the starting byte of the file range to be synchronized.
|
|
// nbytes specifies the length of the range to be synchronized.
|
|
// This asks the OS to initiate flushing the cached data to disk,
|
|
// without waiting for completion.
|
|
// Default implementation does nothing.
|
|
virtual Status RangeSync(off_t offset, off_t nbytes) {
|
|
return Status::OK();
|
|
}
|
|
|
|
private:
|
|
size_t last_preallocated_block_;
|
|
size_t preallocation_block_size_;
|
|
// No copying allowed
|
|
WritableFile(const WritableFile&);
|
|
void operator=(const WritableFile&);
|
|
|
|
protected:
|
|
Env::IOPriority io_priority_;
|
|
};
|
|
|
|
// A file abstraction for random reading and writing.
|
|
class RandomRWFile {
|
|
public:
|
|
RandomRWFile() {}
|
|
virtual ~RandomRWFile() {}
|
|
|
|
// Write data from Slice data to file starting from offset
|
|
// Returns IOError on failure, but does not guarantee
|
|
// atomicity of a write. Returns OK status on success.
|
|
//
|
|
// Safe for concurrent use.
|
|
virtual Status Write(uint64_t offset, const Slice& data) = 0;
|
|
// Read up to "n" bytes from the file starting at "offset".
|
|
// "scratch[0..n-1]" may be written by this routine. Sets "*result"
|
|
// to the data that was read (including if fewer than "n" bytes were
|
|
// successfully read). May set "*result" to point at data in
|
|
// "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
|
|
// "*result" is used. If an error was encountered, returns a non-OK
|
|
// status.
|
|
//
|
|
// Safe for concurrent use by multiple threads.
|
|
virtual Status Read(uint64_t offset, size_t n, Slice* result,
|
|
char* scratch) const = 0;
|
|
virtual Status Close() = 0; // closes the file
|
|
virtual Status Sync() = 0; // sync data
|
|
|
|
/*
|
|
* Sync data and/or metadata as well.
|
|
* By default, sync only data.
|
|
* Override this method for environments where we need to sync
|
|
* metadata as well.
|
|
*/
|
|
virtual Status Fsync() {
|
|
return Sync();
|
|
}
|
|
|
|
/*
|
|
* Pre-allocate space for a file.
|
|
*/
|
|
virtual Status Allocate(off_t offset, off_t len) {
|
|
return Status::OK();
|
|
}
|
|
|
|
private:
|
|
// No copying allowed
|
|
RandomRWFile(const RandomRWFile&);
|
|
void operator=(const RandomRWFile&);
|
|
};
|
|
|
|
// Directory object represents collection of files and implements
|
|
// filesystem operations that can be executed on directories.
|
|
class Directory {
|
|
public:
|
|
virtual ~Directory() {}
|
|
// Fsync directory
|
|
virtual Status Fsync() = 0;
|
|
};
|
|
|
|
enum InfoLogLevel : unsigned char {
|
|
DEBUG_LEVEL = 0,
|
|
INFO_LEVEL,
|
|
WARN_LEVEL,
|
|
ERROR_LEVEL,
|
|
FATAL_LEVEL,
|
|
NUM_INFO_LOG_LEVELS,
|
|
};
|
|
|
|
// An interface for writing log messages.
|
|
class Logger {
|
|
public:
|
|
enum { DO_NOT_SUPPORT_GET_LOG_FILE_SIZE = -1 };
|
|
explicit Logger(const InfoLogLevel log_level = InfoLogLevel::INFO_LEVEL)
|
|
: log_level_(log_level) {}
|
|
virtual ~Logger();
|
|
|
|
// Write an entry to the log file with the specified format.
|
|
virtual void Logv(const char* format, va_list ap) = 0;
|
|
|
|
// Write an entry to the log file with the specified log level
|
|
// and format. Any log with level under the internal log level
|
|
// of *this (see @SetInfoLogLevel and @GetInfoLogLevel) will not be
|
|
// printed.
|
|
void Logv(const InfoLogLevel log_level, const char* format, va_list ap) {
|
|
static const char* kInfoLogLevelNames[5] = {"DEBUG", "INFO", "WARN",
|
|
"ERROR", "FATAL"};
|
|
if (log_level < log_level_) {
|
|
return;
|
|
}
|
|
|
|
if (log_level == InfoLogLevel::INFO_LEVEL) {
|
|
// Doesn't print log level if it is INFO level.
|
|
// This is to avoid unexpected performance regression after we add
|
|
// the feature of log level. All the logs before we add the feature
|
|
// are INFO level. We don't want to add extra costs to those existing
|
|
// logging.
|
|
Logv(format, ap);
|
|
} else {
|
|
char new_format[500];
|
|
snprintf(new_format, sizeof(new_format) - 1, "[%s] %s",
|
|
kInfoLogLevelNames[log_level], format);
|
|
Logv(new_format, ap);
|
|
}
|
|
}
|
|
virtual size_t GetLogFileSize() const {
|
|
return DO_NOT_SUPPORT_GET_LOG_FILE_SIZE;
|
|
}
|
|
// Flush to the OS buffers
|
|
virtual void Flush() {}
|
|
virtual InfoLogLevel GetInfoLogLevel() const { return log_level_; }
|
|
virtual void SetInfoLogLevel(const InfoLogLevel log_level) {
|
|
log_level_ = log_level;
|
|
}
|
|
|
|
private:
|
|
// No copying allowed
|
|
Logger(const Logger&);
|
|
void operator=(const Logger&);
|
|
InfoLogLevel log_level_;
|
|
};
|
|
|
|
|
|
// Identifies a locked file.
|
|
class FileLock {
|
|
public:
|
|
FileLock() { }
|
|
virtual ~FileLock();
|
|
private:
|
|
// No copying allowed
|
|
FileLock(const FileLock&);
|
|
void operator=(const FileLock&);
|
|
};
|
|
|
|
extern void LogFlush(const shared_ptr<Logger>& info_log);
|
|
|
|
extern void Log(const InfoLogLevel log_level,
|
|
const shared_ptr<Logger>& info_log, const char* format, ...);
|
|
|
|
// a set of log functions with different log levels.
|
|
extern void Debug(const shared_ptr<Logger>& info_log, const char* format, ...);
|
|
extern void Info(const shared_ptr<Logger>& info_log, const char* format, ...);
|
|
extern void Warn(const shared_ptr<Logger>& info_log, const char* format, ...);
|
|
extern void Error(const shared_ptr<Logger>& info_log, const char* format, ...);
|
|
extern void Fatal(const shared_ptr<Logger>& info_log, const char* format, ...);
|
|
|
|
// Log the specified data to *info_log if info_log is non-nullptr.
|
|
// The default info log level is InfoLogLevel::ERROR.
|
|
extern void Log(const shared_ptr<Logger>& info_log, const char* format, ...)
|
|
# if defined(__GNUC__) || defined(__clang__)
|
|
__attribute__((__format__ (__printf__, 2, 3)))
|
|
# endif
|
|
;
|
|
|
|
extern void LogFlush(Logger *info_log);
|
|
|
|
extern void Log(const InfoLogLevel log_level, Logger* info_log,
|
|
const char* format, ...);
|
|
|
|
// The default info log level is InfoLogLevel::ERROR.
|
|
extern void Log(Logger* info_log, const char* format, ...)
|
|
# if defined(__GNUC__) || defined(__clang__)
|
|
__attribute__((__format__ (__printf__, 2, 3)))
|
|
# endif
|
|
;
|
|
|
|
// a set of log functions with different log levels.
|
|
extern void Debug(Logger* info_log, const char* format, ...);
|
|
extern void Info(Logger* info_log, const char* format, ...);
|
|
extern void Warn(Logger* info_log, const char* format, ...);
|
|
extern void Error(Logger* info_log, const char* format, ...);
|
|
extern void Fatal(Logger* info_log, const char* format, ...);
|
|
|
|
// A utility routine: write "data" to the named file.
|
|
extern Status WriteStringToFile(Env* env, const Slice& data,
|
|
const std::string& fname,
|
|
bool should_sync = false);
|
|
|
|
// A utility routine: read contents of named file into *data
|
|
extern Status ReadFileToString(Env* env, const std::string& fname,
|
|
std::string* data);
|
|
|
|
// An implementation of Env that forwards all calls to another Env.
|
|
// May be useful to clients who wish to override just part of the
|
|
// functionality of another Env.
|
|
class EnvWrapper : public Env {
|
|
public:
|
|
// Initialize an EnvWrapper that delegates all calls to *t
|
|
explicit EnvWrapper(Env* t) : target_(t) { }
|
|
virtual ~EnvWrapper();
|
|
|
|
// Return the target to which this Env forwards all calls
|
|
Env* target() const { return target_; }
|
|
|
|
// The following text is boilerplate that forwards all methods to target()
|
|
Status NewSequentialFile(const std::string& f,
|
|
unique_ptr<SequentialFile>* r,
|
|
const EnvOptions& options) {
|
|
return target_->NewSequentialFile(f, r, options);
|
|
}
|
|
Status NewRandomAccessFile(const std::string& f,
|
|
unique_ptr<RandomAccessFile>* r,
|
|
const EnvOptions& options) {
|
|
return target_->NewRandomAccessFile(f, r, options);
|
|
}
|
|
Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
|
|
const EnvOptions& options) {
|
|
return target_->NewWritableFile(f, r, options);
|
|
}
|
|
Status NewRandomRWFile(const std::string& f, unique_ptr<RandomRWFile>* r,
|
|
const EnvOptions& options) {
|
|
return target_->NewRandomRWFile(f, r, options);
|
|
}
|
|
virtual Status NewDirectory(const std::string& name,
|
|
unique_ptr<Directory>* result) {
|
|
return target_->NewDirectory(name, result);
|
|
}
|
|
bool FileExists(const std::string& f) { return target_->FileExists(f); }
|
|
Status GetChildren(const std::string& dir, std::vector<std::string>* r) {
|
|
return target_->GetChildren(dir, r);
|
|
}
|
|
Status DeleteFile(const std::string& f) { return target_->DeleteFile(f); }
|
|
Status CreateDir(const std::string& d) { return target_->CreateDir(d); }
|
|
Status CreateDirIfMissing(const std::string& d) {
|
|
return target_->CreateDirIfMissing(d);
|
|
}
|
|
Status DeleteDir(const std::string& d) { return target_->DeleteDir(d); }
|
|
Status GetFileSize(const std::string& f, uint64_t* s) {
|
|
return target_->GetFileSize(f, s);
|
|
}
|
|
|
|
Status GetFileModificationTime(const std::string& fname,
|
|
uint64_t* file_mtime) {
|
|
return target_->GetFileModificationTime(fname, file_mtime);
|
|
}
|
|
|
|
Status RenameFile(const std::string& s, const std::string& t) {
|
|
return target_->RenameFile(s, t);
|
|
}
|
|
Status LockFile(const std::string& f, FileLock** l) {
|
|
return target_->LockFile(f, l);
|
|
}
|
|
Status UnlockFile(FileLock* l) { return target_->UnlockFile(l); }
|
|
void Schedule(void (*f)(void*), void* a, Priority pri) {
|
|
return target_->Schedule(f, a, pri);
|
|
}
|
|
void StartThread(void (*f)(void*), void* a) {
|
|
return target_->StartThread(f, a);
|
|
}
|
|
void WaitForJoin() { return target_->WaitForJoin(); }
|
|
virtual unsigned int GetThreadPoolQueueLen(Priority pri = LOW) const {
|
|
return target_->GetThreadPoolQueueLen(pri);
|
|
}
|
|
virtual Status GetTestDirectory(std::string* path) {
|
|
return target_->GetTestDirectory(path);
|
|
}
|
|
virtual Status NewLogger(const std::string& fname,
|
|
shared_ptr<Logger>* result) {
|
|
return target_->NewLogger(fname, result);
|
|
}
|
|
uint64_t NowMicros() {
|
|
return target_->NowMicros();
|
|
}
|
|
void SleepForMicroseconds(int micros) {
|
|
target_->SleepForMicroseconds(micros);
|
|
}
|
|
Status GetHostName(char* name, uint64_t len) {
|
|
return target_->GetHostName(name, len);
|
|
}
|
|
Status GetCurrentTime(int64_t* unix_time) {
|
|
return target_->GetCurrentTime(unix_time);
|
|
}
|
|
Status GetAbsolutePath(const std::string& db_path,
|
|
std::string* output_path) {
|
|
return target_->GetAbsolutePath(db_path, output_path);
|
|
}
|
|
void SetBackgroundThreads(int num, Priority pri) {
|
|
return target_->SetBackgroundThreads(num, pri);
|
|
}
|
|
void LowerThreadPoolIOPriority(Priority pool = LOW) override {
|
|
target_->LowerThreadPoolIOPriority(pool);
|
|
}
|
|
std::string TimeToString(uint64_t time) {
|
|
return target_->TimeToString(time);
|
|
}
|
|
|
|
private:
|
|
Env* target_;
|
|
};
|
|
|
|
// Returns a new environment that stores its data in memory and delegates
|
|
// all non-file-storage tasks to base_env. The caller must delete the result
|
|
// when it is no longer needed.
|
|
// *base_env must remain live while the result is in use.
|
|
Env* NewMemEnv(Env* base_env);
|
|
|
|
} // namespace rocksdb
|
|
|
|
#endif // STORAGE_ROCKSDB_INCLUDE_ENV_H_
|