ARCHIVE/xahaud
3
0
Fork 0
mirror of https://github.com/Xahau/xahaud.git synced 2025-11-30 07:25:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Squashed 'src/rocksdb2/' changes from 1fdd726a8..aead40417

Browse Source 
aead40417 fix HISTORY.md typo
a0cdc3cec Bump version to 5.8.7
7513f6350 Fix IOError on WAL write doesn't propagate to write group follower
9e47084ce Bump version to 5.8.6
36074ba5d Enable cacheline_aligned_alloc() to allocate from jemalloc if enabled.
aa00523e0 Add -DPORTABLE=1 to MSVC CI build
cf2b98237 Bump version to 5.8.5
e8c9350f2 Blob DB: not using PinnableSlice move assignment
4907d2463 Bump version to 5.8.4
5d928c795 Blob DB: Fix race condition between flush and write
725bb9d66 Blob DB: Fix release build
b7367fe84 Bump version to 5.8.3
13b2a9b6f Blob DB: use compression in file header instead of global options
5dc70a15c Fix PinnableSlice move assignment
9019e9125 dynamically change current memtable size
7f1815c37 Bump version to 5.8.2
2584a18ef Blob DB: Fix BlobDBTest::SnapshotAndGarbageCollection asan failure
17f67b546 PinnableSlice move assignment
6fb56c582 Blob DB: Add compaction filter to remove expired blob index entries
f90ced92f Blob DB: fix snapshot handling
632f36dcd Blob DB: option to enable garbage collection
11bacd578 Blob DB: Fix flaky BlobDBTest::GCExpiredKeyWhileOverwriting test
f98efcb1e Blob DB: Evict oldest blob file when close to blob db size limit
c1e99eddc Blob DB: cleanup unused options
ffc3c62ca Blob DB: Initialize all fields in Blob Header, Footer and Record structs
9e8254090 Blob DB: update blob file format
d66bb21e1 Blob DB: Inline small values in base DB
05d5c575a Return write error on reaching blob dir size limit
2b8893b9e Blob DB: Store blob index as kTypeBlobIndex in base db
419b93c56 Blob DB: not writing sequence number as blob record footer
8afb0036c fix lite build
dded348dd Blob DB: Move BlobFile definition to a separate file
374736123 add GetLiveFiles and GetLiveFilesMetaData for BlobDB
8cff6e945 Enable WAL for blob index
c29347290 Add ValueType::kTypeBlobIndex
eae53de3b Make it explicit blob db doesn't support CF
65aec19df Fix memory leak on blob db open
30b38c98c TableProperty::oldest_key_time defaults to 0
2879f4beb Bump version to 5.8.1
88595c882 Add DB::Properties::kEstimateOldestKeyTime
266ac245a Bumping version to 5.8
64185c23a update HISTORY.md for DeleteRange bug fix
e83d6a02e Not using aligned_alloc with gcc4 + asan
0980dc6c9 Fix wrong smallest key of delete range tombstones
b76797231 avoid use-after-move error
c41744270 CMake: Fix formatting
c21ea8f7a CMake: Add support for CMake packages
544434558 add Erlang to the list of language bindings
2972a702a Minor updates to FlushWAL blog
fbfa3e7a4 WriteAtPrepare: Efficient read from snapshot list
b01f426f5 Blog post for FlushWAL
503db684f make blob file close synchronous
3c840d1a6 Allow DB reopen with reduced options.num_levels
92bfd6c50 Fix DropColumnFamily data race
7fdf735d5 Pinnableslice examples and blog post
7fbb9ecca support disabling checksum in block-based table
19cc66dc4 fix clang bug in block-based table reader
7eba54eb9 test compaction input-level split range tombstone assumption
cd26af347 Add unit test for WritePrepared skeleton
a12479819 Improved transactions support in C API
c10b39131 LANGUAGE-BINDINGS.md: add another rust binding
90177432e Remove leftover references to phutil_module_cache
234f33a3f allow nullptr Slice only as sentinel
ccf7f833e Use PinnableSlice in Transactions
1dfcdb15f Extend pin_l0 to filter partitions
39ef90055 stop calling memcmp with nullptrs
78cb6b611 Provide byte[] version of SstFileWriter.merge to reduce GC Stall
867fe92e5 Scale histogram bucket size by constant factor
f004307e9 CMake improvements
09ac6206a Circumvent ASAN false positive
5b68b114f Blob db create a snapshot before every read
4624ae52c GC the oldest file when out of space
8ace1f79b add counter for deletion dropping optimization
0d8e992b4 Revert the mistake in version update
5358a8056 add VerifyChecksum to HISTORY.md
ed0a4c93e perf_context measure user bytes read
1efc600dd Preload l0 index partitions
bddd5d363 Added mechanism to track deadlock chain
c1384a707 fix db_stress uint64_t to int32 cast
29877ec7b Fix blob db crash during calculating write amp
8f2598ac9 Enable Cassandra merge operator to be called with a single merge operand
9a44b4c32 Allow merge operator to be called even with a single operand
ac8fb77af fix some misspellings
23593171c minor improvements to db_stress
af012c0f8 fix deleterange with memtable prefix bloom
1c8dbe2aa update scores after picking universal compaction
eb6425303 Update WritePrepared with the pseudo code
132306fbf Remove PartialMerge implementation from Cassandra merge operator
71598cdc7 Fix false removal of tombstone issue in FIFO and kCompactionStyleNone
3204a4f64 Fix missing stdlib include required for abort()
7aa96db7a db_stress rolling active window
dfa6c23c4 Update RocksDBCommonHelper to use escapeshellarg
e367774d1 Overload new[] to properly align LRUCacheShard
ad42d2fcb Remove residual arcanist_util directory
279296f4d properly set C[XX]FLAGS during CMake configure-time checks
c5f0c6cc6 compile with correct flags to determine SSE4.2 support
185ade4c0 cmake: support more compression type
5449c0990 rocksdb: make buildable on aarch64
a144a9782 Fix for CMakeLists.txt on Windows for RocksJava
acf935e40 fix deletion dropping in intra-L0
8254e9b57 make sst_dump compression size command consistent
74f18c130 db_bench support for non-uniform column family ops
5de98f2d5 approximate histogram stats to save cpu
3f5888430 Fix c_test ASAN failure
e5a1b727c Fix blob DB transaction usage while GC
6f051e0c7 fix corruption_test valgrind
ac098a462 expose set_skip_stats_update_on_db_open to C bindings
666a005f9 Support prefetch last 512KB with direct I/O in block based file reader
ad77ee0ea Revert "Makefile: correct faligned-new test"
b87ee6f77 Use more keys per lock in daily TSAN crash test
25df24254 Add column families related functions (C API)
64f848435 block_cache_tier: fix gcc-7 warnings
0cecf8155 Write batch for `TransactionDB` in C API
6a9de4347 Windows.h macro call fix
23c7d1354 fix comment
1fbad84b6 Makefile: correct faligned-new test
7848f0b24 add VerifyChecksum() to db.h
47ed3bfc3 fix WinEnv assertions
d97a72d63 Try to repair db with wal_dir option, avoid leak some WAL files
36375de76 gcc-7/i386: markup intentional fallthroughs
bdc056f8a Refactor PessimisticTransaction
a9a4e89c3 Fix valgrind complaint about initialization
4ca11b4b0 Update USERS.md
c9804e007 Refactor TransactionDBImpl
20dc5e74f Optimize range-delete aggregator call in merge helper.
0d4a2b733 Avoid blob db call Sync() while writing
627c9f1ab Don't add -ljemalloc when DISABLE_JEMALLOC is set
dce6d5a83 db_bench background work thread pool size arguments
4f81ab38b Makefile: fix for GCC 7+ and clang 4+
92afe830f Update all blob db TTL and timestamps to uint64_t
5883a1ae2 Fix /bin/bash shebangs
cc01985db Introduce bottom-pri thread pool for large universal compactions
0b814ba92 Allow concurrent writes to blob db
2c45ada4c Blob DB garbage collection should keep keys with newer version
58410aee4 Fix the overflow bug in AwaitState
c3d5c4d38 Refactor TransactionImpl
060ccd4f8 support multiple CFs with OPTIONS file
345387067 Fix statistics in RocksJava sample
1900771bd Dump Blob DB options to info log
3218edc57 Fix universal compaction bug
6a36b3a7b fix db get/write stats
a84cee812 Add a missing "once" in .h
21696ba50 Replace dynamic_cast<>
e85f2c64c Prevent empty memtables from using a lot of memory
ac748c57e Fix FIFO Compaction with TTL tests
aaf42fe77 Move blob_db/ttl_extractor.h into blob_db/blob_db.h
aace46516 Fix license headers in Cassandra related files
50a969131 CacheActivityLogger, component to log cache activity into a file
6083bc79f Blob DB TTL extractor
710411aea fix asan/valgrind for TableCache cleanup
3a3fb00b7 TARGETS file not setting sse explicitly
fca4d6da1 Build fewer tests in Travis platform_dependent tests
8f553d3c5 remove unnecessary internal_comparator param in newIterator
7f6d012d7 "ccache -C" in Travis
d12691b86 move TableCache::EraseHandle outside of db mutex
f33f11368 fix db_bench argument type
e7697b8ce Fix LITE unit tests
3ce20e985 Fix use of RocksDBCommonHelper in cont_integration.sh
c281b4482 Revert "CRC32 Power Optimization Changes"
9980de262 Fix FIFO compaction picker test
2289d3811 CRC32 Power Optimization Changes
30b58cf71 Remove the orphan assert on !need_log_sync
fe1a5559f Fix flaky write_callback_test
addbd279c 5.6.1 release blog post
30edff308 buckification: remove explicit `-msse*` compiler flags
2b259c9d4 Lower num of iterations in DeadlockCycle test
277f6f23d Release note for partitioned index/filters
5e731a138 Remove unused rocksdb arcanist lib
9b11d4345 Fix broken links
06f191744 add vcpkg as an windows option
ea8ad4f67 Fix compaction div by zero logging
34112aeff Added db paths to c
1d8aa2961 Gcc 7 ParsedInternalKey replace memset with clear function.
a4c42e800 Fix UBSAN issue of passing nullptr to memcmp
16e038820 LRUCacheShard cache line size alignment
216644c61 enable UBSAN macro in TARGETS
e67b35c07 Add Iterator::Refresh()
a34b2e388 Fix caching of compaction picker's next index
72502cf22 Revert "comment out unused parameters"
1d7048c59 comment out unused parameters
534c255c7 Cassandra compaction filter for purge expired columns and rows
63163a8c6 Remove make_new_version.sh
0302da47a Reduce blob db noisy logging
3e5ea29a8 Fix Flaky DeleteSchedulerTest::ImmediateDeleteOn25PercDBSize
a22b9cc6f overlapping endpoint fixes in level compaction picker
ffd2a2eef delete ExpandInputsToCleanCut failure log
3e6e863b1 Remove arcanist_util directory
36651d14e Moving static AdaptationContext to outside function
6e3ee015f Update java/rocksjni.pom
ecff9d5e3 Include write_buffer_manager in ImmutableDBOptions::Dump
ae28634e9 Remove some left-over BSD headers
33b1de82a Remove format compatibility hack
2f375154e checkout local branch in check_format_compatible.sh
ddb22ac59 avoid collision with master branch in check format
0c03a7f17 set the remote for git checkout
7ac184c6d Revert cmake -DNDEBUG for non-MSVC
0655b5858 enable PinnableSlice for RowCache
00464a314 Fix column_family_test with LITE build
b2dd192fe tools/write_stress.cc: Correct "1204" typos.
cbaab3044 table/block.h: change memset
f1a056e00 CodeMod: Prefer ADD_FAILURE() over EXPECT_TRUE(false), et cetera
4a2e4891f Add back the LevelDB license file
a7321fc97 Remove the licensing description in CONTRIBUTING.md
3c327ac2d Change RocksDB License
132013366 Make TARGETS file portable
ccf5f08f8 Set CACHE_LINE_SIZE for s390, PPC, ARM64
67510eeff db_crashtest.py: remove need for shell
4267eb00d Remove punit tests
5bfb67d90 Enable write rate limit for updaterandom benchmark
20a691d98 Update HISTORY to release 5.7
98d1a5510 db_bench to by default verify checksum
26ce69b19 Update blob db to use ROCKS_LOG_* macro
43e4eef77 remove unnecessary fadvise
21b17d768 Fix BlobDB::Get which only get out the value offset
70440f7a6 Add virtual func IsDeleteRangeSupported
7550255c5 Add JAVAC_ARGS as Makefile variable
7a0b5de77 Gcc 7 ignored quantifiers
000bf0af3 Improve the design and native object management of Stats in RocksJava
269d383d5 Bump version to 5.7
c32f27223 Fixes db_bench with blob db
fcd99d27c db_bench_tool: fix buffer size
87128bd5c fix regression test
8f927e5f7 Fix undefined behavior in Hash
643b787c7 Added a note about LZ4 compression dependency
56656e12d Temporarily disable FIFOCompactionWithTTLTest
b5fb85ec5 fix valgrind init complaint
657df29ea Add max_background_jobs to db_bench
a43c053ad remove duplicated utilities/merge_operators/cassandra/test_utils.cc in src.mk
7c4a9e6c9 Initialize a variable in ldb to make code analysis tool happy
98669b535 init filters_in_partition_
0013bf14e fix asan and valgrind leak report in test
521b4c28b rocksdb 5.5.1 release post
33042573d Fix GetCurrentTime() initialization for valgrind
f6b9d9355 Fix clang error in PartitionedFilterBlockBuilder
45b9bb033 Cut filter partition based on metadata_block_size
f4ae1bab0 update history for OnBackgroundError and DeleteRange fix
1cb8c6de6 Add -enable_pipelined_write to db_bench and add two defaults
7604b463b Update the AddDBStats in LITE
1e34d07e1 Simplify and document sync rules for logs_ etc
d310e0f33 Regression test for empty dedicated range deletion file
e9f91a517 Add a fetch_add variation to AddDBStats
c1b375e96 skip generating empty sst
67b417d62 fix format compatible test
afbef6518 Bug fix: Fast CRC Support printing is not honest
397ab1115 Improve Status message for block checksum mismatches
18c63af6e Make "make analyze" happy
01534db24 Fix the reported asan issues
1cd45cd1b FIFO Compaction with TTL
982cec22a Fix TARGETS file tests list
b49b37109 allow numa >= 2.0.8
e517bfa2c CLANG Tidy
dc3d2e4d2 update compatible test
89468c01d Fix Windows build broken by 5c97a7c0664d4071768113814e9ba71fe87e18cf
51778612c Encryption at rest support
7061912c2 Trivial typo in HISTORY.md
2a9cd8799 Fix jni WriteBatchThreadedTest
0025a3640 revert perf_context and io_stats to __thread
5c97a7c06 Unit Tests for sync, range sync and file close failures
4cee11f4e Intra-L0 blog post
857e9960b Improve the error message for I/O related errors.
d757355cb Fix bug that flush doesn't respond to fsync result
8e6345d2d Update rename of ParanoidCheck
499ebb3ab Optimize for serial commits in 2PC
0ac4afb97 Sanitize partitioning options
521724ba8 fixed wrong type for "allow_compaction" parameter
71f5bcb73 Introduce OnBackgroundError callback
88cd2d96e Downgrade option sanitiy check level for prefix_extractor
6837a1762 Fix Data Race Between CreateColumnFamily() and GetAggregatedIntProperty()
af1746751 WriteBufferManager will not trigger flush if much data is already being flushed
9467eb614 Fix flush assertion with tsan
048446fc7 Fix cassandra ASAN use-after-free
a21db161c Implement ReopenWritibaleFile on Windows and other fixes
c430d69ee fix coredump for release nullptr
0d278456c default implementation for InRange
cbd825dee Create a MergeOperator for Cassandra Row Value
2c98b06bf Remove pin_slice option by making it the default
c80c6115d add db_bench options for partitioning
6a3377f45 Synchronize statistic enumeration values between statistics.h and java API
53dda8797 Do not run RateLimiterTest.Rate test on Travis+Mac OSX.
ae8571f5c Fix blob db compression bug
7a380deff Update blob_db_test
89ad9f3ad Allow ignoring unknown options when loading options from a file
6b5a5dc5d fixed typo
0f228be3b fixed typo in util/dynamic_bloom.h
c217e0b9c Call RateLimiter for compaction reads
91e2aa3ce write exact sequence number for each put in write batch
6f4154d69 record index partition properties
5d5a28a98 Fix Clang release build broken by 5582123dee8426a5191dfd5e846cea8c676c793c
0175d58c3 Make direct I/O write use incremental buffer
7a270069b GNU C library for struct tm has 2 additional fields.
d713471da Limit trash directory to be 25% of total DB
9bb91e932 Dedup release
27b450165 Update HistogramTypes in the Java API
e97304c68 update history for 5.6
5582123de Sample number of reads per SST file
db818d2d1 Fix RocksDB Lite build with CLANG
a472c4ae4 update 5.5 change log
bc09c8a0d Fix crash in PosixWritableFile::Close() when fstat() fails
6d0f22e42 Fix mock_env.cc uninitialized variable
c2012d488 Java APIs for put, merge and delete in file ingestion
85dace2af Disable DBRangeDelTest::TailingIteratorRangeTombstoneUnsupported for ubsan
d4f7731b6 fix travis error with init time in mockenv
550a1df72 Fix clang errors by asserting the precondition
cc5f9339e Fix concurrency issue with filter_block_set_
2e64f450d bump version to 5.6
afbc2d0d2 Force travis to build with clang on MacOS
b172a3f1a Fix warnings while generating RocksJava documentation
52a7f38b1 WriteOptions.low_pri which can throttle low pri writes if needed
26a8a8071 Switch from CentOS 5 to CentOS 6 for crossbuilding RocksJava
dba9f3722 Fix db_write_test clang/windows build failure
c7662a44a fixed typo
7e8d95cc0 Fix the Java build which was broken by a4d9c02
7e5fac2c3 remove test dir before exit when current regression is running
7f6c02dda using ThreadLocalPtr to hide ROCKSDB_SUPPORT_THREAD_LOCAL from public…
138b87eae Fix interaction between CompactionFilter::Decision::kRemoveAndSkipUnt…
95b0e89b5 Improve write buffer manager (and allow the size to be tracked in block cache)
a4d9c0251 Pass CF ID to MemTableRepFactory
f68d88be5 Fix DBWriteTest::ReturnSequenceNumberMultiThreaded data race
215076ef0 Fix TSAN: avoid arena mode with range deletions
3a8a848a5 account for L0 size in estimated compaction bytes
0fae3f5dd codemod: format TARGETS with buildifier [5/5] (D5092623)
872199606 add checkpoint support for single db in regression test
5a9b4d743 Retire memenv https://github.com/facebook/rocksdb/pull/2082
d6019651b sync internal/external TARGETS
bbaba51bb Add missing index type to C-API
292edfd51 travis: test with xcode8.3 (OS X 10.12)
0dc3040d5 db: avoid `#include`ing malloc and jemalloc simultaneously
9b3ed8350 fix regression test
9c9909bf7 Support ingest file when range deletions exist
ad19eb868 Fixing blob db sequence number handling
51ac91f58 Histogram of number of merge operands
345878a7f update blob_db_test
cbc821c25 change regression rebuild to one level
103d0692e Avoid unsupported attributes when not building with UBSAN
5fd04566c travis: reduce the number of travis builders
2d05002b9 RocksDB 5.4.5 release blog post
7eca90f18 Update RocksDB blog authors
d03c34497 update comment of GetNextFile
f7bb1a006 support merge and delete in file ingestion
c2c62ad4e Reorder variables of ReadOptions
7bb1f5d48 Increase of compaction threads should be logged at info level instead of a warning
6c456ecae Clean zstd files
347e16f83 codemod: replace `headers = AutoHeaders.*` with `auto_headers`
0be636bf7 Fix db_bench build break with blob db
135ee6a3f fix tsan crash data race
a99fb9928 fix column_family_test asan
f41bffb3d travis: clang-3.6 -> clang-4.0
e7612798b update buckifer/TARGETS
bb01c1880 Introduce max_background_jobs mutable option
5a2530462 Fix the CMakeLists for RocksJava
41cbb7274 options.delayed_write_rate use the rate of rate_limiter by default.
506803466 range sync should be enabled
02594b5f1 Fix build errors in blob_dump_tool with GCC 4.8
52d9e5f7b Fix column family seconds_up accounting
7d8207f1f Fix errors in clang-analyzer builds
85b8569ae Fix release build on Linux
69ec8356b Allow SstFileWriter to use the rate limiter
6cc9aef16 New API for background work in single thread pool
9d0a07ed5 Fix rocksdb.estimate-num-keys DB property underflow
578fb0b1d Simple blob file dumper
ac39d6bec Core-local statistics
3e86c0f07 disable direct reads for log and manifest and add direct io to tests
15ba4d6c4 Address MS Visual Studio 2017 issue with autovector
88c818e43 Replace deprecated RocksDB#addFile with RocksDB#ingestExternalFile
228f49d20 Fix data races caught by tsan
4c9d2b104 remove #include port/port.h in public header file
07bdcb91f New WriteImpl to pipeline WAL/memtable write
d746aead1 Suppress clang-analyzer false positive
217b866f4 column_family_test: EnvCounter::num_new_writable_file_ to be atomic
9f839a7f6 keep util/build_version.cc when make clean
7eecd40a4 add emacs tags file - etags
9bbba4fec Remoe unused BlockBasedTable::compaction_optimized_
f5ba131bf Fixed some spelling mistakes
146b7718f Fix mingw compilation with -DNDEBUG
a36220ccf fix unity test
0ebdd7057 fixed typo
8032f4cb3 Remove -pie in TSAN
362ba9b02 Release RocksDB 5.5.0
ba685a472 Support ingest_behind for IngestExternalFile
01ab7b528 Add ROCKSDB_LIBRARY_API macro to a few C APIs, to fix windows build
cb9392a09 add Transactions and Checkpoint to C API
445f1235b s/std::snprintf/snprintf
cd593c283 Fix travis java_test
f720796e2 fixed typo
a48a62d5b define NDEBUG in CMake non-debug builds
1ca723dbd C API: support pinnable get
2ef15b85d Core-local stats blog post
4f9e69ccf fix log err
11c5d4741 cross-platform compatibility improvements
d00433302 Put lib files into suitable path in RPM package
86d549253 Fix build error with blob DB.
254c46800 Fix the RocksJava Release on Windows
7a47b431f Fix .gitignore pattern
fa5a15ceb Make sure that zstd is statically linked correctly in the Java static build
3fa9a39c6 Add GetAllKeyVersions API
1a60982a5 Simplified instructions for CentOS
a5cc7ecec Facility for cross-building RocksJava using Docker
ccd3dddf5 Blog post for partitioned index/filters
b145c34d7 Update blog authors
bbe9ee7dd core-local array type conversions
c2be43430 Build and link with ZStd when creating the static RocksJava build
c61e72c42 Add missing files of blob_db to CMake file
3907c94ff Fix ColumnFamilyTest:BulkAddDrop
cda5fde2d CoreLocalArray class
93949667c update TARGETS
4e83b8001 title: Bulkoading -> title: Bulkloading
d85ff4953 Blob storage pr
492fc49a8 fix readampbitmap tests
be421b0b1 portable sched_getcpu calls
0f559abdb Add NO_UPDATE_BUILD_VERSION option to makefile
3a04a254a Flink state
35df23fe8 Fix suite exclisions
e7cea86f7 Fixes the CentOS 5 cross-building of RocksJava
259a00eac unbiase readamp bitmap
a62096696 port: updated PhysicalCoreID()
df035b682 Print compaction_options_universal.stop_style in LOG file
4897eb250 dont skip IO for filter blocks
3f73d54bb Add C API to set max_file_opening_threads option
0b69e5079 Define CACHE_LINE_SIZE only when it's not defined
2cd00773c Add bulk create/drop column family API
40af2381e Object lifetime in cache
fdaefa030 travis: add Windows cross-compilation
a30a69603 do not read next datablock if upperbound is reached
2d42cf5ea Roundup read bytes in ReadaheadRandomAccessFile
264d3f540 Allow IntraL0 compaction in FIFO Compaction
8c3a180e8 Set lower-bound on dynamic level sizes
7c1c8ce5a Avoid calling fallocate with UINT64_MAX
a45e98a5b max_open_files dynamic set, follow up
6b99dbe04 fix memory alignment with logical sector size
e7ae4a3a0 Max open files mutable
60847a3b0 port: updated PhysicalCoreID()
b551104e0 support PopSavePoint for WriteBatch
498693cf3 Remove orphaned Java classes
5e2ebf2bd travis: add CMake compilation
af6fe69e4 Fix an issue of manual / auto compaction data race
6798d1f3b Revert "Delete filter before closing the table"
89833577a Delete filter before closing the table
47a09b0a8 Avoid pinning when row cache is accessed
aeaba07b2 Remove an assert that causes TSAN failure.
0b90aa951 Set VALGRIND_VER
a2b05210e Make PosixLogger::flush_pending_ atomic
da4b2070b Fix WriteBatchWithIndex address use after scope error
d616ebea2 Add GPLv2 as an alternative license.
4439b4596 Add documentation license
0ca3ead0c add GetRootDB() in DeleteFilesInRange
cdad04b05 Remove double buffering on RandomRead on Windows.
e15382c09 Disable two flaky tests
2150cc1f3 fix WritableFile buffer size in direct IO
efc361ef7 Add user stats Reset API
6616e4d62 add prefetch to PosixRandomAccessFile in buffered io
f6a27d0bc Extract statistics tests into separate file
7eddecce1 support bulk loading with universal compaction
3b4d1b7a4 add <sys/sysmacros.h> to avoid warning with glibc 2.25
e5e545a02 Reunite checkpoint and backup core logic
72c21fb3f call GetRootDB() before cast to DBImpl* in CancelAllBackgroundWork
4c9447d88 Add erase option to release cache
04d58970c AIX and Solaris Sparc Support
afff9951e Respect deprecated flag in table options
066cfbacc Adding -noprofile to CMakeLists for Windows
cb885bccf set compaction_iterator earliest_snapshot to max if no snapshot
7534ba7bd StackableDB should pass ResetStats()
c1fbf91b2 Fixing Solaris Sparc crash due to cached TLS
963eeba48 Revert how check_format_compatible.sh checkout release branches.
97005dbd5 tools/check_format_compatible.sh to cover option file loading too
8f6196788 Add cpu usage to regression benchmarks (4th attempt)
df74b775e Limit backups opened
1dd776051 Change L0 compaction score using level size
927bbab25 Revert "Add cpu usage to regression benchmarks (3rd attempt)"
8e84a388e Re-add index_per_partition but as deprecated
1553659d6 Add more recent versions to tools/check_format_compatible.sh
966ebb02f Hide event listeners from lite build
476e80be8 Add cpu usage to regression benchmarks (3rd attempt)
c49d70465 Add DB:ResetStats()
0fcdccc33 Blob storage helper methods
a6439d797 CMake: compile with -O2
e67f0adf3 enable O2 optimization for lz4
bc3973259 CMake: add support for SSE4.2
7d5f5aa97 Separate compile and link for shared library
071652734 remove warning
6e8d6f429 readahead backwards from sst end
ca96654d8 Change Build Env to gcc-5
e9e6e5324 Simplify write thread logic
6799c7e00 Pass in remote as a param to branch creation script
44fa8ece9 change use_direct_writes to use_direct_io_for_flush_and_compaction
13b50358f add space for buggy kernel warning
b6b9359ec Fix BYTES_WRITTEN accounting
13369fbd4 Update ShipIt to honor TARGETS updates
f2449ce92 Remove .deprecated_arcconfig
415be221c RocksDB Release 5.4 : Update HISTORY.md and build version.
3eab41d7c java dependencies test -s -> use test -d
a22ed4eab internal_repo_rocksdb to build Java and RocksDB LITE
9f2cc59ec sync TARGETS file
10d754696 set readahead buffer size from roundup(user_size) + 4k to roundup(use…
ba7da434a fix db_stress crash caused by buggy kernel warning
6257837d8 Add ROCKSDB_JAVA_NO_COMPRESSION flag
6a6723ee1 Move MergeOperatorPinning tests to be with other merge operator tests
6a8d5c015 Revert "Report cpu usage using time command"
8f47a9751 File level histogram should be printed per CF, not per DB
9300ef545 Fix shared lock upgrades
1f8b119ed Limit maximum memory used in the WriteBatch representation
97ec8a134 Report cpu usage using time command
20778f2f9 Adding comments to the write path
7124268a0 Reduce the number of params needed to construct DBIter
04abb2b2d FreeBSD only requires WITH_JEMALLOC, not the rest
61730186d dummy diff
360e9960f Summary: Remove .arcconfig
69a5e6461 Deprecate .arcconfig
9690653db Add a verify phase to benchmarks
dd8f9e38e Fix compilation for GCC-5
c2954f9b6 Add Travis job to build examples
72fc1e9d0 avoid non-existent O_DIRECT on OpenBSD
ff9728701 Refactor compaction picker code
9e7293902 only FALLOC_FL_PUNCH_HOLE when ftruncate is buggy
343b59d6e Move various string utility functions into string_util
1d068f606 Fix CompactRange incorrect buffer release
df6f5a377 Move memtable related files into memtable directory
107c5f6a6 CMake: more MinGW fixes
d2dce5611 Move some files under util/ to separate dirs
c50e3750d Use a human readable size for level report
ce64b8b71 Divide db/db_impl.cc
02799ad77 Revert "delete fallocate with punch_hole"
e2a7b202c Release note for partition filters
af256eb2b build db every monday
e5a1372b2 Rework test running script.
d659faad5 Level-based L0->L0 compaction
a12306fab Add a notice on gflags installation in INSTALL.md
43010a929 Revert "[rocksdb][PR] CMake: more MinGW fixes"
a30b75cdc Add buckifier script to github repo
3450ac8c1 CMake: more MinGW fixes
90cfd4645 update IterKey that can get user key and internal key explicitly
e2c6c0636 add TimedEnv
9e4453180 Refactor WriteImpl (pipeline write part 1)
6ef8c620d Move auto_roll_logger and filename out of db/
a1c469d71 Add release notes for PinnableSlice
0537f515c fix run_remote with strong quoting
72e600094 fixed misses on Centos library installation instructions
88cc81df5 auto_roll_logger_test to move away from real sleep
d25e28d58 replace sometimes-undefined uint type with unsigned int
a1d7e487b Add L0 write-amp to compaction level stats
b6d609063 CMake: support AVX2 in MinGW
bd7d13835 test remote instead run remote in regression test
c81a805fe test db existence on the remote host
5fc1e6765 add -rf when remove db in regression test
4ab4049f2 gflags has moved to GitHub
4e0065015 make all DB::Get overloads virtual
6401a8b76 Fix build with MinGW
80fe5b385 disable test: DeleteSchedulerTest.DynamicRateLimiting1
a9c86f51b backup garbage collect shared_checksum tmp files
da175f7ec exit with code 2 when there is already a db_bench running in regression test
0ee7f0403 Added missing options to RocksJava
c6d04f2ec Option to fail a request as incomplete when skipping too many internal keys
58179ec4a Cleanup of ThreadStatusUtil structures should use the DB's reference
f3607640a add ldb build to regression test
8a8c96746 Enable Fast CRC32 for Win64
f9813b853 Added SstFileWriter construtor without explicit comparator to JNI api
8d3cb4f20 Added naming of backup engine threads
67d762379 Expose the stalling information through DB::GetProperty()
0fd574926 delete fallocate with punch_hole
41fe9ad75 Hide usage of compaction_options_fifo from lite build
e7731d119 Configure index partition size
69c8d524a Fix jni library name for PowerPC Architecture
34a70859b Fix segmentation fault caused by #1961
8dee8cad9 Enable fifo compaction benchmark to db_bench
a5c8b5434 fix a header include
91b5feb37 Fix Windows Build broken by a recent commit
41ccae6d2 Add C API functions (and tests) for WriteBatchWithIndex
88bb6f6bf non_shm CI should run tests on /tmp
8888de2b1 Update .gitignore file in examples
203136e79 Fix Compilation errors when using IBM Java
f4fce4751 Fix clang compile error - [-Werror,-Wunused-lambda-capture]
a084b26a5 Blog post for releasing 5.2.1
15950fe3a Remove ASSERT_EQ(boolean, ...)
3e56c7e0c make total_log_size_ atomic
909028e21 HISTORY.md for log_size_for_flush in CreateCheckpoint()
be723a8d8 Optionally construct Post Processing Info map in MemTableInserter
e474df947 db_bench: not need to check mmap for PlainTable
8b0097b49 Readers for partition filter
9ef3627fd Allow checkpointing without flushing
17866ecc3 Allow Users to change customized ldb tools' header in help printing
a2a883318 remove deleted option from benchmark.sh
78cb19559 add checkpoint to ldb
4b04addfc updated solution if "make format" command fails
8f5bf0446 Flush triggered by DB write buffer size picks the oldest unflushed CF
6908e24b5 dynamic setting of stats_dump_period_sec through SetDBOption()
93c68b642 change regression bash file with debug mode
21d8c3179 remove LIKELY from public headers
36ad75778 INSTALL: document USE_SSE
9272e12f1 avoid ftruncate twice in buffered io
d52f334cb Break stalls when no bg work is happening
e66221add fix db_bench rate limiter callsites
dbae438a0 Replace "DEPRECATED" comment to "not supported"
995618a82 Support SstFileManager::SetDeleteRateBytesPerSecond()
e19163688 Add macros to include file name and line number during Logging
d525718a9 cleanup direct io flag in WritableFileWriter
5fa927aa9 Add Xpress and ZSTD CompressionType values to C header
11526252c Pinnableslice (2nd attempt)
e5bd8def1 update history.md for fixing the bug that skips keys
1ffbdfd9a Add a new SstFileWriter constructor without explicit comparator
ebd5639b6 Add ability to search for key prefix in sst_dump tool
e6725e8c8 Fix some bugs in MockEnv
900c62be6 fix compile for VS2015
fe1835617 release 5.3
5dae01947 Revert "Report cpu usage using time command"
f2817fb7f avoid ASSERT_EQ(false, ...);
5b11124e3 add max to histogram stats
d43adf21b Report cpu usage using time command
18fc1bc0e minor changes for rate limiter test flakiness
12ba00ea6 Reset DBIter::saved_key_ with proper user key anywhere before pass to DBIter::FindNextUserEntry
c9df05d1e Fix random access alignment
f64991537 Add Bulkoading IngestExternalFile blog post
54b434110 Builders for partition filter
97edc72d3 Add a memtable-only iterator
72202962f fix db_sst_test flakiness
5f65dc877 Expose DB::DeleteRange and WriteBath::DeleteRange in Java
58b12dfe3 Set logs as getting flushed before releasing lock, race condition fix
f8a4ea020 Move db_test and external_sst_file_test out of Travis's MAC OS run
534581a35 Fix a bug in tests in options operator=
a2f7a514d Refactoring
2a5daa06f Add stderr log level for ldb backup commands
4561275c2 fix rate limiter test flakiness
7c80a6d7d Statistic for how often rate limiter is drained
0ad5af42d Clarify VerifyBackup behavior
6fb901344 sanitize readahead when direct read enabled
f89b3893c Remove skip_table_builder_flush and default it to true
cc253982d Use more default options in db_bench
8432bcf55 Make compaction_pri settable through option string
d5b607a43 Make db_wal_test slightly faster
ba4c77bd6 Divide external_sst_file_test
e877afa08 Remove bulk loading and auto_roll_logger in rocksdb_lite
90d835507 Fix the wrong address for PREFETCH in DynamicBloom::Prefetch
08864df21 Move advanced column family options to advanced_options.h
2ca2059f6 Get unique_ptr to use delete[] for char[] in DumpMallocStats
253799c06 Add missing include for `abort()`
c6d464a9d Fixed various memory leaks and Java 8 JNI Compatibility
be3e5568b Fix unaligned reads in read cache
8ad0fcdf9 Separate small subset tests in DBTest
6c951c43c Run fewer tests in OSX
f7997f134 add direct I/O to version notes 5.2.0
3b8ba703c Fix flaky DBTestUniversalCompaction.UniversalCompactionTrivialMoveTest2
96c7e1504 Fix Java build
e0b87afc7 Black list some slow valgrind tests
e67232cff Handle failed Finish() in SST file writer
8efb5ffa2 [rocksdb][PR] Remove option min_partial_merge_operands and verify_checksums_in_comp…
1ba2804b7 Remove XFunc tests
e7d902e69 add direct_io and compaction_readahead_size in db_stress
1ef5f50e8 detect logical sector size
ed50308d2 check backup directory exists before listing children
4d7c06ced Make WriteBatchWithIndex moveble
5040414e6 Gracefully handle previous backup interrupted
f206af56f add use_direct_io() to ReadaheadRandomAccessFile
082493442 truncate patch
286a36db7 posix writablefile truncate
f0879e4c3 Page size isn't always 4k on linux
18eeb7b90 Fix interference between max_total_wal_size and db_write_buffer_size checks
1560b2f5f Temporarly return deprecated functions to fix MongoRocks build
2a0f3d0de level compaction expansion
ebc8a7998 alignment is on in ReadaheadRandomAccessFile::Read()
381fd3224 Remove timeout_hint_us from WriteOptions
fce7a6e19 Fail IngestExternalFile when bg_error_ exists
a618a16f4 New subcode for IOError to detect the ESTALE errno
7ab005183 Remove deprecated DB::AddFile and DB::CompactRange
401667366 Adding Dlang to the list
756c5924e Allow adding external v1 sst file with no global seqno support
aa0298fa9 Remove trailing whitespace from examples Makefile
db2b4eb50 avoid direct io in rocksdb_lite
43e9f01c2 Fix repair_test on ROCKSDB_LITE
7106a994f Use monotonic time points in write_controller.cc and rate_limiter.cc
c2247dc1c Make DBImpl::has_unpersisted_data_ atomic
eb912a927 Remove disableDataSync option
0227c16d6 Update static library versions and add checksums
b3aae4d07 Add repair_test to make check
421ce7c2b Add support for JNI Library on Linux on PowerPC.
9afa20cf2 Increase build version and HISTORY.md for releasing 5.2
a5adda064 Fix repair issues
b48e4778b Consolidate file cutting logic in compaction loop
ac2a77a74 Announce the experimetnal two-level index feature in HISTORY.md
c4a37dcb4 Print the missed last layer in cfstats
a12818afc Blog post for 5.1.2 release
3b4ac8076 Clarify ldb column family argument
d70ce7ee0 Move db_bench flags out of unnamed namespace
186c7eedb Remove incorrect statistics warning
53bb01516 [rocksdb][PR] compaction_style and compaction_pri should output their value as a st…
69d5262c8 Two-level Indexes
0a4cdde50 Windows thread
1aaa898cf Adding GetApproximateMemTableStats method
9fc23c55f Use gcc-4.9-glibc-2.20-fb python in precommit_checker
b797e4215 Dump compression dictionary meta-block
036d668b1 Fix wrong result in data race case related to Get()
574b543f8 Rename merger.h -> merging_iterator.h
add8b50cc Move ThreadLocal implementation into .cc
71d2496af Fix arc setting for Facebook internal tools
f289d9f4a Fix OSX build break after the fallocate change
4a3e7d320 Change the default of delayed slowdown value to 16MB/s
0513e21f9 RangeSync() should work with ROCKSDB_FALLOCATE_PRESENT not set
8b369ae5b Cleaner default options using C++11 in-class init
ec79a7b53 Dedup code in option.cc and db_options.cc
b96372dea improving the C wrapper
04c4ec41d Change corruption_test to use 4 bits.
2d75cd40d NewLRUCache() to pick number of shard bits based on capacity if not given
f25f1ec60 Add test DBTest2.GetRaceFlush which can expose a data race bug
37d4a79e9 Deserialize custom Statistics object in db_bench
3b35134e4 Avoid cache lookups for range deletion meta-block
94a0c32e7 Fix LRU Ref() for handles with external references only
17c118060 Generalize Env registration framework
07dddd5f7 EnvPosixTestWithParam should wait for all threads to finish
5dad9d6d2 Avoid logs_ operation out of DB mutex
a7b13919b Fix CompactFiles() bug when used with CompactionFilter using SuperVersion
616a1464e Fix DeleteRange including sentinels in output files
c918c4b76 Update USERS.md add user Pika
03ca2ac8a Remove function from DBImpl that are not used anywhere
b0029bc7f Test merge op covered by range deletion in memtable
d438e1ec1 Test range deletion block outlives table reader
fba726e55 Version librocksdb.so
9da4d542f Range deletions unsupported in tailing iterator
f2b4939da fixed typo
973f1b78f memtable: delete merge value for range deleteion
aebfd1703 fix non-portable behavior in encoder
753ff84a3 Fix get approx size
d7ea44f2f Fixup a couple of builds errors on Linux.
537da370d c: allow set savepoint to writebatch
af6ec4d78 fix batchresult handle leak
e29bb934f Zlib 1.2.8 is no longer available, switched to 1.2.10
5ac97314e Fix std::out_of_range when DBOptions::keep_log_file_num is zero
4e35ffdfa cmake: check -momit-leaf-frame-pointer before using it
3c0852d1d Make ingest external file backward compatible
0e8dfd606 Fix OptimizeForPointLookup()
e840213d6 Change DB::GetApproximateSizes for more flexibility needed for MyRocks
9239103cd Flush job should release reference current version if sync log failed
da54d36a9 Disable IngestExternalFile in ReadOnly mode
5cf176ca1 Fix for 2PC causing WAL to grow too large
4a73bb0b4 Split travis jobs
c70d3c7ad Enable DBTest.GroupCommit as it runs in a reasonlable time now.
602c13a96 Remove fadvise with direct IO read
f9d18e22d Fix DeleteRange file boundary correctness issue with max_compaction_bytes
3ce091fd7 Add KEEP_DB env var option
77b480662 Fix 2PC with concurrent memtable insert
e8a096000 util/thread_local.h: silence a clang-build warning
324a0f988 Follow up for DirectIO refactor
bc5d7b702 travis: For linux, do all tests under gcc
3e6899d11 change UseDirectIO() to use_direct_io()
d4e07a845 fix warning of unused direct io helper functions
dc2584eea direct reads refactor
d18dd2c41 Abort compactions more reliably when closing DB
62384ebe9 Guarding extra fallocate call with TRAVIS because its not working pro…
9f246298e Performance: Iterate vector by reference
fe395fb63 Allow incrementing refcount on cache handles
2172b660e Fix build on FreeBSD
3c233ca4e Fix Windows environment issues
763173456 Fix the error in ColumnFamiliesTest
7a02ad070 Update travis to ubuntu trusty
60c509ff1 Fix valgrind failure in test CurrentFileModifiedWhileCheckpointing2PC
d0ba8ec8f Revert "PinnableSlice"
54d94e9c2 PinnableSlice
e04480fae Fix MS warnings. Use ROCKSDB_Prsz for size_t.
c081f7215 5.0.1 release blog post
ac73d7558 Add GetSupportedCompressions() convenience function
b104b8781 Maintain position in range deletions map
640d72480 Update db_bench and sst_dump to test with block cache mid-point inser?
653ac1f9c C API: support total_order_mode
85ac1a320 Fix rocksdb::Status::getState
76711b6e7 Make ExternalSSTFileTest::CompactionDeadlock more deterministic
c963460db Fix tests under GCC_481
33c86d677 Fix backupable db test
e425ec116 utilities/backupable: backup should limit the copy size of wal.
0712d541d Delegate Cleanables
d58ef52ba Allow SstFileWriter to Fadvise the file away from page cache
17a4b75cc Always fsync the file after file copying
2fb70dc79 examples: Add options_file_example to target all
a738af8f8 db/pinned_iterators_manager.h: bugfix
906523d98 Add description to the 2PC checkpooint bug in HISTORY.md
438f22bc5 Fix bug of Checkpoint loses recent transactions with 2PC
335981d47 Fix the directory path for RocksDB repo
548b62805 Enable conditionally using adaptive mutexes
4e07b08ef include/rocksdb/utilities/env_librados: fix typo
ab48c165a Print cache options to info log
972f96b3f direct io write support
989e644ed Remove sst_file_manager option from LITE
1beef6569 Fix c_test
3d692822f persistent_cache: fix two timer
046099c9b The array is malloced by backtrace_symbols(), and must be freed
6ff2c8d7f Remove gflags as travis build dependency
3cd9ed1c3 Show sandcastle URL in phabricator
50e305de9 Collapse range deletions
5d1457dbb Dump persistent cache options
7bd725e96 db_bench: introduce --benchmark_read_rate_limit
296691847 Update Netflix section of USERS.md
342370f1d Simplify MemTable::Update
1a136c1f1 Expose file size
fbff4628a Reduce compaction iterator status checks
bd6cf7b51 WritableFileWriter: default buffer size equal min(64k,options.writabl?
fc0c6fd98 "make format" format diff since last commit from master
816c1e30c gcc-7 requires include <functional> for std::function
c27073586 Iterator should be in corrupted status if merge operator return false
a8bf4d63f Make format shows wrong curl command for retrieving clang-format-diff.py
8f5d24ae6 C API: support get usage and pinned_usage for cache
0ab6fc167 Gcc-7 buffer size insufficient
b7239bf7e Gcc 7 fallthrough
477b6ea57 std::remove_if requires <algorithm>
83f9a6fd2 Fail BackupEngine::Open upon meta-file read error
a79eae4b0 Add pcache documentation images
d71e728c7 Print user collected properties in sst_dump
7004a6f7b Add missing copyright header
3cdfaeca3 Fixes for MSVC compilation
e097222e6 util/logging.cc: buffer of insufficient size (gcc-7 -Werror=format-length)
cfc34d7c4 Missing break in case in DBTestBase::CurrentOptions
bfbcec233 Gcc 7 error expansion to defined
6653e32ac build: make it easier to pass PORTABLE
67adc937b intentional fallthough (prevents gcc-7/clang-4 error)
1a146f89c break Flush wait for dropped CF
c3e5ee715 util/histogram.cc: HistogramStat::toString buffer insufficient
5334d8b44 table/block_based_table_builder.cc: intentional fallthrough - comment to match gcc pattern
36d42e65d Disable test to unblock travis build
b57dd9262 C API: support writebatch delete range
2ba59b5a1 Disallow ingesting files into dropped CFs
1f6f7e3e8 cast to signed char in ldb_cmd_test for ppc64le
243975d5d More accurate error status for BackupEngine::Open
f0c509e2c Return finer-granularity status from Env::GetChildren*
dc64f46b1 Add db_bench option for stderr logging
2cabdb8f4 Increase buffer size
4a17b47bb Remove unnecessary header include
8c2b921fd Fixed a crash in debug build in flush_job.cc
20ce081fa Fix issue where IngestExternalFile insert blocks in block cache with g_seqno=0
5241e0dbf fix db_bench argument type
c04f6a0b4 Specify shell in makefile
45c7ce137 CompactRangeOptions C API
2c2ba6824 db_stress support for range deletions
b821984d3 DeleteRange read path end-to-end tests
2f4fc539c Compaction::IsTrivialMove relaxing
1dce75b2e Update USERS.md
304b3c706 Update USERS.md
fa50fffaf Option to expand range tombstones in db_bench
c26a4d8e8 Fix compile error in trasaction_lock_mgr.cc
ed8fbdb56 Add EventListener::OnExternalFileIngested() event
2005c88a7 Implement non-exclusive locks
0b0f23572 Mention IngestExternalFile changes in HISTORY.md
23db48e8d Update HISTORY.md for 5.0 branch
beb36d9c1 Fixed CompactionFilter::Decision::kRemoveAndSkipUntil
67f37cf19 Allow user to specify a CF for SST files generated by SstFileWriter
9053fe2a5 Made delete_obsolete_files_period_micros option dynamic
edde954e7 fix clang build
56281f3a9 Add memtable_insert_with_hint_prefix_size option to db_bench
4a21b1402 Cache heap::downheap() root comparison (optimize heap cmp call)
e39d08087 Fix travis (compile for clang < 3.9)
3f407b065 Kill flashcache code in RocksDB
b77007df8 Bug: paralle_group status updated in WriteThread::CompleteParallelWorker
247d0979a Support for range skips in compaction filter
96fcefbf1 c api: expose option for dynamic level size target
00197cff3 Add C API to set base_backgroud_compactions
5b219eccb deleterange end-to-end test improvements for lite/robustness
aad119176 pass rocksdb oncall to mysql_mtr_filter otherwise tasks get created w…
e33352899 DeleteRange write path end-to-end tests
7784980fc Fix mis-reporting of compaction read bytes to the base level
3c6b49ed6 Fix implicit conversion between int64_t to int
b3b875657 Remove unused assignment in db/db_iter.cc
4f6e89b1d Fix range deletion covering key in same SST file
a2bf265a3 Avoid intentional overflow in GetL0ThresholdSpeedupCompaction
52fd1ff2c disable UBSAN for functions with intentional -ve shift / overflow
1886c435b Fix CompactionJob::Install division by zero
63c30de80 fix options_test ubsan
13e66a8f5 Fix compaction_job.cc division by zero
01eabf737 Fix double-counted deletion stat
7ffb10fc1 DeleteRange compaction statistics
236d4c67e Less linear search in DBIter::Seek() when keys are overwritten a lot
cd7c4143d Improve Write Stalling System
dfb6fe675 Unified InlineSkipList::Insert algorithm with hinting
3068870cc Making persistent cache more resilient to filesystem failures
734e4acaf Eliminate redundant cache lookup with range deletion
182b940e7 Add WriteOptions.no_slowdown
4118e1333 Persistent Cache: Expose stats to user via public API
f2a8f92a1 rocks_lua_compaction_filter: add unused attribute to a variable
4444256ab Remove use of deprecated LZ4 function
548d7fb26 Fix fd leak when using direct IOs
fd43ee09d Range deletion microoptimizations
23a18ca5a Reword support a little bit to more clear and concise
481856ac4 Update support to separate code issues with general questions
a0deec960 Fix deadlock when calling getMergedHistogram
fe349db57 Remove Arena in RangeDelAggregator
e63350e72 Use more efficient hash map for deadlock detection
a13bde39e Skip ldb test in Travis
73843aa63 Direct I/O Reads Handle the last sector correctly.
9d60151b0 Implement PositionedAppend for PosixWritableFile
3f6221521 Lazily initialize RangeDelAggregator's map and pinning manager
41e77b839 cmake: s/STEQUAL/STREQUAL/
c1038d283 Release RocksDB 5.0
635a7bd1a refactor TableCache Get/NewIterator for single exit points
f39452e81 Fix heap use after free ASAN/Valgrind
a4eb7387b Allow plain table to store index on file with bloom filter disabled
36e4762ce Remove Ticker::SEQUENCE_NUMBER
86eb2b9ad Fix src.mk
0765babe1 Remove LATEST_BACKUP file
647eafdc2 Introduce Lua Extension: RocksLuaCompactionFilter
760ef68a6 fix deleterange asan issue
327085b7b fix valgrind
715591bba Ask travis to use JDK 7
972e3ff29 Enable allow_concurrent_memtable_write and enable_write_thread_adaptive_yield by default
420bdb42e option_change_migration_test: force full compaction when needed
1543d5d92 Report memory usage by memtable insert hints map.
018bb2ebf DeleteRange support for db_bench
dc51bd716 CMakeLists.txt: FreeBSD has jemalloc as default malloc
48e8baebc Decouple data iterator and range deletion iterator in TableCache
4b0aa3c4c Fix failed compaction_filter_example and add it into make all
53b693f5f ldb support for range delete
661e4c926 DeleteRange unsupported in non-block-based tables
489d14280 DeleteRange interface
eba99c28e Fix min_write_buffer_number_to_merge = 0 bug
2ef92fea5 Remove all instances of relative_url until GitHub pages problem is fixed.
91300d01f Dynamic max_total_wal_size option
ec2f64794 Consider subcompaction boundaries when updating file boundaries for range deletion
800e51553 Fix CSS issues again
b952c898b Parallize persistent_cache_test and transaction_test
3b192f618 Handle full final subcompaction output file with range deletions
6c5795200 Make range deletion inclusive-exclusive
425210cc4 CSS issues are arising on the Github Pages side. Temp fix.
1ea79a78c Optimize sequential insert into memtable - Part 1: Interface
df5eeb85c Optimize sequential insert into memtable - Part 2: Implementation
5ed650857 Fix SstFileWriter destructor
adb665e0b Allowed delayed_write_rate option to be dynamically set.
307a4e80c sst_dump support for range deletion
361010d44 Exporting compaction stats in the form of a map
672300f47 Use relative Urls for stylesheets
b39b2ee12 do not call get() in recovery mode
1ca5f6d13 Fix 2PC Recovery SeqId Miscount
e095d0cbc Rocksdb contruns to new Sandcastle API
14c0380e7 Convenience option to parse an internal key on command line
c90fef88b fix open failure with empty wal
4e20c5da2 Store internal keys in TombstoneMap
a9fb346e4 Fix RocksDB Lite build failure in c_test.cc
d133b08f6 Use correct sequence number when creating memtable
144cdb8f1 16384 as e.g .value for compression_max_dict_bytes
9bd191d2f Fix deadlock between (WriterThread/Compaction/IngestExternalFile)
a9fae0a9d CSS problems again :(
193221e0a Fix Forward Iterator Seek()/SeekToFirst()
e48f3f8b9 remove tabs and duplicate #include in c api
85bd8f518 Minor fix to GFLAGS usage in persistent cache
a7875272d c: support seek_for_prev
0f17f9279 Make the header links a bit more flexible
cf19f559d single quotes in feed
2dc019e09 Fix header links
f1aedda06 More Jekyll 3.3 fixes
c54cdc378 More Jekyll 3.3 updates
2bcaf8246 Update product and feature template for Jekyll 3.3
24bceb096 Java API - Implement GetFromBatch and GetFromBatchAndDB in WBWI
815f54afa Insert range deletion meta-block into block cache
9e7cf3469 DeleteRange user iterator support
5c5d01ae7 Fix wrong comment (Maximum supported block size)
f998c9790 DeleteRange Get support
879f36636 Add C api for RateLimiter
557034f36 Remove all instances of baseurl
437942e48 Add avoid_flush_during_shutdown DB option
2b16d664c Change max_bytes_for_level_multiplier to double
16fb04434 expose IngestExternalFile to c abi
ce22ea99a Fix casts for MSVC
196af035c Introduce FAIL_ON_WARNINGS CMake variable (default ON)
40a2e406f DeleteRange flush support
d5555d95a Fix MSVC compile error in 32 bit compilation
da61f348d Print compression and Fast CRC support info as Header level
f9eb56791 db_bench: --dump_malloc_stats takes no effect
6a4faee5c fix freebsd build include path err and so & jar file name
c90c48d3c Show More DB Stats in info logs
1b295ac8a DBTest.GetThreadStatus: Wait for test results for longer
25f5742f0 Update documentation to point at gcc 4.8
b50a81a2b Add a test for tailing_iterator
04751d534 L0 compression should follow options.compression_per_level if not empty
2946cadc4 Improve RangeDelAggregator documentation
0a9fd05c2 Update Vagrant file (test internal phabricator workflow)
fcd1e0bf6 Make rocksdb work with internal repo
0aab5e55f FreeBSD: malloc_usable_size is in <malloc_np.h> (#1428)
9c0bb7f17 cmake: drop "-march=native" from CXX_FLAGS (#1429)
eeb27e1bb Add handy option to turn on direct I/O in db_bench (#1424)
c6168d13a removed some declarations from c.h which resulted in undefined symbols (#1407)
bc429de49 revert fractional cascading in farward iterator
b9bc7a2aa Use skiplist rep for range tombstone memtable
60a2bbba9 Makefile: generate util/build_version.cc from .in file (#1384)
9ee84067f Disable DBTest.RepeatedWritesToSameKey (#1420)
f41df3045 OptionChangeMigration() to support FIFO compaction
2e8004e60 Changing the legocastle run to use valgrind_test instead of _check
9de2f7521 revert Prev() in MergingIterator to use previous code in non-prefix-seek mode
24495186d DBSSTTest.RateLimitedDelete: not to use real clock
1168cb810 Fix a bug that may cause a deleted row to appear again
99c052a34 Fix integer overflow in GetL0ThresholdSpeedupCompaction (#1378)
f83cd64c0 Fix a bug that mistakenly disable regression_test.sh to update commit (#1415)
0e926b84f Passing DISABLE_JEMALLOC=1 to valgrind_check if run locally
4dfaa6610 Make IsDeadlockDetect() virtual member of Transaction
59a7c0337 Change ioptions to store user_comparator, fix bug
869ae5d78 Support IngestExternalFile (remove AddFile restrictions)
1d9dbef64 Restrict running condition of UniversalCompactionTrivialMoveTest2
4edd39fda Implement deadlock detection
48fd619a4 Minor fixes to RocksJava Native Library initialization (#1287)
48e4e842b Disable auto compactions in memory_test and re-enable the test (#1408)
fb2e41294 column_family_test: disable some tests in LITE
5af651db2 fix data race in compact_files_test
a0ba0aa87 Fix uninitialized variable gcc error for MyRocks
b88f8e87c Support SST files with Global sequence numbers [reland]
08616b493 [db_bench] add filldeterministic (Universal+level compaction)
52c9808c3 not split file in compaciton on level 0
5e0d6b4cc fix db_stress assertion failure
ab5399837 Bump RocksDB version to 4.13 (#1405)
b4d07123c SamePrefixTest.InDomainTest to clear the test directory before testing
aa09d0338 Avoid calling GetDBOptions() inside GetFromBatchAndDB()
6fbe96baf Compaction Support for Range Deletion
257de78d9 Remove "-Xcheck:jni" from Java tests (#1402)
d88dff4ef add seeforprev in history
5027dd17a Fix a minor bug in the ldb tool that was not selecting the specified (#1399)
fea6fdd67 Fix @see in two Java functions (#1396)
b1031d6c1 Remove function local statics that interfere with memory pooling (#1392)
f47054015 Handle WAL deletion when using avoid_flush_during_recovery
e29d3b67c Make max_background_compactions and base_background_compactions dynamic changeable
21e8daced fix assertion failure in Prev()
b9311aa65 Implement WinRandomRW file and improve code reuse (#1388)
a249a0b75 check_format_compatible.sh to use some branch which allows to run with GCC 4.8 (#1393)
040328a30 Remove an assertion for single-delete in MergeHelper::MergeUntil
8cbe3e10c Relax the acceptable bias RateLimiterTest::Rate test be 25%
f26a139d8 Log successful AddFile
5691a1d8a Fix compaction conflict with running compaction
017de666c fixup commit
1b7af5fb1 Redo handling of recycled logs in full purge
27bfe327b Editorial change to README.md
89cc404de A bit of doc restructuring
9e7fda829 Fix arcanist
2e4b5cab0 Add missing RateLimiter class to the Windows build (#1382)
ce4963fdf [doc] Document that Visual Studio 2015+ is now required for Windows builds (#1389)
e48927098 Fix scoped arena iterator (#1387)
f8d8cf53f Fix log_write_bench -bytes_per_sync option. (#1375)
02b3e3985 Make txn->GetState() const
447f17127 new Prev() prefix support using SeekForPrev()
991b585ee More block cache tickers
d6ae6dec6 Add Statistics::getAndResetTickerCount().
aea3ce4c8 Avoid string CONCAT which is not supported in cmake 2.6 (#1383)
2ad68b971 Support running consistency checks in release mode
67501cfc9 Fix -ve std::string::resize
04b02dd12 Testing asset links after config change
8c55bb87c Make Lock Info test multiple column families
d06232897 Revert "Support SST files with Global sequence numbers"
5cd28833a [RocksJava] Adjusted RateLimiter to 3.10.0 (#1368)
37737c3a6 Expose Transaction State Publicly
2c1f95291 Add facility to write only a portion of WriteBatch to WAL
043cb62d6 Fix record_size in log_write_bench, swap args to std::string::assign. (#1373)
4985f60fc env_mirror: fix a few leaks (#1363)
5aded67dd update of c.h (#1371)
912aec13c "Recent Posts" -> "All Posts"
7cbb298db Make sure that when contribtuing we call out creating appropriate directories
a06ad4711 Add top level doc information to CONTRIBUTING.md
3fdd5b971 A little more generic CONTRIBUTING.md
ed4fc31db Add link to CONTRIBUTING.md to main docs README.md
e4922e181 Forgot to truncate one blog post
6d8cd7ede Add CONTRIBUTING.md for rocksdb.org contribution guidance
bd55e5a1e Fix some formatting of compaction blog post
0f60358b0 CRLF -> LF mod (including removing trailing whitespace for those files)
b90e29c90 Truncate posts on the main /blog/ page
0d7acadaf Add author fields to blog posts
01be44181 Add GitHub link to the landing page header
9d6c96138 Fix Mac build
ab01da543 Support SST files with Global sequence numbers
d346ba246 Minor fixes around Windows 64 Java Artifacts (#1366)
e91b4d0cf Add factory method for creating persistent cache that is accessible from public
be1f1092c Expose transaction id, lock state information and transaction wait information
6009c473c Store range tombstones in memtable
3c21c64c7 Use size hint for HashMap in multiGet. Similar to https://github.com/facebook/rocksdb/pull/1344 (#1367)
13f7a01f6 Fixing JNI release build for gcc (#975)
7260662b3 Add Java API for SstFileWriter
26388247a delete unused variable for PrevInterval()
87dfc1d23 Fix conflict between AddFile() and CompactRange()
eb44ed655 Update 2016-09-28-rocksdb-4-11-2-released.markdown
e4437610d Update 2016-09-28-rocksdb-4-11-2-released.markdown
501f05108 Update 2016-09-28-rocksdb-4-11-2-released.markdown
dec9009f8 Update 2016-09-28-rocksdb-4-11-2-released.markdown
4ed69dd0b Create 2016-09-28-rocksdb-4-11-2-released.markdown
21f4bb5a8 cmake support for linux and osx (#1358)
4defe306f fix typo in comments (#1360)
f517d9dd0 Add SeekForPrev() to Iterator
eb3894cf4 Recompute compaction score on SetOptions (#1346)
5c64fb67d Fix AddFile() conflict with compaction output [WaitForAddFile()]
9e9f5a0b9 Fix CompactFilesTest.ObsoleteFiles timeout (#1353)
c2a62a4cb not cut compaction output when compact to level 0
9ed928e7a Split DBOptions into ImmutableDBOptions and MutableDBOptions
4bc8c88e6 Recover same sequence id from WAL (#1350)
0a1bd9c50 add cfh deletion started listener
da5a9a65c Fix mac build
d45eb6c6d Fix typo (#1349)
abc0ae462 Add AddFile() InternalStats for Total files/L0 files/total keys ingested
715256338 forbid merge during recovery
5735b3dc2 Fix compiling under -Werror=missing-field-initializers
654ed9a28 loose the assertion condition of rate_limiter_test
e4d3f5d9b Fix DBImpl::GetWalPreallocateBlockSize Mac build error
7afbb7420 solve the problem of table_factory_to_write_=nullptr (#1342)
d78a4401b DBImpl::GetWalPreallocateBlockSize() should return size_t
42ac9c5f1 Retry getting arcanist token on failure
b666f8544 Consider more factors when determining preallocation size of WAL files
4c3f4496b Add TableBuilderOptions::level and relevant changes (#1335)
3edb9461b Avoid hard-coded sleep in EnvPosixTestWithParam.TwoPools
0a88f38b7 Remove ColumnFamilyData::options()
41a9070f8 Fix java makefile dependencies
8d9bf5c49 Fix DBOptionsTest.GetLatestOptions
40cfa3e02 Fix DBWALTest.RecoveryWithLogDataForSomeCFs with mac
06b4785fe Fix recovery for WALs without data for all CFs
d7242ff4d Fix GetSortedWalFiles when log recycling enabled
17f76fc56 DB::GetOptions() reflect dynamic changed options
215d12826 Fix typo (#903)
a958c2643 Rename jvalue to jval in rocksjni
0a165bd7d Have Facebook link point to RocksDB on FB
3639f3288 Fix bug in UnScSigned-off-by: xh931076284 <931076284@qq.com> (#1336)
8e061f974 Refactor GetMutableOptionsFromStrings
81747f1be Refactor MutableCFOptions
ba65c816b Support POSIX RandomRWFile
1d980a8e3 Create CNAME
2adab1dde Add API links to the header bar
a182b2981 Preserve blog comments in markdown
f54de9230 Adding Dgraph to list of Users (#1291)
9e4aa798c Summary: (#1313)
a10e8a056 Fix C api memtable rep bugs. (#1328)
eb1d4d53c Release RocksDB 4.12
22d88e24d Allow an offset as well as a length to be specified for byte[] operations in RocksJava JNI (#1264)
b06b19136 add C api for set wal_recovery_mode (#1327)
1cca09129 Temporarily revert Prev() prefix support
de28a2553 Update HISTORY.md for thread-local stats
0fcb6dbed Remove extraneous function prototypes from c.h (#1326)
52ee07b02 Move AddFile() tests to external_sst_file_test.cc
66a91e260 Add NoSpace subcode to IOError (#1320)
67036c040 Fix Flaky ColumnFamilyTest.FlushCloseWALFiles
0e2da497c fix typo in option.h's comment (#1321)
6d61358a0 Add real Google Analytics ID
2d9d36ea4 Have "Edit on GitHub" point to master instead of gh-pages
937751898 Update landing page content
1ec75ee76 Add redirects from old blog posts link to new format
607628d34 Support ZSTD with finalized format
ce1be2ce3 Fix build error on Windows (AppVeyor) (#1315)
f7669b40b Fix Windows Build
22696b088 Fix uninitlized CompactionJob::SubcompactionState::current_output_file_size
c1865e0f7 Trigger more tests per diff
a88677d2c Remove ImmutableCFOptions from public API
80c75593e Fix data race in AddFile() with multiple files + custom comparator bug
5051755e3 Fix db_bench memory use after free (detected by clang_analyze)
4fd08f4b8 Ensure Correct Behavior of StatsLevel kExceptDetailedTimers and kExceptTimeForMutex  (#1308)
e14fbaae2 Add FAQ based on the front page of the current rocksdb.org
3c2262400 Migrate the RocksDB Worpdress blog over to Jekyll
ee0e2201e Transfer the current Getting Started contents to GitHub Pages
5a0e9a4cf Initial Landing Page
9447a8540 Remove the `doc` directory
32149059f Merge options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor into max_compaction_bytes
4590b53a4 add stats to Cache::LookUp()
85bb30825 Expose Utility function StringToMap() (#1306)
8ce1b8440 Fix Travis on Mac
380e651af Fix Mac build failure (#1309)
1613fa949 Thread-specific histogram statistics
6a14d55bd add prefix_seek_mode to db_iter_test
de47e2bd4 Fix ClockCache memory leak
f099af4c7 Fix travis
db74b1a21 fix bug in merge_iterator when data race happens
b18f9c9ea add nullptr check to internal_prefix_transform
4e395e875 Update docs README.md
2482d5fb4 support Prev() in prefix seek mode
7541c7a79 Fix cache_test valgrind_check failure
c8513cde0 Update the download location of Snappy (#1304)
b49b92cf2 Introduce Read amplification bitmap (read amp statistics)
c7004840d store prefix_extractor_name in table
4ad928e17 add comment to SimCache to estimate actual capacity
e9b2af87f Expose ThreadPool under include/rocksdb/threadpool.h
23a057007 Document memtable flush behavior in CancelAllBackgroundWork()
dade61ac2 Mitigate regression bug of options.max_successive_merges hit during DB Recovery
cce702a6e [db_bench] Support single benchmark arguments (Repeat for X times, Warm up for X times), Support CombinedStats (AVG / MEDIAN)
3586901f8 cat tests logs sorted by exit code
b2ce59537 Persist data during user initiated shutdown
4b3438d2d Fix parallel valgrind (valgrind_check)
a081f798b Relax consistency for thread-local ticker stats
b10d65c2a Update and slightly clarify instructions in build_detect_platform (#1301)
f85f99bf6 Fix the Windows build of RocksDB Java. Similar to https://github.com/facebook/rocksdb/issues/1220 (#1284)
7b8109517 Fix a crash when compaction fails to open a file
7c9586837 Thread-specific ticker statistics
ea9e0757f Add initial GitHub pages infra for RocksDB documentation move and update. (#1294)
2a9c97108 [Flaky Test] Disable DBPropertiesTest.GetProperty
d76ddf327 Disable ClockCache db_crashtest
cec2c6436 fix data race in NewIndexIterator() in block_based_table_reader.cc
badbff65b Not insert into block cache if cache is full and not holding handle
4a16c32ec Option to cache index/filter blocks with priority
99c4af716 Make ClockCache available with TSAN build
f57bc1d03 Fix lambda expression for clang/windows
5440675c3 Fix lambda capture expression for windows
6584cec8f Fold function for thread-local data
817eeb29b Add singleDelete to RocksJava (#1275)
ffdf6eee1 Add Status to RocksDBException so that meaningful function result Status from the C++ API isn't lost (#1273)
ecf900386 Fix bug in printing values for block-based table
72f8cc703 LRU cache mid-point insertion
6a17b07ca Add TablePropertiesCollector support in SstFileWriter
78837f5d6 TableBuilder / TableReader support for range deletion
4cc37f59e Introduce ClockCache
ff17a2abf Adding TBB as dependency.
49d88be02 c abi: allow compaction filter ignore snapshot (#1268)
0b63f51fb fixes 1215: execute_process(COMMAND mkdir ${DIR}) fails to create a directory with cmake on Windows (#1219)
3981345be Small nits (#1280)
2a2ebb6f5 Move LRUCache structs to lru_cache.h header
2fc2fd92a Single Delete Mismatch and Fallthrough statistics
3771e3797 WriteBatch support for range deletion
236756f2c Make SyncPoint return immediately when disabled
64a0082c6 Fix DBSSTest::AddExternalSstFileSkipSnapshot valgrind fail
dd7a748cf Fix java build
4fe12baa6 Make db_bench less space for --stats_per_interval
6525ce4ca Compaction stats printing: "batch" => "commit group"
a117891b4 Fixed typo (#1279)
b248e98cf Fix a destruction order issue in ThreadStatusUpdater
deda159b5 Added min/max/avg data block size output to sst_dump
e408e98c8 add Name() to Cache
a297643f2 Fix valgrind memory leak
d11c09d9e Eliminate memcpy from ForwardIterator
d36755502 Added further Java API options for controlling concurrent writes
ebdfe34cc Exposed further Java API options for controlling compaction
d1be59463 Improve documentation of SliceTransform.
6056d6317 Improve comment and bug fix for GetOptionsFromMap functions in convenience.h
76a67cf74 support stackableDB as the baseDB of transactionDB
67c1ae883 Travis build break fix
b693ba68b Minor PinnedIteratorsManager Refactoring
db3dfb164 Fixes for arcanist config (#1271)
87c91bd87 Persistent Read Cache (8) Benchmark tooling
2914de64e add sim_cache stats to Statistics
8b79422b5 [Proof-Of-Concept] RocksDB Blob Storage with a blob log file.
4beffe001 Fix test data race in two FaultInjectionTest tests
821bcb0b3 util/arena.cc: FreeBSD: More portable use of mmap(MAP_ANON) (#1254)
5370f44a8 Increase RocksDB version
56dd03411 read_options.background_purge_on_iterator_cleanup to cover forward iterator and log file closing too.
ccecf3f4f UniversalCompaction should ignore sorted runs being compacted (when compacting for file num)
1b0069ce2 Remove non-gtest from parallelized tests
638c49f24 Change HISTORY.md for release 4.11
6b8e9c68b fix vs generator (#1269)
c38b075e7 Update HISTORY.md
8f399e3fe Update HISTORY.md
98d0b78ea Added check_snapshot option in the DB's AddFile function (#1261)
9fd68b7fb set travis open file descriptor limit
59ddb5059 Fix travis build break
f4d986364 Added SetOptions support to RocksJava (#1243)
7882cb977 Make DBOptionsTest::EnableAutoCompactionAndTriggerStall less falky
44f5cc57a Add time series database (resubmitted)
7c4615cf1 A utility function to help users migrate DB after options change
5bb0a7f73 Update appveyor.yml
86396cc18 Update appveyor.yml
c1db098dc Update appveyor.yml
7da2eaf0d Update appveyor.yml
34723b4c4 Cleanup unused variable pending_fsync_.
7cc0dbd66 cat all logs in sandcastle output
9253767a6 Correct geHistogramData() -> getHistogramData() (#1257)
f35b16f24 db_bench add an option of --base_background_compactions
c3a4bea5d Fix flaky test `ObsoleteFiles`
8234faabf Fix failed test
4990c0d1a Remove deprecated LEVELDB_PLATFORM_POSIX
7323e4c8a Fix clang on macOS
ee027fc19 Ignore write stall triggers when auto-compaction is disabled
e4609a749 Fix Windows build issues (#1253)
2306167d3 Fix clang build failure and refactor unit test
343304e1d Use StopWatch to do statistic job in db_impl_add_file.cc
cdc4eb689 Add a GetComparator() function to the ColumnFamilyHandle base class so that the user's comparator can be retrieved.
712dd27e6 Build break fixes
0155c73de Fix parallel tests `make check -j`
c49ea68c5 Fix to enable running CI jobs locally
726c2f7e5 Build break fix
d51dc96a7 Experiments on column-aware encodings
c116b4780 Persistent Read Cache (part 6) Block Cache Tier Implementation
64046e581 Write a benchmark to emulate time series data
9ae92f50b More granular steps in the Makefile, can help with running all or single Java tests (and with ASAN build - https://github.com/facebook/rocksdb/wiki/JNI-Debugging) (#1237)
7c01d6534 [Fix Java] Remove duplicate cases in LoggerJniCallback::Logv
8796934af Added missing Java ReadOptions settings (#1109)
5e2c79658 Make DBTest.CompressionStatsTest more deterministic
557748ff7 Fix db_stress failure (pass merge_operator even if not used)
811ee2111 Bugfix to ensure that logging can be achieved from threads that are not known to the JVM (#1106)
afad5bd1c Simplify thread-local static initialization
6920cde89 Remove an extra apostrophe
e72ea485e add InDomain regression test
9c8ac144b Avoid duplicate task creation for RocksDB contruns
d4c45428a db_stress shouldn't assert file size 0 if file creation fails
d3bfd3397 Testing out parallel sandcastle changes
7efd9c25c Increse timeout in some tests
50b8d29b9 fixes 1230: Error:string sub-command REGEX, mode REPLACE needs at least 6 arguments total to command (#1231)
5c858ddd2 fix errata in libnuma test (#1244)
e5b5f12b8 Change options memtable_prefix_bloom_huge_page_tlb_size => memtable_huge_page_size and cover huge page to memtable too
0ce258f9b Compaction picker to expand output level files for keys cross files' boundary too.
ac0d93b08 fixes 1217: rocksdbjni javac and javah execute_processes fail on windows (#1218)
1ae46094d Appveyor badge to show master branch
8745f013f [Fix java build] Stop using non standard std::make_unique
e12270dfe fix previous typo
bbd6a5a18 ldb restore subcommand
9498069fc Run error-filtering script on diff-triggered tests
f8061a237 Fix Statistics TickersNameMap miss match with Tickers enum
16e225f70 Fix MergeContext::copied_operands_ strings moving
a4955b39a Run sandcastle tests in /dev/shm
ae0ad719d Fix flaky DBSSTTEST::DeleteObsoleteFilesPendingOutputs
b2a8016df Update db_bench_tool.cc (#1239)
c6654588b Disable two dynamic options tests under lite build
2a6d0cde7 Ignore stale logs while restarting DBs
ee8bf2e41 fixes 1228: rockdbjni loadLibraryFromJarToTemp fails when file is already present (#1232)
f85df120f Re-enable tsan crash white-box test with reduced killing odds
89e4c4882 Update README.md to include appveyor badge
b06ca5f86 ldb load, prefer ifsteam(/dev/stdin) to std::cin (#1207)
4ea0ab3cc Revert "Remove bashism from `make check` (#1225)"
12767b313 fixes 1220: rocksjni build fails on Windows due to variable-size array declaration (#1223)
a9d512a76 Update .gitignore for internal release
d5a51d4de Need to make sure log file synced before flushing memtable of one column family
89f319c2d Fix unit test which breaks lite build
b50632920 Add unit test not on /dev/shm as part of the pre-commit tests
b9a97181a Bump next release version
663afef88 Add EnvLibrados - RocksDB Env of RADOS (#1222)
32604e660 Fix flush not being commit while writing manifest
9ab38c45a Remove %z Format Specifier and Fix Windows Build of sim_cache.cc (#1224)
08ab1d83a Remove bashism from `make check` (#1225)
f9b14be49 Re-enable TSAN crash test but only with black box crash test
68f3eb746 Run release build for CLANG and GCC 4.8.1 in pre-commit tests too
e70020e4f Only cache level 0 indexes and filter when opening table reader
7bedd9440 Build break fix
68a8e6b8f Introduce FullMergeV2 (eliminate memcpy from merge operators)
e70ba4e40 MemTable::PostProcess() can skip updating num_deletes if the delta is 0
2a282e5f5 DBTablePropertiesTest.GetPropertiesOfTablesInRange: Fix Flaky
d9cfaa2b1 Persistent Read Cache (6) Persistent cache tier implentation - File layout
9430333f8 New Statistics to track Compression/Decompression (#1197)
515b11ffa fixes #1210: rocksdb/java/CMakeLists.txt lacks cmake_minimum_required (#1214)
876cb8bfb fixes #1212: rocksdbjni maven build does not escape slashes in groovy script (#1213)
21c55bdb6 DBTest.DynamicLevelCompressionPerLevel: Tune Threshold
4b9525358 Refactor cache.cc
c6a8665b3 Update LANGUAGE-BINDINGS.md
880ee363e ldb backup support
6797e6ffa Avoid updating memtable allocated bytes if write_buffer_size is not set
dda6c72ac Add DestroyColumnFamilyHandle(ColumnFamilyHandle**) to db.h
56222f57d Avoid FileMetaData copy
15b7a4ab8 Fixed output size and removed unneeded loop
6ea41f852 Fix deadlock when trying update options when write stalls
efd013d6d Miscellaneous performance improvements
e6f68faf9 Update Makefile to fix dependency
816ae098e fix test failure
e295da126 Fix Log() doc for default level
8e6b38d89 update DB::AddFile to ingest list of sst files
296545a2c Fix clang analyzer errors
61dbfbb6c Add release build to RocksDB per-diff/post-commit tests
907f24d0e Concurrent memtable inserter to update counters and flush state after all inserts
0f691c4b5 CLI option & Rename() allow overwrite
7c190070b delete unnessary pointer cast in beginInternalTransaction() function
e1b3ee8a7 Cleanup auto-roll logger flush-while-rolling test
cd4178a01 Add a new feature to enforce a sync point only active on a thread
b954847fc Fix release build for MyRocks by using debug-only code only in debug builds
a00bf1b3c Add More Logging to track total_log_size
01f77cb19 Update USER.md to include more services at Facebook.
eb53c05a3 Add comment for GetBackupInfo about returned BackupInfos order
32df9733d Add options.write_buffer_manager: control total memtable size across DB instances
5aaef91d4 group multiple batch of flush into one manifest file (one call to LogAndApply)
a45ee8318 Fix a bug that accesses invalid address in iterator cleanup function
38fae9e65 fix typos in HISTORY.md (#1192)
1a11c934d Disable some persistent cache tests on linux/clang
9b5adea97 Add More Logging to track total_log_size
95d96eeeb remove LockFile
ff45d1b54 if read only backup engine can't find meta dirs, return NotFound() instead of IOError()
cb2476a0c fix rate limiter to avoid starvation
6b7167651 Run env_basic_test on Env::Default
9eb0b5395 Move env_basic_test cleanup to TearDown
1fe3bf829 Re-enable linux on travis
43692793e Fixed Minor Bug on Windows Build and db_bench_tool.cc (#1189)
95c192475 writable file close before reset
197b832af Update USERS.md
bdb1d19a6 Fix UBSan build break caused by variable not initialized
b726bf596 FreeBSD does not have std::to_string (#1190)
faa7eb3b9 Improve regression_test.sh
c4cef07f1 Update DBTestUniversalCompaction.UniversalCompactionSingleSortedRun to use max_size_amplification_percent = 0
892e9d304 make transaction WriteOptions modifiable
4f2b0946d fix simple typos (#1183)
3b7ed677d ColumnFamilyOptions API [CF + RepairDB part 3/3]
56ac68629 Detect column family from properties [CF + RepairDB part 2/3]
3fc713ed9 delete 2nd level children for default env
343507afb Refactor to use VersionSet [CF + RepairDB part 1/3]
aa432be4b Workarounds for continuous build implementation
8cd9f04fe Test change to verify new commit detection
8a4ee7e90 Trivial change to test cont. build
af6248d8b Fix max_bytes_for_level_base comment
0d7b26123 add tests to env_basic_test.cc
6576fa05a Fix minor typos and PHP source file name used to trigger the builds
c4e19b77e Add a read option to enable background purge when cleaning up iterators
fa813f747 Update DB::AddFile() to ingest the file to the lowest possible level
d6b79e2fd Remove filter_deletes from crash_test
a52e4d7d0 Framework for enabling continuous RocksDB build and tests
f9bd66779 Makefile warning for invalid paths in make_config.mk
88a2776db Update SstFileWriter to use bottommost_compression if avaliable
e87d5df1a TiKV use-case (#1172)
7b79238b6 Deprectate filter_deletes
4939fc389 Bulk load mode shouldn't stop ingest
3a2bccc84 Fixed a crash bug that incorrectly parse deprecated options in options_helper
cf8adc971 Allow arcanist_util to work with both new and old arc versions
30a24f2d3 Add InternalStats and logging for AddFile()
d26a84807 Temporarily disable travis on linux
249e796df Fix Flaky DBCompactionTest.SkipStatsUpdateTest
162c9170d Make sandcastle access secure
8366e10ff Fix clang build
0babce57f Move away from enum char value -1
812dbfb48 Optimize BlockIter::Prev() by caching decoded entries
550bf895e Minor bug fix with log name
886af5910 Fix examples/Makefile jemalloc error
e3b1e3dfa Expose save points in Java WriteBatch and WBWI (#1092)
f5177c761 Remove wasteful instrumentation in FullMerge (stacked on D59577)
97fd2a638 Remove dead Jenkins code and support `arc diff --preview` in RocksDB
7c919decc Reuse TimedFullMerge instead of FullMerge + instrumentation
9a33a723b Remove the comments saying allow_concurrent_memtable_write and enable_write_thread_adaptive_yield are not stable
81f6b33d9 Fix tsan error
bc8af90e8 add option to not flush memtable on open()
8100ec2cd Fix libgcc broken lib path
7360db39e Add a check mode to verify compressed block can be decompressed back
2a79af1c5 Fix Java Break Related to memtable bloom bits to size ratio change
6faddd7c5 Merge db/slice.cc into util/slice.cc
5009b5326 BlockBasedTable::FullFilterKeyMayMatch() Should skip prefix bloom if full key bloom exists
2d05eaeb2 Fix name conflict in delete_shceduler_test and db_sst_test
bde7d1055 Fix clang_analyze path in fbcode_config.sh
ca3db5478 Fetch branches from github for format compatibility test
20699df84 memtable_prefix_bloom_bits -> memtable_prefix_bloom_bits_ratio and deprecate memtable_prefix_bloom_probes
e9c1face6 Minor fix to precommit-check.py
fcc47fa5f New features to precommit check script
56887f6cb Backup Options
a683d4aba URI-based Env selection for db_bench
53a4bd8a6 duplicate line
3e8686961 release 4.9 update version and history
b2973eaae Remove options builder
5b197bff4 Enabled Windows build for volatile tier implementation
281fbdddc Temporarily remove PersistentCacheOptions from persistent_cache_tier.h to fix unity build
2ae15b2d5 Format compatible test should cover forward compatibility up to 4.8.
00a058725 netflix use-case
5091dfc1e use branch names in format compatibility test
edc764e91 Use valgrind built with gcc-4.9-glibc-2.20
8ff59b2b4 Disable PersistentCacheTierTest.VolatileCacheInsertWithEviction test under TSAN temporarily
1ba452226 Fix for GCC 5.4 (#1157)
972c895c3 Previously WARN level logging became FATAL level logging in the Java API (#1089)
a73b26f60 Adding test for contiguous WAL detection
098da8348 Fix CLANG build break caused by the recent Persistent Cache change
54db29b8f Use gvfs links in dependencies.sh
d755c62f9 Persistent Read Cache (5) Volatile cache tier implementation
fda098461 Allow regression_test.sh to specify OPTIONS_FILE.  Add header comments.
0fee89684 Fix Windows build
10d46b9c8 Update tp2 clang path to fix clang build
774a6aa29 Java API - Rename geHistogramData -> getHistogramData (#1107)
3070ed902 Persistent Read Cache (4) Interface definitions
e42dc9192 Update paths for fbcode dependencies
5647fa427 stack_trace,cc: The current Stacktrace code does not compile for FreeBSD (#1153)
0d65acec0 threadpool.cc: abort() lives in stdlib.h on FreeBSD (#1155)
19dd5a61c env_chroot.cc: FreeBSD likes stdlib.h for realpaht() and friends (#1154)
5aca977be env_basic_test library for testing new Envs [pluggable Env part 3]
1147e5b05 Adding support for sharing throttler between multiple backup and restores
6e6622abb Create env_basic_test [pluggable Env part 2]
e53287794 Add statistics field to show total size of index and filter blocks in block cache
a791a2cf2 Java API - Bugfix for native linking of Compaction Filter (#1099)
af0c9ac01 Env registry for URI-based Env selection [pluggable Env part 1]
02ec8154e allow updating block cache capacity from C (#1149)
630b732cb fix flaky sim_cache_test
62d548098 Add persistent cache to Windows build system
842958651 Fix race condition in SwitchMemtable
88acd932f Allows db_bench to take an options file
3a276b0cb Add a callback for when memtable is moved to immutable (#1137)
8cf0f86d3 Allow regression test to run db_bench at a remost host
27ad17071 Fix Windows build break
936973d14 Small tweaks to logging to track the number of immutable memtables
21c047ab4 add readahead size option (#1146)
71c7eed91 Assert boundary checks for SetPerfLevel()
5d85fdb2c add missing lock
c40c4cae1 LDBCommand::SelectCommand to use a struct as the parameter
590e2617e fix delete file bug when do checkpoint (#1138)
8dfa980cb Add statically-linked library for tools/benchmarks
f62fbd2c8 Handle overflow case of rate limiter's paramters
57461fba8 In-memory environment read beyond EOF
0e2000017 LDBCommand::InitFromCmdLineArgs() to move from template to function wrapper
472c06e90 Add low and upper bound values for rocksdb::PerfLevel enum
157e0633e MutexLock -> ThreadPoolMutexLock in util/threadpool.cc
23d4cf483 include/rocksdb/sst_file_writer.h should not depend on util/mutable_cf_options.h
345fd73fa Fix flaky DBTestDynamicLevel.DynamicLevelMaxBytesBase2
8fc75de32 Minor fix to disable DynamicLevelMaxBytesBase2
9dd50d990 Fix db_bench
5d660258e add simulator Cache as class SimCache/SimLRUCache(with test)
d379d110e Update CMakeLists.txt for added test
21f847eda Direct IO fix for Mac
99765ed85 Clean up the ComputeCompactionScore() API
def2f7bd0 Expose report_bg_io_stats option in the C API. (#1131)
f89caa127 Direct IO capability for RocksDB
8f1214531 C API: Expose DeleteFileInRange (#1132)
11f329bd4 db/db_impl: restrict WALRecoveryMode when using recycled log files
2b2a898e0 db/log_reader: combine kBadRecord{Len,Checksum} for readability
34df1c94d db/log_reader: treat bad record length or checksum as EOF
7947aba68 db/log_reader: move kBadRecord{Len,Checksum} handling into ReadRecord
847e471db db/log_test: add recycle log test
4e7e41ba7 Disable lite build/testing for persistent read cache
1d725ca51 Deprecate BlockBasedTableOptions.hash_index_allow_collision=false.
0e77246ba backupable_db.cc: lambada to explictly caputre "this" when escaping scope
2073cf377 Eliminate use of 'using namespace std'.  Also remove a number of ADL references to std functions.
26adaad43 Split WinEnv into separate classes. (#1128)
bb98ca3c8 Implement GetUniqueId for Mac
1f2dca0ea Add MaxOperator to utilities/merge_operators/
f6e404c20 Added "number of merge operands" to statistics in ssts.
7383b64b3 Fix formatting of HISTORY.md (#1126)
0e665c399 Disable long running GroupCommitTest (#1125)
3c69f77c6 Move IO failure test to separate file
533cda90c Add GetStringFromCompressionType to include/rocksdb/convenience.h
c70a9335d Fix mutex unlock issue between scheduled compaction and ReleaseCompactionFiles()
05c5c39a7 Fix build
a6254f2bd Long outstanding prepare test
2ead11511 Fix TransactionTest.TwoPhaseMultiThreadTest under TSAN
1f0142ce1 Persistent Read Cache (Part 2) Data structure for building persistent read cache index
43afd72be [rocksdb] make more options dynamic
bac3be7c4 Fix build issue. (#1123)
f6aedb62c Fix Transaction memory leak
a08c8c851 Added PersistentCache abstraction
5c06e0814 [ldb] Templatize the Selector
aab91b8d8 Use generic threadpool for Windows environment (#1120)
a40033639 TransactionLogIterator sequence gap fix
fa3536d20 Store SST file compression algorithm as a TableProperty
40123b380 signed vs unsigned comparison fix
49815e384 [ldb] Export LDBCommandRunner
c1af07ce8 Disable backupable_db_test.cc on Windows
e61ba052b Isolate db env and backup Env in unit tests
560358dc9 Fix data race in GetObsoleteFiles()
5c1c90487 ldb option for compression dictionary size
c27061dae [rocksdb] 2PC double recovery bug fix
a657ee9a9 [rocksdb] Recovery path sequence miscount fix
8a66c85e9 [rocksdb] Two Phase Transaction
1b8a2e8fd [rocksdb] Memtable Log Referencing and Prepared Batch Recovery
0460e9dcc Modification of WriteBatch to support two phase commit
f548da33e Follow symlinks in chroot directory
d86f9b9c3 Fix lite build
4b3172343 Add bottommost_compression option
bfb6b1b8a Estimate pending compaction bytes more accurately
258459ed5 Properly destroy ChrootEnv in env_test
fca5aa6fc Initial script for the new regression test
e1951b6f2 Add --index_block_restart_interval option in db_bench
730f7e2e2 Fix win build
a9b3c47c8 Fix includes for clang on OS X
3f16a836a Introduce chroot Env
269f6b2e2 Revert "Modification of WriteBatch to support two phase commit"
04dec2a35 [ldb] Export ldb_cmd*.h
72c73cdc8 Java API - Add missing HEADER_LEVEL logging (#1104)
4d02bfa3a Add support for PauseBackgroundWork and ContinueBackgroundWork to the Java API (#1087)
8f65feafc Have sandcastle run lite_test for every diff
0d590d999 Make max_dict_bytes optional in options string
7ccb8d6ef BlockBasedTable::Get() not to use prefix bloom if read_options.total_order_seek = true
e3c6ba37d OptimizeForSmallDb(): revert some options whose defaults were just changed
967476eae Fix valgrind (DBIteratorTest.ReadAhead)
9790b94c9 Add optimize_filters_for_hits option to db_bench
a4ea345b0 Fixing lite build
24a24f013 Enable configurable readahead for iterators
ff4b3fb5b Fix Iterator::Prev memory pinning bug
cba752d58 sst_dump won't print size for unsupported compression type
6e801b0bd Eliminate memcpy in Iterator::Prev() by pinning blocks for keys spanning multiple blocks
1b166928c Release RocksDB 4.8.0
b8cf9130f Fix #1110, 32-bit build failure on Mac OSX (#1112)
21441c09b Fix calling GetCurrentMutableCFOptions in CompactionJob::ProcessKeyValueCompaction()
4ea6e051e Fix multiple issues with WinMmapFile fo sequential writing (#1108)
f3bb024fd Fix clang build
6e56a114b Modification of WriteBatch to support two phase commit
1d2e4ef74 ldb support new WAL records
a92049e3e Added EventListener::OnTableFileCreationStarted() callback
e8115cea4 Revert "Use async file handle for better parallelism (#1049)" (#1105)
6a14f7a97 Change several option defaults
c6c770a1a Use prefix_same_as_start to avoid iteration in FindNextUserEntryInternal. (#1102)
992a8f83b Not enable jemalloc status printing if USE_CLANG=1
029022b0f Fix crash_test
a06faa632 Skip PresetCompressionDict test for lite
e7899c661 Fix build issue. (#1103)
0f428c561 Fix compression dictionary clang osx error
6d4832a99 Merge pull request #1101 from flyd1005/wip-fix-typo
af70f9ac6 Fix typo in build_tools/fbcode_config.sh
54de13aba Fix compression dictionary clang errors
0850bc514 Fix build on machines without jemalloc
4032145ad Configurable compression in db_bench
843d2e313 Shared dictionary compression using reference block
ad573b902 Temporarily disable CompactFiles in db_stress in its default setting
1c80dfab2 Print memory allocation counters
eb7398085 Fix BackupableDBTest
ac0e54b4c CompactedDB should not be used if there is outstanding WAL files
d719b095d Introduce PinnedIteratorsManager (Reduce PinData() overhead / Refactor PinData)
1995e34d6 Retrieve file size from proper Env
7c14abf2c Improve BytewiseComparatorImpl::FindShortestSeparator
f3eb0b5b8 Make EventListenerTest.CompactionReasonLevel more deterministic
7b78d623f Shouldn't report default column family's compaction stats as DB compaction stats
995353e46 Fix null-pointer-dereference detected by Infer (https://github.com/facebook/infer)
24110ce90 Correct Statistics FLUSH_WRITE_BYTES
b71c4e613 Alpine Linux Build (#990)
90ffed1f9 Update USERS.md with link to LinkedIn blog post (#1088)
99a3bf8f6 Merge pull request #1068 from daaku/c-purge-old-backups
b54c34742 Use async file handle for better parallelism (#1049)
c146c9be1 rocksdb_create_mem_env to allow C libraries to create mem env (#1066)
6da70c581 expose more options in the c api (#1067)
6f01687aa C rocksdb_create_iterators to expose NewIterators (#1069)
644f978c1 Fix RocksDB Lite build in db_stress
5bd4022fe Add comparator, merge operator, property collectors to SST file properties (again)
7a6045a3c fix typo in HISTORY.md
73a847ef8 Add per-level compression ratio property
ee221d2de Introduce XPRESS compresssion on Windows. (#1081)
874c96ac1 Merge pull request #1083 from flabby/master
6cbffd50d Enable testing CompactFiles in db_stress
b95510ddf Fix DBTest.RateLimitedDelete flakiness
6356b4d51 Fix nullptr dereference in adaptive_table
9385fd72c Delete deprecated backup classes in Java
a2466c885 [db_stress] Make subcompaction random in crash_test
c3c389d54 Fix column label for L0 write sum
ec84bef24 New legocastle output parsing
725184b04 Fix db_block_cache_test in lite build
290883d94 Fix lite build
23089fd28 write_callback_test: clean test directory before running tests
792762c42 Split db_test.cc
40b840f29 Delete deprecated *BackupableDB interface for backups
6affd45d8 Make more tests run in parallel
47833e0ab Merge branch 'master' of github.com:facebook/rocksdb
e5c614e1d Fixing snapshot 0 assertion
1b1adebe8 Fixing snapshot 0 assertion
6d436a3f8 DBTest.HardLimit made more deterministic
994d9bc82 Make parallel valgrind watch-log more readable
9d35ae649 Make DBTestUniversalCompaction.IncreaseUniversalCompactionNumLevels more deterministic
cea8ed970 Fix backupable_db_test test cases that can't run by itself
4b6833aec Rename options.compaction_measure_io_stats to options.report_bg_io_stats and include flush too.
3894603fe Allow valgrind_check to run in parallel
c9d668c58 Fix unit tests issues on Windows (#1078)
083cadc7e Minor fix to Java sandcastle job definition
80b74a6c6 Include ldb tool in the windows build (#914)
7c14d11eb Minor fix to java build job
535af525d BlockBasedTable::PrefixMayMatch() to skip index checking if we can't find a filter block.
09be5cad5 Minor fix to sandcastle java job definition
1aeca9733 Release RocksDB 4.7
dfc3de8b7 Split Travis unittests Job
c2c8fe47f Add Java job for sandcastle
19ef3de57 Fix ManualCompactionPartial test flakiness
b345b3662 Add a minimum value for the refill bytes per period value
dff4c48ed BlockBasedTable::PrefixMayMatch: no need to find data block after full bloom checking
0353b853c Propagate sandcastle run error to UI
b885f33a5 Parallelize travis jobs
71303e04e Update db_bench_tool.cc (#1073)
63cf15bb9 Fix option settable tests
e208575b2 using java7 in runtime for hdfs env (#1072)
13e6c8e97 Relax an assertion in Compaction::ShouldStopBefore
ae21d71e9 Fixed a bug in RocksDB Statistics where flush is considered as compaction
8e0e22f76 Fix Windows build by replacing strings.h include
5675d5037 Revert travis commit
91f0f1f5e fix travis
a23c6052c Don't run DBOptionsAllFieldsSettable under valgrind
30d72ee43 PrefixTest.PrefixAndWholeKeyTest should run against a different directory from prefix_test
0e3cc2cf1 Add column family info to TableProperties::ToString()
2448f8037 Make sure that if use_mmap_reads is on use_os_buffer is also on
114a1b879 Fix build errors for windows
052299035 Improve sst_dump help message
0930e5e99 Update comments on include/rocksdb/perf_context.h
3b977bcdc instructing people to use java7 for hdfs (#1063)
1518b733e Change default number of cache shard bit to be 6 and max_file_opening_threads to be 16.
ada88b63f fix wrong assignment of level0_stop_writes_trigger in spatialdb (#1061)
2391ef721 Embed column family name in SST file
ab4c62332 Don't use version in the error message
d02eb8d00 Fix unused variable warning
9278097f8 Merge pull request #1056 from facebook/igorcanadi-patch-1
cc87075d6 No need to limit to 20 files in UpdateAccumulatedStats() if options.max_open_files=-1
8a1a603fd Eliminate std::deque initialization while iterating over merge operands
f38540b12 WriteBatchWithIndex micro optimization
200654067 Merge pull request #1053 from adamretter/benchmark-java-comparator
f2c43a4a2 Stderr info logger
b55e2165b Rocksdb backup can store optional application specific metadata
9b5198752 Adding pin_l0_filter_and_index_blocks_in_cache feature and related fixes.
2feafa3db Change some RocksDB default options
a558830f8 Fixed compile warnings in posix_logger.h and coding.h
51c9464df Merge pull request #980 from adamretter/java-arm
925b5d002 Merge pull request #1054 from DCEngines/magic12
63e8f1b55 Formatted lines to adhere to 80 char limit
994b3bd69 Add support for UBsan builds to RocksDB
21700a510 to/from hex refactor
24420947d Replace kHeader by WriteBatchInternal::kHeader in few more places kHeader was moved from write_batch.cc to header file because it is being used wherever the number "12" was being used to check for record size
3bdbe8961 Merge branch 'magic12' of https://github.com/dcengines/rocksdb into magic12
78711524b In all the places where log records are read, there was a check that record.size() should not be less than 12.
e7c64fb11 Imporve sst_file_manager comment
99ffb3d53 Fix perf_context::merge_operator_time_nanos calculation
07bb12d97 Update internal jemalloc and other versions
ad2fdaa82 Correct a typo in a comment
be9816b3d Fix data race issue when sub-compaction is used in CompactionJob
e3802531f Merge pull request #1050 from yuslepukhin/support_db_test2
e7cc49cbd Add support for db_test2 for dev and CI runs
3996770d0 Add comments to perf_context skip counters
4e85b7479 Make WritableFileWrapper not screw up preallocation
ec458dcde Merge pull request #1047 from PraveenSinghRao/wal_filter_ex
583157f71 Avoid overloaded virtual function
b9d4fa890 Options settable tests to use a different special charactor
60e34baef Merge pull request #1044 from PraveenSinghRao/wal_filter_ex
136b8e0ca Merge from master
2dcbb3b4f Addressed review comments
b1fafcaca Revert "Adding pin_l0_filter_and_index_blocks_in_cache feature."
5f8741a69 Revert "Fix failing Java unit test."
43bbb5619 tools/check_format_compatible.sh to use consistent version when testing backward and forward compatibility
d7ae42b0f Fix failing Java unit test.
fbbb8a614 Add test for Snapshot 0
e182f03c1 Add unit tests for RepairDB
7d371863e travis build fixes
fbea4dc66 Merge pull request #1042 from SherlockNoMad/HistFix
780d2b04c Update format compatible checking tool
4f1c74a46 merge from master
f8c218930 Publish log numbers for column family to wal_filter, and provide log number in the record callback
44756260a Reset block cache in failing unit test.
522de4f59 Adding pin_l0_filter_and_index_blocks_in_cache feature.
be2227126 Merge pull request #1041 from yuslepukhin/adjust_for_jemalloc
4ecc03c03 Fix in HistogramWindowingImpl
2ca0994cf Latest versions of Jemalloc library do not require je_init()/je_unint()   calls. #ifdef in the source code and make this a default build option.
90aff0c44 Update --max_write_buffer_number for compaction benchmarks
72224104d Forge current file for checkpoint
33d568611 Merge pull request #1040 from bureau14/master
02e62ebbc Fixes warnings and ensure correct int behavior on 32-bit platforms.
9cad56861 Merge pull request #1039 from bureau14/master
3d29f9146 Improve documentation of the allow_os_buffer parameter.
3ff98bd20 Fix no compression test
b9cc42a72 Merge pull request #1038 from SherlockNoMad/HistFix
f76b260ef Fix FB internal CI build failure
774922c68 Merge pull request #1026 from SherlockNoMad/Hist
17b879b91 Merge pull request #1037 from SherlockNoMad/BuildFix
f11b0df12 Fix AppVeyor build error
6b03f93d4 Fix the build break on Ubuntu 15.10 when gcc 5.2.1 is used
697fab820 Updates to RocksDB subcompaction benchmarking script
58379bfb5 remove division from histogramwidowing impl
1a2cc27e0 ColumnFamilyOptions SanitizeOptions is buggy on 32-bit platforms.
e778c34e7 Merge pull request #1035 from bureau14/master
5bd3da1c5 Added quasardb to the USERS.md file
b2ae5950b Index Reader should not be reused after DB restart
0267655da Update change log for 4.6 release
08304c086 Expose RepairDB as ldb command
fd664a27b Fix Build Error
580fede34 Aggregate hot Iterator counters in LocalStatistics (DBIter::Next perf regression)
54f6b9e16 Histogram Concurrency Improvement and Time-Windowing Support
790252805 Add multithreaded transaction test
e8e6cf017 fix: handle_fatal_signal (sig=6) in std::vector<std::string, std::allocator<std::string> >::_M_range_check | c++/4.8.2/bits/stl_vector.h:794 #174
d9620239d Cleanup stale manifests outside of full purge
f71fc77b7 Cache to have an option to fail Cache::Insert() when full
ee8cc3520 Merge pull request #938 from alexander-fenster/master
765597fa7 Update compaction score right after CompactFiles forms a compaction
f0161c37b formatting fix
aa3f02d50 Improve comment in compaction.h and compaction_picker.h
2200295ee optimistic transactions support for reinitialization
badd6b784 Ignore db_test2
200080ed7 Improve snapshot handling for Transaction reinitialization
171c8e80b Update dependencies / Fix Clang
294bdf9ee Change Property name from "rocksdb.current_version_number" to "rocksdb.current-super-version-number"
bf1c4089d Use pure if-then check instead of assert in EraseColumnFamilyInfo
a7d4eb2f3 Fix a bug where flush does not happen when a manual compaction is running
68189f7e1 Update benchmarks used to measure subcompaction performance
dfe96c72c Fix WriteLevel0TableForRecovery file delete protection
451678c8c Merge pull request #1025 from SherlockNoMad/BuildFix
58ecd9132 Fix Windows build
501927ffc [backupable db] Remove file size embedded in name workaround
ef204df7e Compaction always needs to be removed from level0_compactions_in_progress_ for universal compaction
e79ad9e18 Add Iterator Property rocksdb.iterator.version_number
19ea40f8b Subcompaction boundary keys should not terminate after an empty level
deb08b822 Add parsing of missing DB options
f8e90e875 Get file attributes in bulk for VerifyBackup and CreateNewBackup
12fd9b186 Change BlockBasedTableOptions.format_version default to 2
4572a2d8c Update current version to 4.6
74b660702 Rename iterator property "rocksdb.iterator.is.key.pinned" => "rocksdb.iterator.is-key-pinned"
6743135ea Fix DB::AddFile() issue when PurgeObsoleteFiles() is called
432f3adf2 Add DB Property "rocksdb.current_version_number"
188bb2e7a Fix formatting identified by `arc lint`
0f2d2fcff Refactored tests to use try-with-resources
f8e02c782 Deprecate org.rocksdb.AbstractNativeReference#dispose() and implement java.lang.AutoCloseable
0f2fdfe23 Fix the javadoc and the formatting of the base-classes for objects with native references
b5b1db167 Recompute compaction score after scheduling manual compaction
5ea9aa3c1 TransactionDB:ReinitializeTransaction
1f5954147 Introduce Iterator::GetProperty() and replace Iterator::IsKeyPinned()
67789419f Merge pull request #1020 from gongsu832/master
69c471bd9 Handle concurrent manifest update and backup creation
3373c81fa Modify build_tools/build_detect_platform to detect and set -march=z10 on Linux s390x.
990509045 Merge branch 'master' of https://github.com/gongsu832/rocksdb
3492889ab Merge pull request #1019 from javacruft/wip-omit-leaf-frame-pointer-archs
7ca731b12 build: Improve -momit-leaf-frame-pointer usage
21f17aaa6 Modified Makefile and build_tools/build_detect_platform to compile on Linux s390x.
8800975fb Make DBTestUniversalCompaction.IncreaseUniversalCompactionNumLevels more robust
cd3fe675a Remove stale TODO
69c98f043 Reorder instance variables in backup test for proper destruction order
82f15fb15 Add test to make sure DropColumnFamily doesn't impact existing iterators
38201b359 Fix assert failure when DBImpl::SyncWAL() conflicts with log rolling
2568985ab IOStatsContext::ToString() add option to exclude zero counters
b04691665 Redo SyncPoints for flush while rolling test
291ae4c20 Revert "Revert "Fixed the bug when both whole_key_filtering and prefix_extractor are set.""
eef63ef80 Fixed CompactFiles() spuriously failing or corrupting DB
79ca039eb Relax the check condition of prefix_extractor in CheckOptionsCompatibility
4b1b4b8ae Merge pull request #1004 from yuslepukhin/child_attr
9ea2968d2 Implement ConsistentChildrenAttribute   by using default implementation for now as it works.
c7f1a8a46 Fix LITE build thread_local_test
0914f0ca5 Merge pull request #1003 from yuslepukhin/fix_mutexlock_pthread_build
d37d348da This addresses build issues on Windows  https://github.com/facebook/rocksdb/issues/1002
d825fc70d Use condition variable in log roller test
6b2a047df Fix SstFileManager uninitialized data member
a3db93c26 Remove the SyncPoint usage in the destructor of PosixEnv
df9ba6df6 Introduce SstFileManager::SetMaxAllowedSpaceUsage() to cap disk space usage
3943d1678 Fix race conditions in auto-rolling logger
d733dd572 [build] Fix env_win.cc compiler errors
cf38e56f2 Fix broken appveyor build caused by D53991
351252b49 Merge pull request #998 from fengjian0106/master
133605249 fix ios build error
d08d50295 Fix transaction locking
730a422c3 Improve the documentation of LoadLatestOptions
a7b6f0748 Improve write_with_callback_test to sync WAL
5bcf952a8 Fix WriteImpl empty batch hanging issue
871cc5f98 fix build without gflags
c90d63a23 can_unlock set but not used
44371501f Fixed a segfault when compaction fails
2f084d39b Merge pull request #992 from jofusa/jdennison/options-typo-fix
7bd284c37 Separeate main from bench functionality to allow cusomizations
bd5f842bb fixes typo in options logging
1c868d684 Fix includes for env_test
545a19395 Add J to commit_prereq so comilation/execution happens in parallel
5bb7371ca [build] Evaluate test names only when db_test exists
6a2b4fcb8 Add flag to forcibly disable fallocate
92a9ccf1a Add a new compaction priority that picks file whose overlapping ratio is smallest
3dc3d1c14 Merge pull request #984 from petermattis/pmattis/comparator-iterate-upper-bound
239aaf2fc Use user_comparator when comparing against iterate_upper_bound.
908100399 Fixed a dependency issue of ThreadLocalPtr
337671b68 Add universal compaction benchmarks to run_flash_bench.sh
3a67bffaa Fix an ASAN error in transaction_test.cc
c5af85eca Fix a memory leak of Slice objects from org.rocksdb.WBWIRocksIterator#entry1
e84137c8a Remove unnessecary java.util.List expense in JNI
76e8beeeb Pass by pointer from/to Java from JNI not by object
0e7e6f6e4 Improve Javadoc
18eb56305 Improve the speed and synchronization around the construction of Java/JNI objects
2a04268be Temporarily disable unstable tests in memory_test.cc
08a78b6b4 Merge pull request #979 from facebook/update_licenses
21e95811d Updated all copyright headers to the new format.
59b3ee658 Env function for bulk metadata retrieval
4a8cbf4e3 Allows Get and MultiGet to read directly from SST files.
fe93bf9b5 Transaction::UndoGetForUpdate
2608219cc crash_test: cover concurrent memtable insert in default crash test
a76e9093f Fix LITE db_test build broken by previous commit
b1887c5dd Explictly fail when memtable doesn't support concurrent insert
8ed343877 Add option to run fillseq with WAL enabled in addition to WAL disabled
73a9b0f4b Update version to 4.5
6f71d3b68 Improve perf of Pessimistic Transaction expirations (and optimistic transactions)
8e6172bc5 Add BlockBasedTableOptions::index_block_restart_interval
34a40bf91 Add --allow_concurrent_memtable_write in stress test and run it in crash_test
73bf330c7 Merge pull request #973 from yuslepukhin/moveout_testcode
f7c0f4e3e perf_context.cc and iostats_context.cc use different output macro (fix unity build)
9656eab00 This partially addresses issue https://github.com/facebook/rocksdb/issues/935   testutil.cc and testharness.cc could not be moved out at this time   as they are used by 4 benchmarks in release builds.
14a322033 Remove references to files deleted in commit abb405227848581d3e6d2ba40d94dbc0a5513902
8445e5380 Add a mechanism to run all tests in sandcastle
461cec4e8 Merge pull request #972 from adamretter/wb-threads
9ab269ab3 Threaded tests for WriteBatch
bf767c641 Minor fix to makefile
2c1db5ea5 always invalidate sequential-insertion cache for concurrent skiplist adds
c12ff20ab Merge pull request #965 from koldat/jni_for_windows
a09ce4fcd Skip some of the non-critical tests in ./tools/run_flash_bench.sh
284aa613a Eliminate duplicated property constants
94be872ea Merge branch 'master' of github.com:facebook/rocksdb
0c2bd5cb4 Removing data race from expirable transactions
5fcd1ba30 disable kConcurrentSkipList multithreaded test
466c2c1bf Generate tags for *.c files
70c068c97 Merge pull request #960 from koldat/masterFixes2
a62c519bb RollLogFile tries to find non conflicting file until there is no conflict.
57a95a700 Making use of GetSystemTimePreciseAsFileTime dynamic - code review fixes
502d41f15 Making use of GetSystemTimePreciseAsFileTime dynamic to not break compatibility with Windows 7. The issue with rotated logs was fixed other way.
52153930d Adding support for Windows JNI build - fix Java unit test for release build of JNI DLL
e2972803a Adding support for Windows JNI build
9c2cf9479 Fix for --allow_concurrent_memtable_write with batching
ac3fa9a6f Travis CI to disable ROCKSDB_LITE tests
7b943da1b Merge pull request #967 from SherlockNoMad/ValueSize
b5750790e Merge pull request #968 from yuslepukhin/one_shot_buffer
1ad818295 Fix WriteBatchTest.ManyUpdates, WriteBatchTest.LargeKeyValue under clang
ad7ecca72 Add unit tests to verify large key/value
fdd70d149 Skip filters for last L0 file if hit-optimized
aa5e3b7c0 PerfContext::ToString() add option to exclude zero counters
36300fbbe Enable per-request buffer allocation in RandomAccessFile  This change impacts only non-buffered I/O on Windows.  Currently, there is a buffer per RandomAccessFile  instance that is protected by a lock. The reason we  maintain the buffer is non-buffered I/O requires an aligned  buffer to work.  XPerf traces demonstrate that we accumulate a considerable  wait time while waiting for that lock.  This change enables to set random access buffer size to zero  which would indicate a per request allocation.  We are expecting that allocation expense would be much less than  I/O costs plus wait time due to the fact that the memory heap  would tend to re-use page aligned allocations especially with the  use of Jemalloc.  This change does not affect buffer use as a read_ahead_buffer for  compaction purposes.
1d854fa3d Fixed the asan error on column_family_test
37159a644 Add histogram for value size per operation
3b2a1ddd2 Add options.base_background_compactions as a number of compaction threads for low compaction debt
6ee38bb15 Slowdown of writing to the last memtable should not override stopping
d6c838f1e Add SstFileManager (component tracking all SST file in DBs and control the deletion rate)
45768ade4 transaction allocation perf improvements
77926f93e Merge pull request #964 from benoitc/fix/pi2
03a5661a1 fix build for raspberry 2
4b50f1354 Should not skip bloom filter for L0 during the query.
eadd221d3 Merge pull request #959 from koldat/master
d209076fa Merge pull request #961 from wingify/master
26c618004 Add Wingify to USERS.md
4265f81e8 Remove util/auto_roll_logger.cc (it was moved to different directory)
d7f22b6d2 Fixing generated GenerateBuildVersion.vcxproj when one builds on different locale than english. The problem is that date and time CLI utilities generates different format so that REGEX in CMake does not work.
d20915d52 Disable stats about mutex duration by default
0c433cd1e Fix issue in Iterator::Seek when using Block based filter block with prefix_extractor
035857a31 Fix RocksDB lite build
77ef87ccb Update fbcode_config4.8.1.sh to use update_dependencies.sh
955ecf8b4 Fix an ASAN error in compact_files_test
b0afcdeea Fix bug in block based tables with full filter block and prefix_extractor
167bd8856 [directory includes cleanup] Finish removing util->db dependencies
acd7d5869 [directory includes cleanup] Remove util->db dependency for ThreadStatusUtil
46f9cd46a [directory includes cleanup] Move cross-function test points
22ecb752d Add valgrind to pre-commit sandcastle testing
b7ecf3d21 Fix intermittent hang in ColumnFamilyTest.FlushAndDropRaceCondition
38e1d7fea ldb to support --column_family option
da33dfe18 Parameterize DBTest.Randomized
fb9811ee9 Add a perf context level that doesn't measure time for mutex operations
f7ef1a613 Include rest of dependencies in dependencies.sh
3e9209a07 Updated GetProperty documentation
40911e0b3 Run unit tests in parallel to find failing tests
2fbc59a34 Disallow SstFileWriter from creating empty sst files
f53c95f81 Cosmetic fixes and comments for the reader
f1ed17010 Add tests to make sure new DB or ColumnFamily options are settable through string
f57596b0c Improvements to pre-commit
538eec066 Update fbcode_config.sh to use latest versions automatically
8019aa9b5 improve test for manifest write failure
bcd4ccbc3 Revert D7809
b0a15e7fb Mechanism to run CI jobs on local branch via commit_prereq
bb2888738 Cleanup property-related variable names
29289333d Add named constants for remaining properties
2c2b72218 Disable OptionsParserTest.BlockBasedTableOptionsAllFieldsSettable under CLANG
a300d9928 Added sandcastle pre-commit
202be23e4 Add test that verifies all options in BlockBasedTableOptions is settable through GetBlockBasedTableOptionsFromString()
eceb5cb1b Split db_test.cc (part 1: properties)
94918ae84 db_bench: explicitly clear buffer in compress benchmark
fdbff4239 Crash test to make kill decision for every kill point
39c3e94ff Merge pull request #954 from davidbernard/solaris_build
df7c2f3b5 As per google coding standard replace "using" in option_builder.cc and geodb_impl.cc
12809b44b Revert "Change notification email for travis"
34704d5c7 [easy] Fixed a crash in LogAndApply() when CF creation failed
791dbafa9 Merge pull request #953 from sselva/master
594a5ef02 Merge pull request #955 from bcbrock/ppc64-build
f423f05dc Simple changes to support builds for ppc64[le] consistent with X86
3f12e16f2 Make alloca.h optional
eaa563756 Change notification email for travis
d78c6b28c Changes for build on solaris
2e9fae3f2 Add Rakuten Marketing to USERS.md
83e1de92a move internal build to use zstd 0.4.5
aec10f734 Guard falloc.h inclusion to avoid build breaks
f7ebc2f34 Update HISTORY.mc for 4.4.0
addd9545f Merge pull request #947 from yuslepukhin/align_and_yield
ac50fd3a7 Align statistics   Use Yield macro to make it a little more portable between platforms.
b54d4dd43 tools/sst_dump_tool_imp.h not to depend on "util/testutil.h"
48a8667c3 Merge pull request #929 from warrenfalk/fix32
d9bca1e14 Reduce iterator deletion overhead
45d794068 Merge pull request #940 from yuslepukhin/fix_windows_build_signed_unsigned
20d7902df Fix compile error.   Use constructor style initialization instead of a cast for   simplicity.
e16438bb8 fixing build warning
b73fbbaf6 added --no_value option to ldb scan to dump key only
1477dcb37 Merge pull request #937 from petehunt/master
c7cb1076a Add Smyte to USERS.md
df7e3b622 Include <array> in table/plain_table_key_coding.h
235b162be Not scheduling more L1->L2 compaction if L0->L1 is pending with higher priority
9a8e3f73e plain table reader: non-mmap mode to keep two recent buffers
7ece10ecb DeleteFilesInRange: Mark files to be deleted as being compacted before applying change
94d9df248 fix an unused function compiler warning in crc32c in 32-bit mode
2f01e10fa use static_cast in crc32c instead of c-style cast
601f1306a fix shorten-64-to-32 warning in crc32c
f3fb39814 Fix BlockBasedTableTest.NoopTransformSeek failure
55b37efa1 fix a compile error on 32-bit
8c71eb5af Optimize DBIter::Prev() by reducing stack overhead
73c31377b Revert "Fixed the bug when both whole_key_filtering and prefix_extractor are set."
57605d7ef Fixed the bug when both whole_key_filtering and prefix_extractor are set.
6935eb24e Add ColumnFamilyHandle::GetDescriptor()
9760c842c fix valgrind failure in backupable_db_test
b1a3b4c0d Make ldb automagically determine the file type and use the correct dumping function
ba8344736 Merge pull request #923 from petermattis/pmattis/prefix-may-match
da032495d Optimize GetLatestSequenceForKey
260c29762 Fix index seeking in BlockTableReader::PrefixMayMatch.
e541dcc8f Fix issue #921
51adc5457 fix sporadic failure in fault_injection_test
a2422f053 fix potential test SleepingTask race condition
1627c4b1b Merge pull request #918 from mkurdej/fix/assertion-on-no-disk-space
92d0850f1 Fix failing assertion in logger on Windows when the disk is full.
7699439b7 Prevent the user from setting block_restart_interval to less than 1
4041903ec Enhance db_bench write rate limit
399343205 Add Airbnb and Pinterest to USERS.md
d74c9f0a5 DeleteFilesInRange: Clean job context if no files deleted
1dec5b8f5 Merge pull request #916 from warrenfalk/capi_huge_page_option
12fa27b4f Merge pull request #915 from warrenfalk/capi_full_bloom
0fde291ab expose memtable_prefix_bloom_huge_page_tlb_size option to C API
7e81dba5c Support creation of "full" format bloom filter from C API
bae5b0a1d Fix clang build
ac16663bd use -Werror=missing-field-initializers, to closer match MyRocks build
ab5a9a66d Merge pull request #911 from shuzhang1989/fix_envhdfs_virtual_func
eb5a13904 update posix env schedule call
a41f68ac2 fix inconsistency between env_hdfs and env
7238be090 Fix clang build in db_compaction_test
c9e2490bc Fix DynamicBloomTest.concurrent_with_perf to pass TSAN
63ddb783d Delete files in given key range
d8677a8d2 Upgrade internal CLANG version for FB-internal gcc 4.8.1
edf1cd497 Not generating "__attribute__((__unused__))" for padding fields if it is not CLANG
9eb4f9596 Merge pull request #907 from siying/master
22c0ed8a5 Disable Visual Studio Warning C4351
fcafac053 Fix memory leak in ColumnFamilyTest.WriteStall*
b99d4276f Fix java test buid broken by 7d87f02799bd0a8fd36df24fab5baa4968615c86
11672df19 Fix CLANG errors introduced by 7d87f02799bd0a8fd36df24fab5baa4968615c86
7fafd52dc Merge pull request #900 from shuzhang1989/hdfs_env_fix
2b7c810db more foramt
b79ccbd57 indent
7d87f0279 support for concurrent adds to memtable
5b2587b5c DBTest.HardLimit use special memtable
b4aa82366 format
4dfdd1d92 format
298ba27ae Merge pull request #846 from yuslepukhin/enble_c4244_lossofdata
7810aa802 Merge pull request #899 from zhipeng-jia/fix_clang_warning
4c5560d70 Merge pull request #895 from zhipeng-jia/develop
d43da8ae0 DBTest.DelayedWriteRate: fix assert of sign and unsign comparison
3280ae9a2 Fix warning in release
ec2664fef Fix clang compile error under Linux
4fd23fb13 add a factory method for creating hdfs env
9c176ef90 Update liblz4 to r131
15b890226 Change default options.delayed_write_rate
73b175a77 Fix clang warnings regarding unnecessary std::move
b9f77ba12 When slowdown is triggered, reduce the write rate
445d5b8c5 Fix clang build
e089db40f Skip bottom-level filter block caching when hit-optimized
06c05495e Merge pull request #898 from zhipeng-jia/fix_move_warning
aa515823b Fix clang warning
2ba03196d Merge pull request #897 from yuslepukhin/enable_status_move
dbb8260f7 Make Status moveable   Status is a class which is frequently returned by value from functions.   Making it movable avoids 99% of the copies automatically   on return by value.
2bf9b968c Fix lite_build
d005c66fa Report compaction reason in CompactionListener
728f944f0 Fix computation of size of last sub-compaction
8ac7fb837 Merge pull request #863 from zhangyybuaa/fix_hdfs_error
e53e8219a Merge pull request #894 from zhipeng-jia/develop
e0abec158 Sorting std::vector instead of using std::set
33e09c0e1 add call to install superversion and schedule work in enableautocompactions
22c6b50ee Merge pull request #893 from zhipeng-jia/develop
24c7dae13 Fix clang warning regarding implicit conversion
eff309867 Do not use timed_mutex in TransactionDB
97ea8afaa compaction assertion triggering test fix for sequence zeroing assertion trip
521da3abb Fix BlockBasedTableTest.BlockCacheLeak valgrind failure
a48382399 Fix use-after free in db_bench
bf8ffc1d6 Merge pull request #890 from zhipeng-jia/develop
131f7ddf6 fix typo: sr to picking_sr
c37729a6a db_bench: --soft_pending_compaction_bytes_limit should set options.soft_pending_compaction_bytes_limit
7b12ae97d Add signalall after removing item from manual_compaction deque
d72b31774 Slowdown when writing to the last write buffer
6b2a3ac92 Add documentation for unschedFunction
167fb919a ZSTD to use CompressionOptions.level
32ff05e97 Bump version to 4.4
aececc209 Introduce ReadOptions::pin_data (support zero copy for keys)
e6e505a4d Fix examples
aa29cc128 Improve examples/README.md
97265f5f1 Fix minor bugs in delete operator, snprintf, and size_t usage
b68dc0f83 Merge pull request #885 from yuslepukhin/fix_size_t_formatting
b6d19adcf Use port size_t formatting
963660eb5 Merge pull request #883 from zhipeng-jia/master
99ae549d3 Fix typo
636cd3c71 Clean up listener_test (reuse db_test_util)
030215bf0 Running manual compactions in parallel with other automatic or manual compactions in restricted cases
d26a4ea62 Merge pull request #882 from SherlockNoMad/BuildFix
768a61486 Fix appVeyor Build problem
aca403d2b Fix another rebase problems.
a6fbdd64e Fix rebase issues and new code warnings.
3fa68af31 Enable MS compiler warning c4244.   Mostly due to the fact that there are differences in sizes of int,long   on 64 bit systems vs GNU.
236fe21c9 Enable MS compiler warning c4244.   Mostly due to the fact that there are differences in sizes of int,long   on 64 bit systems vs GNU.
84f98792d Transaction::SetWriteOptions()
3bfd3d39a Use SST files for Transaction conflict detection
362d819a1 Improving parser
00d6edf6a Ensure the destruction order of PosixEnv and ThreadLocalPtr
64fa43843 Merge pull request #862 from ceph/wip-env
2074ddd62 env: add EnvMirror
a3ba5915c Correct a comment in include/rocksdb/cache.h
f0a8e5a2d Fixed the valgrind error in ColumnFamilyTest::CreateAndDropRace
9e4462906 Change SingleDelete to support conflict checking
c5af8bffb Merge pull request #879 from charsyam/feature/typos
c30b49954 fix typos in comments
56e77f096 Deprecate options.soft_rate_limit and add options.soft_pending_compaction_bytes_limit
d6e1035a1 A new compaction picking priority that optimizes for write amplification for random updates.
de6958b2e Merge pull request #877 from yuslepukhin/fix_unnecessary_type_truncation
0991cee6c Merge pull request #815 from SherlockNoMad/CounterFix
49957f9a9 Prefer integer arithmetics   The code had conversion to double then casting to size_t   and then casting uint32_t which caused compiler warning (VS15).
0836d265c Merge pull request #876 from warrenfalk/wf_win_master
c6fedf2bf Add compaction_iterator and delete_scheduler tests to Windows build
ac8e56f05 db_bench: in uncompress benchmark, get Snappy size from compressed stream
9c227923c Merge pull request #788 from OpenChannelSSD/to_fb_master2
fa3dbf203 Merge pull request #853 from Vaisman/enable_C4267_warning
ad6aaf4fa Merge pull request #848 from SherlockNoMad/db_bench
56bbecc31 Merge pull request #867 from SherlockNoMad/CacheFix
188170fb4 Updating HISTORY.md
758dbec7f Fix fb-only build for gcc 4.8.1
774b80e99 Resubmit the fix for a race condition in persisting options
afc84731f Include ldb_tools and sst_dump_tools libraries in shared library
e5c5f2381 Support marking snapshots for write-conflict checking - Take 2
ea1192355 Upgrade to ZSTD 0.4.2
b60cb88c7 Update examples/rocksdb_option_file_example.ini
3d8bb2c89 Fix valgrind failure in IncreaseUniversalCompactionNumLevels
7af91d425 Merge pull request #873 from yuslepukhin/make_vs15_build
1d63c3d61 Revert "Support marking snapshots for write-conflict checking"
78de0c922 Fix up VS 15 build.  Fix warnings  Take advantage of native snprintf on VS 15
ec704aafd Support marking snapshots for write-conflict checking
770dea932 Fix occasional failure of DBTest.DynamicCompactionOptions
ebc2d490d Split histogram per OperationType in db_bench
f307036bd Revert "Fix a race condition in persisting options"
2fa3ed518 Fix a race condition in persisting options
f276c3a82 Fix valgrind failures in 3 tests in db_compaction_test due to new skiplist changes
291088ae4 Fix undeterministic failure of ColumnFamilyTest.DifferentWriteBufferSizes
3c2b995fb Merge branch 'master' of https://github.com/facebook/rocksdb into CacheFix
355fa9436 EstimatedNumKeys Counter Inaccurate
b2863017b Move posix threads into a library
3a98a7ae7 Replace malloc with new for LRU Cache Handle
a9ca9107b Fix db_universal_compaction_test
d3bb572da Build break fix.
b28b7c6dd Added callback notification when a snapshot is created
e8180f990 added public api to schedule flush/compaction, code to prevent race with db::open
19b1201b2 Merge pull request #865 from yuslepukhin/fix_db_table_properties_test
e0de7ef87 Avoid empty ranges vector with subsequent zero element access
a330f0b3b Fix incorrect merge in db/db_compaction_test.cc
bd7a49d44 Make DBCompactionTestWithParam::CompactionTrigger more deterministic
be006d288 fix LinkFile() undefined reference error
4687ced5d fix ToString() not declared error
bcd7bd122 Relax verification condition of DBTest.SuggestCompactRangeTest
f9103d9a3 DBTest.DynamicCompactionOptions: More deterministic and readable
0ad68518b Fix DBCompactionTestWithParam.CompactionTrigger in non-jemalloc build.
459c7fba3 Revert previous behavior of internal_key_skipped_count
481f9edb1 Fix CLANG build
d7421c22f Fixed some typos in the comments of rocksdb options file example
ef8ed3681 Fix DBTest.SuggestCompactRangeTest for disable jemalloc case
db320b1b8 DB to only flush the column family with the largest memtable while option.db_write_buffer_size is hit
4a009f917 Merge pull request #860 from SherlockNoMad/BuildFix
b4efaebff Fix ms version Appveyor build error
d27ea4c9e Initialize options.row_cache
72930485b Fix clang build
6bbfa1874 BackupDB to have a mode to use file size in file name
f3ea00bc8 Merge pull request #856 from ceph/wip-env
4cedd6b03 EnvWrapper: add ReuseWritableFile
33e0c9382 Reduce extra key comparision in DBIter::Next()
9a9d4759b InlineSkipList part 3/3 - new skiplist type that colocates key and node
520172954 InlineSkipList - part 2/3
78812ec6b InlineSkipList - part 1/3
ffb466da4 Merge pull request #855 from yuslepukhin/enable_3rdparty_override
10d257d64 Enable override to 3rd party linkage
41b32c605 Enable C4267 warning
c5b467306 Fix race condition that causes valgrind failures
efb01a055 Merge pull request #850 from yuslepukhin/enable_2015_build
81be49c75 Have a way for compaction filter to ignore snapshots
047bd22aa Build on Visual Studio 2015 Update 1
88e052772 Reduce moving memory in LDB::ScanCommand
890f44f46 Merge pull request #844 from yuslepukhin/enable_C4804_unsafe_bool
9d0b8f19d plain table reader: avoid re-read the same position for index and data in non-mmap mode
89bacb7e7 Enable MS Warning C4804 : unsafe use of type 'bool' in operation
d5239f870 build_tools/fbcode_config4.8.1.sh: upgrade versions of some dependencies
c4ebb66d6 Not to build forward_iterator_bench now
c342549d0 Merge pull request #841 from yuslepukhin/fix_test_rerun_logic
7cb1293b6 Fix log names when scheduling runs and reruns
51fce92e1 "ldb compact" should force bottommost level compaction
f83164120 Merge pull request #837 from yuslepukhin/rerun_concurrency_value
4159ab816 Merge pull request #839 from SherlockNoMad/memtableOption
6170fec25 Fix build broken by previous commit of "option helper refactor"
3a6643c2f Merge pull request #805 from SherlockNoMad/OptionHelperFix
189b3e03d Fix uninitilizeded SpecialEnv::time_elapse_only_sleep_
d5540e18e DBTest.MergeTestTime to only use fake time to be determinstic
bd7be035e Support Memtable Factory Parse in option_helper.cc
94e39e236 Exclude DBTest.FileCreationRandomFailure as a long running test   Increase concurrency to 18   Fix exclusion but in the ps script
4189c0f9a Fix Java Makefile
605a24d94 Block forward_iterator_bench under MAC and Windows
2a0510c9f Failed tests must be rerun with concurrency 1
9b8c9be0b Fix forward_iterator allocation of vector.
5cbb7e43e DBTest.MergeTestTime: relax counter upper bound verification
52e04b3d0 Merge pull request #833 from yuslepukhin/fix_win_build_after_lint
314f62194 Remove headers from the cc since they are in the module's header.
472c74006 Add necessary headers after cpplint rearranged includes
9bc9c93bd Move to version 4.3
3381e2c3e Handle multiple calls to DBImpl::PauseBackgroundWork() and DBImpl::ContinueBackgroundWork()
65a042921 Merge pull request #831 from yuslepukhin/remove_forward_iter_bench_win
ca5566d20 Fix clang build
4175472ad Merge pull request #832 from yuslepukhin/fix_forward_iter_outofbounds
cb9459f85 Fix empty vector write in ForwardIterator
a163cc2d5 Lint everything
8f01f2541 Remove forward_iter_bench from Win build.
dac5b248b UniversalCompactionPicker::PickCompaction(): avoid to form compactions if there is no file
d06b63e99 Fix Rocksdb lite build failure in forward_iterator_bench
7824444bf Reuse file iterators in tailing iterator when memtable is flushed
2ae4d7d70 Make sure that CompactFiles does not run two parallel Level 0 compactions
d781da816 Add CheckOptionsCompatibility() API to options_util
2391b459b Merge pull request #824 from yuslepukhin/try_ci_tests_on_daily
2ab3e2df2 Fix a build break so tests can run
247c49a40 Merge branch 'master' into try_ci_tests_on_daily
935d1495c Run tests imporvements   Add sequential rerun for any failed tests. Add env_test case.   Limit concurrency   Allow to specify individual tests   Take $Limit into account when displaying number of tests
5ac16300b Fixed valgrind error in options_util_test
6ce42dd07 Don't merge WriteBatch-es if WAL is disabled
56245ddcf Fixed DBCompactionTest.SkipStatsUpdateTest
e11f676e3 Add OptionsUtil::LoadOptionsFromFile() API
e78389b55 Fixed build failure of RocksDBLite test on options_file_test.cc
e114f0abb Enable RocksDB to persist Options file.
7ed2c3e45 Merge pull request #823 from yuslepukhin/fix_off_t_type
7f59e33b1 Make CI build debug/optimized
ae2dfe404 Try running db_test during integration build
720af2269 Merge branch 'fix_off_t_type' of https://github.com/yuslepukhin/rocksdb into fix_off_t_type
5270b33bd Make use of portable `uint64_t` type to make possible file access   in 64-bit.
631863c63 track WriteBatch contents
505accda3 remove constexpr from util/random.h for MSVC compat
5421c9728 Make use of portable `uint64_t` type to make possible file access   in 64-bit.
b81b43098 Switch to thread-local random for skiplist
75a8bad2a Merge pull request #821 from yuslepukhin/continue_windows_warnings
986230b8c Revert "Fix TSAN build for fbcode"
f3ca28ab0 Correct the comment of GetApproximateMemoryUsageByType
838676c17 Revert "Adding new table properties"
7c86d5049 Enable C4305 'identifier' : truncation from 'type1' to 'type2'
85a2ce9c1 Enable C4702 unreachable code
62aa1b1b7 Enable C4200 warning   nonstandard extension used : zero-sized array in struct/union
5b9ce1a32 Merge pull request #820 from yuslepukhin/enable_compiler_warnings
20f57b171 Enable Windows warnings C4307 C4309 C4512 C4701   Enable C4307 'operator' : integral constant overflow   Longs and ints on Windows are 32-bit hence the overflow   Enable C4309 'conversion' : truncation of constant value   Enable C4512 'class' : assignment operator could not be generated   Enable C4701 Potentially uninitialized local variable 'name' used
8be568a9c Adding new table properties
2b42000f4 incorrect batch group size computation for write throttling
c745f1d2c Fix TSAN build for fbcode
fe789c5f2 Document SingleDelete
e89e5b253 Merge pull request #818 from yuslepukhin/improve_test_concurrency
ae7940b62 Fix regression failure in PrefixTest.PrefixValid
3277d172b Improve concurrency when running tests   PowerShell seems to have a hard time when a flood of async tasks is   scheduled at the same time.   I speculated that WaitForMultipleObjects() in Windows can only take up   to 64 process handles and if you want to handle more than you should write   some additional code which can be sub-optimal. I.e to implement Wait-Job -Any.   I decided to test that suggestion and introduced a $Concurrency parameter with a default value of 62.   So in the new version the script fires up up to $Concurrency value   and wait for anything to complete before starting any more processes.   This improved matters greatly. Individual tests against ramdrive now   run in 8 minutes and all of the 200+ db_tests run in 9 minutes with concurrency   values of 8-16. About 48 is required to load a CPU on my box running against HD   but that does not improve running times much.
c8e01ef98 Delete test iterators
9d50afc3b Prefix-based iterating only shows keys in prefix
14c6e1a04 Add write_stress to RocksDB Legocastle runs
db3f5e494 Update HISTORY.md
042fb053f Fix clang
2419f435a Merge pull request #816 from SherlockNoMad/GeoDBTestFix
2e4540991 Fix appveyor build failure
183cadfc8 Add OptionsSanityCheckLevel
dba5e0074 Fixed the compile error in RocksDBLite in memory_test.cc
7d7ee2b65 Add Memory Insight support to utilities
3ecbab004 Add GetAggregatedIntProperty(): returns the aggregated value from all CFs
93a966722 Merge branch 'master' of github.com:facebook/rocksdb
c9aef3c41 Add RocksDb/GeoDb Iterator interface
f31442fb5 Merge pull request #803 from SherlockNoMad/SkipFlush
dcc898b02 Merge pull request #812 from yuslepukhin/fix_windows_warnings
df7ed91ef Fix white space at end of line
a0163c068 Do not disable compiler warnings:   C4101 'identifier' : unreferenced local variable   C4189 'identifier' : local variable is initialized but not referenced   C4100 'identifier' : unreferenced formal parameter   C4296 'operator' : expression is always false
279c8e0cd Merge pull request #811 from OverlordQ/unused-variable-warning
affd83369 Fix introduced in 2ab7065 was reverted by 18285c1.
db68a2c09 Merge pull request #806 from yuslepukhin/signed_unsigned_warning
ccc8c10c0 Move skip_table_builder_flush to BlockBasedTableOption
eaaf081d1 Do not suppress C4018 'expression' : signed/unsigned mismatch   The code compiles cleanly for the most part. Fix db_test.   Move debug file to testutil library.
ff4499e29 Update DB::AddFile() to have less restrictions
9ac88c855 Merge branch 'master' of https://github.com/facebook/rocksdb into OptionTestFix
84992d647 Option Helper Refactoring
11c71a365 db_bench: --compaction_pri default should be rocksdb::Options().compaction_pri
335e4ce8c options_test: fix a bug of assertion
66a3a87ab Merge pull request #797 from SherlockNoMad/optionHelper
550af4ee6 Fix Travis Build Error
a6dd0831d Add Option to Skip Flushing in TableBuilder
2872e0c8c Clean and expose CreateLoggerFromOptions
296c3a1f9 "make format" in some recent commits
6388e7f4e Merge pull request #798 from yuslepukhin/readahead_buffermanagement
f4cbb90c4 Merge pull request #799 from yuslepukhin/fix_random_generator_compile
1277a48f1 Fix 80 character limit issue.
ee2c3236d Fix compilation problem on Windows.   char is not a valid template parameter for std::uniform_int_distribution   according to the standard. Replacing with int should be just fine.
b69b9b624 Support PlainTableOption in option_helper
c97667d9f Fix RocksDB lite build for write_stress
0d720dfc1 Use the correct variable when fetching table properties.
4b66d9534 Write stress test
47414c6cd Move include/posix/io_posix.h to util/io_posix.h
2889df84c Revert "Avoid to reply on ROCKSDB_FALLOCATE_PRESENT in include/posix/io_posix.h"
28c8758a3 Merge pull request #795 from yuslepukhin/fix_mocktable_id
5c8f2ee78 Fix MockTable ID storage   On Windows two tests fail that use MockTable:   flush_job_test and compaction_job_test with the following message:   compaction_job_test_je.exe : Assertion failed: result.size() == 4,   file c:\dev\rocksdb\rocksdb\table\mock_table.cc, line 110
72d6e758b Fix WritableFileWriter::Append() return
d0a18c284 Merge pull request #786 from aloukissas/unused_param
c37223c08 Avoid to reply on ROCKSDB_FALLOCATE_PRESENT in include/posix/io_posix.h
6fbc4f9f3 Implement smart buffer management.   introduce a new DBOption random_access_max_buffer_size to limit   the size of the random access buffer used for unbuffered access.   Implement read ahead buffering when enabled.   To that effect propagate compaction_readahead_size and the new option   to the env options to make it available for the implementation.   Add Hint() override so SetupForCompaction() call would call Hint()   readahead can now be setup from both Hint() and EnableReadAhead()   Add new option random_access_max_buffer_size support   db_bench, options_helper to make it string parsable   and the unit test.
d6219e4d9 Mac build break caused by include/posix/io_posix.h not declearing errno,
beb69d451 Merge pull request #765 from PraveenSinghRao/wal_filter
ab0f3b964 crash_test to trigger some less frequent crash point more frequently
7beb743cf Merge pull request #778 from Vaisman/master
4ce117c4d Merge branch 'master' into wal_filter
32cdec634 Fail recovery if filter provides more records than original and corresponding unit-test, fix naming conventions
44d4057d7 Avoid some includes in io_posix.h
2adad23a1 Fix unused parameter warnings.
b0980ff74 Fix unused parameter warnings.
bc898c5f8 Fix unused parameter warnings.
138876a62 Merge pull request #746 from ceph/wip-recycle
581f20fd8 Add LITE tests to Legocastle
3d56d868c Merge remote-tracking branch 'upstream/master'
d69111114 include/posix/io_posix.h should have a once declartion
a6962edf8 Merge pull request #783 from yuslepukhin/remove_test_conditional_compilation
3c750b59a No need to #ifdef test only code on windows
8c11c5dee Merge pull request #768 from OpenChannelSSD/to_fb_master2
6e6dd5f6f Split posix storage backend into Env and library
980a82ee2 Fix a bug in GetApproximateSizes
d0d13ebf6 fix bug in db_crashtest.py
01a41af0a Merge remote-tracking branch 'upstream/master'
5678c05d8 Use DEBUG_LEVEL=0 in make release and make clean
ac25fe6b9 Merge pull request #779 from yuslepukhin/optimize_windows_build
e154ee086 Do not build test only code and unit tests in Release builds   Test code errors are currently blocking Windows Release builew   We do not want spend time building in Release what we can not run   We want to eliminate a source of most frequent errors when people   check-in test only code which can not be built in Release.   This feature will work only if you invoke msbuild against rocksdb.sln   Invoking it against ALL_BUILD target will attempt to build everything.
cd3286fae Error while cmake by building from zip-archive
e3d4e1407 DBCompactionTestWithParam.ManualCompaction to verify block cache is not filled in manual compaction
033c6f1ad T7916298, bug fix
7717ad1af Adding artifacts to stress_crash CI job
0bf656b90 Don't spew warnings when flint doesn't exist
6d6776f6b Log more information for the add file with overlapping range failure
7951b9b07 make field order match initialization order
90228bb08 Merge pull request #771 from maximecaron/patch-1
2938c5c13 merge upstream changes
e3b1d23d3 Bump version to 4.2
a7b2bedfb log_{reader,write}: recyclable record format
4e07c99a9 Fix iOS build
0c59691dd Handle multiple batches in single log record - allow app to return a new batch + allow app to return corrupted record status
32c291e3c Merge branch 'master' of github.com:facebook/rocksdb into T7916298
4575de5b9 #7916298: merge tools/db_crashtest2.py into tools/db_crashtest.py
5c727de6a Merge pull request #777 from yuslepukhin/fix_win_build_uint
cfaa33f9a Update transaction iterator documentation
2f680ed09 Make index same type as auto deduced uint32_t
09f853550 uint is a not a datatype on windows.
ec1f8354a Fix the default assignment of DEBUG_LEVEL in Makefile
f18acd887 Fixed the clang compilation failure
4104e9bb6 log_reader: introduce kBadHeader; drop wal mode from ReadPhysicalRecord
9c33f64d1 log_reader: pass in WALRecoveryMode instead of bool report_eof_inconsistency
718805210 db_test_util: add recycle_log_files to set of tested options
3ac13c99d log_reader: pass log_number and optional info_log to ctor
5830c699f log_writer: pass log number and whether recycling is enabled to ctor
666376150 db_impl: recycle log files
d666225a0 db_impl: disable recycle_log_files if WAL archive is enabled
543c12ab0 options: add recycle_log_file_num option
1bcafb62f env: add ReuseWritableFile
e1a09a770 Implementation for GetPropertiesOfTablesInRange
ad471453e Allow GetProperty to report the number of currently running flushes / compactions.
277dea78f Add more kill points
a98fbacfa Moving memtable related files from util to a new directory memtable
8f143e03f Add ClearSnapshot()
f9ba79ecd crash_test to trigger fail points other than file appending more frequently
2f2de338c Run ROCKSDB_LITE tests in travis
680156ca6 crash_test to run with data sync on
e1a5ff857 Allow users to disable some kill points in db_stress
d306a7ea8 Merge pull request #773 from yuslepukhin/update_requirements
bb64d6da4 Disabling TSAN crash test
8c2fe68fd Update 4 is required for building with MS Visual Studio 13
952ad994a Fix db_test under ROCKSDB_LITE
6d730b4ae Block tests under ROCKSDB_LITE
5eee1ef2d Merge pull request #770 from Vaisman/master
63e507c59 Move ldb and sst_dump from utils to tools.
dae49e829 Make DBTest.ReadLatencyHistogramByLevel more robust
92060b215 Fix build error using Visual Studio 12
9f7413502 Error while cmake by building from zip-archive
b81b2ec25 Fix benchmarks under ROCKSDB_LITE
e587dbe03 Move manual_compaction_test.cc from util to db
666fb5df4 Remove DefaultCompactionFilterFactory.
d662b8dab Merge pull request #766 from PraveenSinghRao/lockfix
f55d3009c Make db_test_util compile under ROCKSDB_LITE
29a47cd2b Include the time unit in the comment of perf_context timers
2b925ccb4 Correct the comments in db/internal_stats.h
35ad531be Seperate InternalIterator from Iterator
91c041e57 move debug variable under ifndef
198ed5898 Merge pull request #760 from jwlent55/use-static-library-header-files
cc4d13e0a Put wal_filter under #ifndef ROCKSDB_LITE
385b41600 Merge pull request #764 from dmittendorf/fix-java-static-packaging
7062d0ea6 Make perf_context.db_mutex_lock_nanos and db_condition_wait_nanos only measures DB Mutex
1fe78a407 Fix tests failing in ROCKSDB_LITE
a6efefef7 Fix format specifiers
f7b2a7b40 Fix format specifiers
7f58ff7c3 Remove db_impl_debug from release build
1ddd91cd2 Fixed packaging of java crossbuild jar by forcing all compiled binaries to be output to the java/target directory.  The uber crossbuild jar is then assembled within the java/target directory.
eb2417855 merge from master
c64ae05b1 Move TEST_NewInternalIterator to NewInternalIterator
59a0c219b Adding log filter to inspect and filter log records on recovery
0be50ed12 Merge pull request #763 from PraveenSinghRao/lockfix
a1d37602a Fixing mutex to not use unique_lock
9e819d096 Modify the way java static builds are done so that: 1) There is no need to download and install the compression libraries twice just to get access to their header files during the compile phase. 2) Ensure that the compression library headers files used during the compile phase are the same ones used to build the static library that is linked into the library.
f1fdf5205 Clean up dependency: Move db_test_util.* to db directory
237994409 Fixed an incorrect replace of const value in util/options_helper.cc
0bb8ea56b [RocksDB Options File] Add TableOptions section and support BlockBasedTable
4a7970d75 Modify the way java static builds are done so that: 1) There is no need to download and install the compression libraries twice just to get access to their header files during the compile phase. 2) Ensure that the compression library headers files used during the compile phase are the same ones used to build the static library that is linked into the library.
c4366165e Merge pull request #759 from jwlent55/statically-load-compression-libraries
fa4b5b3db Fix for the travis build caused by my previous commit
3d07b815f Passing table properties to compaction callback
64546af83 Adding parser to CI jobs
def74f876 Deferred snapshot creation in transactions
c5f3707d4 DisableIndexing() for Transactions
776bd8d5e Pass column family ID to table property collector
5a7222782 Ensure that the compression libraries are statically linked into dynamic libraries included in the Java jar.  Also build the linux libraries using the portable flag to fix a problem with the linux32 build and improve the general portability of the RocksDB dynamic libraries. ==> linux32: util/crc32c.cc:318:39: error: ‘_mm_crc32_u64’ was not declared in this scope
e61d9c148 Make DBTest.AggregatedTableProperties more deterministic
b77eb16ab New Manifest format to allow customized fields in NewFile.
6732a5765 Merge pull request #756 from viveknelamangala/master
a52888ed0 Install snappy headers to standard locations using yum, so that build_tools/build_detect_platform sets -DSNAPPY flag to g++ . Current jars of rocksdb do no have snappy compression avaliable .
831101b5f Make it harder for users to run debug builds in production
000836a88 CompactionFilter::Context to contain column family ID
3a0bf873b Change RocksDB version to 4.1
77318ee14 Enable crash CI jobs
9803e0d81 compaction_filter.h cleanup
51fa7ecec Bytes read/written from cache statistics
f925208ff Create Makefile target unity_test
a065cdb38 bloom hit/miss stats for SST and memtable
40cdf797d Fix compile error on platforms without fallocate()
77e4ad7ce Fix compile failure on Travis
4049bcde3 Added boolean variable to guard fallocate() calls
aadf49fe6 Travis shouldn't fail when clang-format suggests improvements
d80ce7f99 Compaction filter on merge operands
726d9ce1a Disabling unity
026750265 Support for LevelDB SST with .ldb suffix
5855cdb6d Merge pull request #750 from yuslepukhin/fixup_build_options
7bbe10c01 Merge pull request #751 from yuslepukhin/return_noerror
e95b703b7 Mmap reads should not return error if reading past file
25c58a204 Add shared_linked DEBUG flag, remove port from among the include directories.
60b1c0a70 Fix to CI job definition
9babaeed1 Update dump_tool and undump_tool to accept Options
eb5b637fb Fix condition for bottommost level
9eaff629e Make corruption_test more robust
bf19dbff4 Fix valgrind - Initialize done variable
5c7bf56d3 [RocksDB Options] Support more options in RocksDBOptionParser for sanity check.
115427ef6 Add APIs PauseBackgroundWork() and ContinueBackgroundWork()
a39897369 Adding features to CI job description
a47bf325c Merge pull request #748 from yuslepukhin/improve_test_runs
65324a16d Improve test running scripts   Introduce proper command line arguments so we can control the script   Add appveyor support   Add an ability to run all other (non db_tests) test executables in parallel   Use .NET HashSet instead of empty valued hashtable to improve the looks   TODO: Some of the tests do not use GTests and need to improve log parsing
7e4ee4231 Merge pull request #743 from edsrzf/amalgamation
7a23e4d8c New amalgamation target
e9a6808c4 Merge pull request #745 from yuslepukhin/test_appveyor_baseline
9320ffd67 Improve CI build and fix Windows build breakage   Is there a way to enforce CMake additions for internal changes that seem to come   w/o a PR?
03b08ba9a Return MergeInProgress when fetching from transactions or WBWI with overwrite_key
c29af48d3 Add max_file_opening_threads to db_bench
da1cf8a9b Add a missing check for deprecated options in options_helper.cc
5a51fa907 Fix accidental object copy in transactions
1e73b11af Better handling of deprecated options in RocksDBOptionsParser
a8b8295d1 Fixed a compile warning in options_test.cc under clang
74b100ac1 RocksDB Options file format and its serialization / deserialization.
75134f756 Merge pull request #741 from yuslepukhin/test_appveyor_baseline
2e7506d82 Improve CI build and build switches   Add an optimized build config switch for faster test runs   Change compiler options to introduce more opitmizations and be more inline with MS internal switches.   Make appveyor build to utilize all the avaiable cores on the VM (parallel)   Introduce new appveyor configuration for daily test runs as it would take too long   to run db_test after each checkin even in paralell.   With some exclusions we make it in 38 minutes. We currently fail to install ramdisk during the build.   Add a powershell script to faicilitate paralell run for db_test cases.
1eff1834b Remove non-existing functions. Closes #680
16d1ba700 Clear SyncPoint Trace in DeleteSchedulerTests
30f74fa96 Make CompactionJobStatsTest.UniversalCompactionTest more robust
afe0dc539 SingleDelete support for Transactions
a263002a3 Fixed a tsan warning in db_stress.cc
e4861e7d6 Fixed a compile error in util/arena.h
0fdb4f168 Fixed a compile warning in util/arena.cc when hugetlb is not supported.
94ac8826c Merge pull request #737 from mlin/readonly-syncwal
60fa9cf0b Override DBImplReadOnly::SyncWAL() to return NotSupported. Previously, calling it caused program abort.
7df348b40 Minor fix to CI job definition
dac3f22b7 Fix the test failure
63e0f8679 Fixed a bug which causes rocksdb.flush.write.bytes stat is always zero
7ee445dd6 Fix the compile warning
174e2be5f Merge pull request #735 from jsteemann/fix-potential-leak-in-allocate
aa58958d3 prevent potential memleaks in Arena::Allocate*()
25fd743d7 Fix SingleDelete support in WriteBatchWithIndex
b6aa3f962 Fixed a memory leak issue in DBTest.UnremovableSingleDelete
7b7b5d9f1 [minor] Reuse SleepingBackgroundTask
e01f32cc3 Parameterizing email id
c58bac701 Fix valgrind failure due to memory leaks
a70d08ec0 Fix the bug of using freed memory introduced by recent plain table reader patch
628216fc1 Simplifying valgrind testing
4805fa0ea Remove ldb HexToString method's usage of sscanf
f03b5c987 Add experimental DB::AddFile() to plug sst files into empty DB
3fdb6e523 Fixed old lint errors in db/filename.cc
b349d2278 Fixed old lint errors in db/filename.h
df34aea33 PlainTableReader to support non-mmap mode
d746eaad5 RandomAccessFileReader should not inherit RandomAccessFile
03dd8f3ca Fixing punit job description
d0c31641d Internal stats WAL file synced to match meaning of the stats of the same name
48b4497f7 Merge pull request #730 from yuslepukhin/fix_write_batch_win_const_expr
a6c22e3e4 Disabling parallel test CI job
489a3e95d Re-work to support size_t max constant for 32/64-bit.
ff57c6511 [RocksJava] Fix test failure of InfoLogLevelTest
f1b9f804e Add a mode to always pick the oldest file to compact for each level
5e8f0a66d Use port::constant for std::muneric_limtis<>::max()
2754ec999 Fix Windows constexpr issue and '#ifdef' column_family_test in Release.
dd2e1eeb3 Disabling log running jobs
199744f4c Merge pull request #728 from jsteemann/fix-missing-include-header
0e65693f1 Merge pull request #727 from jsteemann/micro-optimization
4d6eb52d1 Fix to CI jobs.
3bcc072d2 Added more CI jobs
669b892f9 add missing header required for std::function
624ef456d fixed formatting. thanks @4tXJ7f for pointing me at `make format`
bbb18c827 removed unused variable of type Status, fixed indentation
470483335 pass input string to WriteBatch() by const reference
5ec129971 key_ cannot become nullptr, so no check is needed for that
834b12a8d made Size() function const because it does not modify data
1fc16cb2e Fix clang-format on Travis
e244bdf39 Merge pull request #725 from adamretter/fail-failed-java-test
7d937a090 Exit with non-zero status if one or more Java tests fail
1b598213a Check formatting in Travis
c7fba8029 Fix non-deterministic failure in backupable_db_test
014fd55ad Support for SingleDelete()
f35560d00 Merge pull request #723 from jsteemann/fix-typos
f8b770a94 fixed typos
51e1c1125 Do not flag error if file to be deleted does not exist
a5e312a7a Improving condition for bottommost level during compaction
9aca7cd6d DB::Open() to flush info log after printing DB pointer
cecd903ad Fix gflags build in Travis script
16934d495 Fix wrong constants in db_test_util
2e8e7eb39 Fix the verbosity issue in Java makefile
f21c7415a Change the log level of DB start-up log from Warn to Header.
9566342d2 Build gflags from source for Travis
3ebf11ed1 Adding the increment for a counter for a number of WAL syncs
81a61d75d Skipped tests shouldn't be failures [part 2]
1b7ea8ce8 Skipped tests shouldn't be failures
5ba3297d0 Add compaction time to log output
31a27a360 Callback for informing backup downloading added
d93a9f2aa [travis CI] Run ulimit -n 2000
2b683d497 Add DBOption.max_subcompaction to option dump
0e50a3fcc Merge issue with D46773
a7e80379b LogAndApply() should fail if the column family has been dropped
2819a1db3 Minor fix to CI job definition
df22e2fb7 Relax memory order for faster tickers
488607317 Adding Slice::difference_offset() function
5ce63e30e Merge pull request #720 from AMDmi3/fix-constant-overflow
925babc76 Merge pull request #721 from AMDmi3/printf-size_t
f171faa24 Fix printf format for size_t
4b0b0201c Fix `integer overflow in expression' error
f3170b6f6 DBImpl::FindObsoleteFiles() shouldn't release mutex between getting min_pending_output and scanning files
e467bf0de Fix valgrind error
7cb314b9e Skip some tests in ROCKSD_LITE
0bfe0573e Add gflags dependency to Travis script
5de807ac1 Add options.hard_pending_compaction_bytes_limit to stop writes if compaction lagging behind
7143242d1 Fix compaction_job_stats under ROCKSDB_LITE
592f6bf78 Merge pull request #716 from yuslepukhin/refactor_file_reader_writer_win
e2d6011f4 Minor fix to CI job definition
ad0d70ca1 Relax asserts in arena_test
03ddce9a0 Add counters for L0 stall while L0-L1 compaction is taking place
a3fc49bfd Transactions: Release Locks when rolling back to a savepoint
ddc8b4499 Address code review comments both GH and internal  Fix compilation issues on GCC/CLANG  Address Windows Release test build issues due to Sync
9f3a66a93 Improvements to CI jobs
7db1471cc Minor fix to CI job
c67d20689 Fixed arena_test failure due to malloc_usable_size()
34cedaff6 Initialize variable to avoid warning
aeb461268 Add counters for seek/next/prev
45e9e4f0b Refactor NewTableReader to accept TableReaderOptions
ddb950f83 Fixed bug in compaction iterator
30e82d5c4 Refactor to support file_reader_writer on Windows.   Summary. A change https://reviews.facebook.net/differential/diff/224721/   Has attempted to move common functionality out of platform dependent   code to a new facility called file_reader_writer.   This includes:   - perf counters   - Buffering   - RateLimiting
af7cdbf64 Run full test suite in Travis
c25f6a85b Removed __unused__ attribute
6db0a939d Fix DBCompactionTest failure with parallel L0-L1 compactions
8aa1f1519 Refactored common code of Builder/CompactionJob out into a CompactionIterator
41bce0586 CI job improvements
95ffc5d2b Correct ASSERT_OK() in ReadDroppedColumnFamily
3c37b3ccc Determine boundaries of subcompactions
112664408 Relaxing consistency detection to include errors while inserting to memtable as WAL recovery error.
abc7f5fdb Make DBTest.ReadLatencyHistogramByLevel more robust
ac9bcb55c Set max_open_files based on ulimit
4cbd2f9aa Merge pull request #714 from facebook/travisformac
d0df54d1f Run travis tests on OS X
a55e5a52a Merge pull request #711 from facebook/testtravis
2b676d5bb Upgrade travis to new architecture
c66d53fee Fixed minor issue in CompressionTypeSupported()
f3f2032c4 Release RocksDB 4.0.0
44b6e99e1 update max_write_buffer_number_to_maintain docblock
aa6eed0c1 Transaction stats
25dbc579f Update HISTORY file for transactions
52386a1e7 Minor fix to sandcastle jobs commands
b5b2b75e5 better tuning of arena block size
342ba8089 Make DBTest.OptimizeFiltersForHits more deterministic
e17e92ea1 Relaxed assert in forward iterator
5e94f68f3 TransactionDB Custom Locking API
0ccf2db38 Fixed broken build due to format specifier
6bdc484fd Added Equal method to Comparator interface
7a31960ee Tests for ManifestDumpCommand and ListColumnFamiliesCommand
778cf4449 Adding email notification.
3a0df7f16 Fixed comparison in ForwardIterator when computing hint for GetNextLevelIndex()
91f3c9079 Fix case when forward iterator misses a new update
ff1953c89 Merge pull request #707 from dkorolev/master
7b463e657 Fixed a typo in INSTALL.md
d9f42aa60 Adding a verifyBackup method to BackupEngine
50dc5f0c5 Replace BackupRateLimiter with GenericRateLimiter
20ef64cae Moving jobs to use gcc-4.9
0f1aab6c1 Add SetLockTimeout for Transactions
14456aea5 Fix compile
76f286cc8 Optimize bloom filter cache misses
0e6e5472e Fixed a compile warning in rocksjni/loggerjnicallback.cc
b8a962d4f Adding commands for few more CI jobs.
8a2d59a35 Add Cloudera's blog post to USERS.md
3c9cef1ee Unified maps with Comparator for sorting, other cleanup
3e0a672c5 Bug fix: table readers created by TableCache::Get() doesn't have latency histogram reported
b42cd6bed Remove the need for LATEST_BACKUP in BackupEngine
0f763db20 Merge pull request #705 from yuslepukhin/rate_limiter_fix
20c44fefb t6913679: Use fallocate on LOG FILESS
f14c3363e Make WinEnv::NowMicros return system time   Previous change for the function   555ca3e7b7 (diff-bdc04e0404c2db4fd3ac5118a63eaa4a)   made use of the QueryPerformanceCounter to return microseconds values that do not repeat   as std::chrono::system_clock returned values that made auto_roll_logger_test fail.
aad0572f8 Fixed the build issue of rocksdbjavastaticrelease
5508122ed Fix a perf regression in ForwardIterator
b72200777 Fix listener_test when using ROCKSDB_MALLOC_USABLE_SIZE
40cd91b7f Fixed compile warning in rocksdbjava
90415cfeb Fixed a compile warning in linux32 environment.
9d6503f88 Fix arena_test test break using glibc-2.17
77a28615e Support static Status messages
18db1e469 better db_bench options for transactions
0be260523 Merge pull request #702 from PraveenSinghRao/remove_spurious
8b689546b Add Subcompactions to Universal Compaction Unit Tests
c6d870ffb Merge branch 'arcpatch-D45741'
57b3a8773 Adding sandcastle determinator for RocksDB
3d78eb66b Arena usage to be calculated using malloc_usable_size()
effd9dd1e Fix deadlock in WAL sync
64f07deb8 remove spurious compression definitions
72a9b73c9 Removed unnecessary checks in DBTest.ApproximateMemoryUsage
cb164bfc4 Do not delete iterators for immutable memtables.
7a0dbdf3a Add ZSTD (not final format) compression type
e2db15efd Merge pull request #701 from PraveenSinghRao/usewinapi_notcruntime
e853191c1 Fix DBTest.ApproximateMemoryUsage
7c916a5d3 Merge pull request #699 from OpenChannelSSD/to_fb_master
0886f4f66 Helper functions to support direct IO
7e327980a Remove usage of C runtime API that has file handle limitation
8ef0144e2 Add argument --show_table_properties to db_bench
1fb2abae2 ColumnFamilyOptions serialization / deserialization.
5f4166c90 ReadaheadRandomAccessFile -- userspace readahead
16ebe3a2a Mmap reads should not return error if reading past file
d286b5df9 DBIter to out extra keys with higher sequence numbers when changing direction from forward to backward
3795449c9 Fix DBTest.GetProperty
a7834a129 Merge pull request #698 from yuslepukhin/address_noexcept_windows
9ccf1bd3e Correct the comment for GetProperty() API.
fbe2c05f5 s/NOEXCEPT/ROCKSDB_NOEXCEPT
6924d7582 Address noexcept and const integer lambda capture   VS 2013 does not support noexcept.    Complains about usage of ineteger constant within lambda requiring explicit capture.
2f8d71ec0 Moving sequence number compaction variables from SubCompactionState to CompactionJob
bab9934d9 Fix build failure caused by bad merge.
4d28a7d8a Add a whitebox test for deleted file iterators.
249fb4f88 Fix use of deleted file iterators with incomplete iterators
53b88784d Add throttling to multi-threaded backups
09d982f9e Fix compact_files_example
6996de87a Expose per-level aggregated table properties via GetProperty()
86d6c3cde Fix Windows build
20d1e547d Common base class for transactions
205083297 Fixing race condition in DBTest.DynamicMemtableOptions
e46bcc08b Remove an extra 's' from cur-size-all-mem-tabless
4ab26c5ad Smarter purging during flush
4c81ac0c5 Fix benchmark report script
b6def58f7 Changed 'num_subcompactions' to the more accurate 'max_subcompactions'
c85296846 db_iter_test: add more test cases for the data race bug
9130873a1 Add options.new_table_reader_for_compaction_inputs
07d2d3416 Add a counter about estimated pending compaction bytes
41a0e2811 Improve defaults for benchmarks
a203b913c Fixed a rare deadlock in DBTest.ThreadStatusFlush
962aa6429 Merge pull request #695 from yuslepukhin/address_windows_build
5bf890762 More indent adjustment.
e2a9f43d6 Adjust indent
6e9a260b0 Merge branch 'address_windows_build' of https://github.com/yuslepukhin/rocksdb into address_windows_build
1cac89c9b Address windows build issues  Intro SubCompactionState move functionality  =delete copy functionality  #ifdef SyncPoint in tests for Windows Release builds
f25f06ddd Address windows build issues   Intro SubCompactionState move functionality   =delete copy functionality   #ifdef SyncPoint in tests for Windows Release builds
027ca5b2c Total SST files size DB Property
b604d2562 Removing unused variables to fix build
1b114eed4 Free file iterators for files which are above the iterate upper bound to Improve memory utilization
3fd70b05b Rate limit deletes issued by DestroyDB
df79eafcb Introduce GetIntProperty("rocksdb.size-all-mem-tables")
888fbdc88 Remove the contstaint that iterator upper bound needs to be within a prefix
137c37667 Removing variables used only in assertions to prevent build error
b47cc5851 Bounding Number of Subcompactions
e58e1b18e Make tailing iterator show new entries in memtable.
9ec957159 DBOptions serialization and deserialization
b2df20a89 Make HashCuckooRep::ApproximateMemoryUsage() return reasonable estimation.
601b1aaca Fixing Failed Assertion in Subcompaction State Diff
f0da6977a [Parallel L0-L1 Compaction Prep]: Giving Subcompactions Their Own State
f32a57209 Simplify querying of merge results
72613657f Measure file read latency histogram per level
b7198c3af reduce db mutex contention for write batch groups
603b6da8b Add options.compaction_measure_io_stats to print write I/O stats in compactions
dc9d5634f Change master to 3.14
b78c8e07d Merge pull request #689 from msb-at-yahoo/add-tools-target
9f0dd2229 Add a 'tools' target.
463720712 Add test case to repro the mispositional iterator in a low-chance data race case
3bd9db420 [Cleanup] Remove RandomRWFile
c3466eab0 Have Transactions use WriteBatch::RollbackToSavePoint
0db807ec2 Transaction error statuses
c2f2cb021 Pessimistic Transactions
c2868cbc5 Use manual_compaction for compaction_job_test
6b2d57039 Fix Windows build by adding snapshot_impl to CMakeLists
e61fafbe7 Fixed clang-build error in util/thread_local.cc
cee1e8a08 Parallelize LoadTableHandlers
4249f159d Removing duplicate code in db_bench/db_stress, fixing typos
a03085b55 Fix linters on non-fb machines
1ae27113c reduce comparisons by skiplist
b47d65b31 Fixed Segmentation Fault in db_stress on OSX.
a1581eca8 Modernize RocksDB linters
a9dcc0a63 Fix clang build
2cf0f4f47 Adding wal_recovery_mode log message
68f934355 Better CompactionJob testing
22dcaaff3 More accurate time measurement for delete_scheduler_test
0a7ea582c Add auto-build manifest for appveyor
0093271ee Merge pull request #685 from flandr/fix-tls-build
ac04a6cfb Fix OSX + Windows build
16ea1c7d1 simple ManagedSnapshot wrapper
257ee895f Fixed memory leaks
254c4fb88 In HISTORY.md Switch unreleased notes to 3.13
40f893f4a Fix delete_scheduler_test valgrind error
6a4aaadcd Avoid type unique_ptr in LogWriterNumber::writer for Windows build break
d7314ba75 Fixing endless loop if seeking to end of key with seq num 0
48e6e9aa8 Add util/delete_scheduler_impl.cc to CMakeLists.txt
c7742452e Add Statistics.getHistogramString() to print more detailed outputs of a histogram
29b028b0e Make DeleteScheduler tests more reliable
fca88f8e1 valgrind_check to exit on test failures
7d364d0d9 Fix build failure
960d936e8 Add function 'GetInfoLogList()'
7ccd1c80a Add two unit tests for SyncWAL()
3ae386eaf Add statistic histogram "rocksdb.sst.read.micros"
8ecb51a7e "make commit-prereq" should clean up rocksjava properly
9aec75fbb Enable DBTest.FlushSchedule under TSAN
bd2fc5f5f Fix TSAN for delete_scheduler_test
8e01bd114 Fix misplaced position for reversing iterator direction while current key is a merge
c46507102 Removing duplicate code
e06cf1a09 [wal changes 3/3] method in DB to sync WAL without blocking writers
5dc3e6881 Update Tests To Enable Subcompactions
c45a57b41 Support delete rate limiting
102ac118b Update JAVA-HISTORY.md for v3.13
3a1d4e6c9 Merge pull request #670 from skunkwerks/fix_osx_shared_library_names
f5d072ae6 Fixed RocksJava test failure of shouldSetTestCappedPrefixExtractor
f39cbcb0a Merge pull request #654 from adamretter/remove-emptyvalue-compactionfilter
f0b5bcc7b add support for capped prefix extractor in java
18ba58a94 Upgrading jemalloc from 3.6.0 to the latest for fbcode+gcc 4.8.1
ce21afd20 Expose the BackupEngine from the Java API
b0d12a135 Merge pull request #569 from adamretter/travis-java-api
241bb2aef Make DBCompactionTest.SkipStatsUpdateTest more stable.
3424eeb1e Polish HISTORY.md
cf3e05304 crash_test cleans up directory before testing if TEST_TMPDIR is set
24daff6d7 Fix a typo and update HISTORY.md for NewCompactOnDeletionCollectorFactory().
14d0bfa42 Add DBOptions::skip_sats_update_on_db_open
e2a3bfe74 First half of whitebox_crash_test to keep crashing the same DB
2e73bd4ff crash_test to put DB under TEST_TMPDIR
1205bdbce crash_test to cover simply cases
d5c0a6da6 Merge branch 'master' of github.com:facebook/rocksdb
2d41403f4 Made change to fix the memory leak
92f7039ee fix memory corruption issue in sst_dump --show_compression_sizes
be8621ffa Fix compile warning in compact_on_deletion_collector in some environment
26894303c Add CompactOnDeletionCollector in utilities/table_properties_collectors.
20b244fcc Fix CompactFiles by adding all necessary files
87df6295d Make SuggestCompactRangeNoTwoLevel0Compactions deterministic
40c64434d Parallelize L0-L1 Compaction: Restructure Compaction Job
47316c2d0 dump_manifest supports DB with more number of levels
bd852bf11 Fixed typos in db_stress
544be638a Fixing fprintf of non string literal
193dc977e Fixing dead code in table_properties_collector_test
05d4265a2 Merge branch 'master' of github.com:facebook/rocksdb
4be6d4416 Compression sizes option for sst_dump_tool
8161bdb5a WriteBatch Save Points
7bfae3a72 tools/db_crashtest2.py should run on the same DB
d06c82e47 Further cleanup of CompactionJob and MergeHelper
e95c59cd2 Count number of corrupt keys during compaction
221a94a5f Another attempt at adding the Java API and tests to the travis build
1bdfcef7b Fix when output level is 0 of universal compaction with trivial move
6a82fba75 Add missing hashCode() implementation
f73c80143 Fixing Java tests.
14f413760 Correct the comment of DB::GetApproximateSizes
8279d4197 Merge pull request #667 from yuslepukhin/fix_now_microsec_win
eb8e3b4c7 Fix shared library names on OSX
555ca3e7b Fix WinEnv::NowMicrosec   * std::chrono does not provide enough granularity for microsecs and periodically emits     duplicates   * the bug is manifested in log rotation logic where we get duplicate    log file names and loose previous log content   * msvc does not imlement COW on std::strings adjusted the test to use     refs in the loops as auto does not retain ref info   * adjust auto_log rotation test with Windows specific command to remove     a folder. The test previously worked because we have unix utils installed     in house but this may not be the case for everyone.
82f148ef9 Fix test DBCompactionTest.PartialCompactionFailure undeterministic failure
6002801e0 Abandon ROCKSDB_WARNING
4922af6f8 fixed DBTest.GetPropertiesOfAllTablesTest and DBTest.GetUserDefinedTablaProperties flakiness
3bf9f9a83 cleaned up PosixMmapFile a little
fe09a6dae [wal changes 2/3] write with sync=true syncs previous unsynced wals to prevent illegal data loss
06aebca59 Report live data size estimate
85ac65536 Tests to avoid to use TMPDIR directly
f09d45690 Merge pull request #664 from yuslepukhin/add_tests_fix_sanity
31b35c902 Add missing tests, fix db_sanity  Add heap_test, merge_helper_test  Fix uninitialized pointers in db_sanity_test that cause SIGSEV when DB::Open fails in case compression is not linked.
66a3cbc54 Merge pull request #663 from yuslepukhin/fix_windows_build_refactor
ac5e441ad Fix windows build after refactoring   Missing and duplicate files in CMake   Missing definition of port::Crash
02b635fa3 Fix undeterministic failure of DBTest.GetPropertiesOfAllTablesTest
3dbf4ba22 RangeSync not to sync last 1MB of the file
7219088cd Move general compaction tests from db_test.cc to db_compaction_test.cc
0adecd9f4 Add db_inplace_update_test back to Makefile
064294081 Improved FileExists API
9f1de9518 Revert Makefile
6867fb19c Revert "Add missing db_log_iter_test in the test list"
443c6646b Move remaining universal compaction tests from db_test.cc to db_universal_compaction_test.cc
7462286d3 Move in-place-update related tests from db_test.cc to db_inplace_update_test.cc
03467bdd4 Add missing db_log_iter_test in the test list
331954ab8 Fixed DBTestUniversalManualCompactionOutputPathId test
a75f23eb8 Relax assertions in unit DropWrites to be more permissible
ee80432ff db_bench add an option of --universal_allow_trivial_move
58b4209e0 Trigger non-trivial compaction in fault_injection_test
59eca2cc9 Make memenv_test runnable in ROCKSDB_LITE
aa8ac6445 Skip unsupported tests in ROCKSDB_LITE
ce9712d34 Make mock_env_test runnable in ROCKSDB_LITE
c06d1d839 Make merge_test runnable in ROCKSDB_LITE
144d2910d Block backupable_db_test_lite in ROCKSDB_LITE
0d1d9aeeb Block plain_table_db_test in ROCKSDB_LITE
4853e228e Make table_test runnable in ROCKSDB_LITE
f0fe9126f Fix compile for write_callback_test in ROCKSDB_LITE
cf6a7bebc Block cuckoo table tests in ROCKSDB_LITE
20922c4a5 Make compaction_picker_test runnable in ROCKSDB_LITE
6e9fbeb27 Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
5ec829bc4 Cleaning up CYGWIN define of fread_unlocked to port
26ca89319 Block document_db_test in ROCKSDB_LITE
35ca59364 Don't let flushes preempt compactions
79373c372 Fix ROCKSDB_WARNING
74c755c55 Added JSON manifest dump option to ldb command
a96fcd09b Deprecate CompactionFilterV2
1d20fa9d0 Fixed and simplified merge_helper
aede5cd8e Merge pull request #656 from qinzuoyan/fb-master
d730c3677 Merge pull request #657 from yuslepukhin/ensure_clean_public_headers
ac2b9367f Fix a typo in variable
415c47323 Merge after rebasing
d1a457181 Ensure Windows build w/o port/port.h in public headers
6c0c8dee7 Fix data loss after DB recovery by not allowing flush/compaction to be scheduled until DB opened
e4af3bfb2 Test for compaction of corrupted keys
91bf1b80e Java facility to use the RemoveEmptyValueCompactionFilter
3d00271e4 The ability to specify a compaction filter via the Java API
62dec0e2b RemoveEmptyValueCompactionFilter - A compaction filter which removes entries which have an empty value
c5bca5319 Fix compile on Mac
487bba434 extend temp str buffer size
247690fe3 Ensure Windows build w/o port/port.h in public headers
84c3577af fix append bug in DumpDBFileSummary()
43e982562 Fix mongo build -take 2
d8263d958 Unbreak mongo build
12c5528a8 Bump to RocksDB 3.13
81d072623 move convenience.h out of utilities
beb19ad0d Fixing delete files in Trivial move of universal compaction
c61396069 Build fix.
2c8de0eca Update --help message in db_bench.
6b2d44b2f Refactoring of writing key/value pairs
e1c99e10c Replace std::priority_queue in MergingIterator with custom heap, take 2
9a6a0bd8c Style fix in compaction_job.cc
e94c510c3 Make ldb_test not depend on compression
ddad40e93 Fixed nullptr deref and added assert
1bc8eb877 make coverage should execute sequentially
801df912a Move UniversalCompaction related db-tests to db_universal_compaction_test.cc
3ca6b2541 Move TailingIterator tests from db_test.cc to db_test_tailing_iterator.cc
ce829c77e Make TransactionLogIterator related tests from db_test.cc to db_log_iter_test.cc
c3f98bb89 Move CompactionFilter tests in db_test.cc to db_compaction_filter_test.cc
0936362a7 Block SyncPoint in util/db_test_util.h in released Windows mode.
05e194158 Merge pull request #639 from cleaton/setMaxTableFileSize
fc2b71d9c Merge pull request #655 from adamretter/java-make-resolve-maven
8a9fca261 Better error handling in BackupEngine
18d5e1bf8 Remove db_impl_readonly dependency on utilities
9d22a9737 Resolve Java test dependencies from local maven repo if present
49640bd82 Allow write_batch_test to run with ROCKSDB_LITE
a9c510951 Deprecate purge_redundant_kvs_while_flush
5aea98ddd Deprecate WriteOptions::timeout_hint_us
ae29495e4 Avoid manipulating const char* arrays
ab137af4b Partial cleanup of CompactionJob
1879d9370 Add ldb_test.py back to `make check`
b10cf4e2e Move DynamicLevel related db-tests to db_dynamic_level_test.cc
e290f5d3c Block reduce_levels_test in ROCKSDB_LITE
04d201fa0 Block spatial_db_test in ROCKSDB_LITE
49f42ad03 Move global static functions in db_test_util to DBTestBase
625467a08 Move reusable part of db_test.cc to util/db_test_util.h
e8e8c9049 fix compile for optimistic_transaction_test under ROCKSDB_LITE
8bca83e5d Add tombstone information in CompactionJobStats
f9728640f "make format" against last 10 commits
76d3cd328 Fix public API dependency on internal codes and dependency on MAX_INT32
5fd11853c Print Fast CRC32 support information in DB LOG
a6e38fd17 Fix a uncleaned counter in PerfContext::Reset()
e41cbd9c2 Merge pull request #646 from yuslepukhin/ms_win_port
4cab5ebec Merge branch 'ms_win_port' of https://github.com/yuslepukhin/rocksdb into ms_win_port
296de4ae6 Address review comments  Rule of five: add destructor  Add a note to COMMIT.md for 3rd party json.
041b6f95a perf_context: report time spent on reading index and bloom blocks
d08ba9f0c Merge branch 'ms_win_port' of http://vstfbing:8080/tfs/Bing/_git/repo.RocksDB into ms_win_port
805fe84ba Conditional use of third-party libraries  Committed by Alexander Zinoviev <alexander.zinoviev@me.com> 7/9/2015 2:42:41 PM
5555cc500 Improve build system
c903ccc4c Merge from github/master
54d124a38 Conditional use of third-party libraries
7189e90c2 Fix a noisy unit test.
1f4d56570 Add db_bench flag to set cache_index_and_filter_blocks
5c7913233 Revert the changes related to Options, as requested to seperate them into a different patch.
d8586ab22  All of these are in the new code added past 3.10      1) Crash in env_win.cc that prevented db_test run to completion and some new tests      2) Fix new corruption tests in DBTest by allowing a shared trunction of files. Note that this is generally needed ONLY for tests.      3) Close database so WAL is closed prior to inducing corruption similar to what we did within Corruption tests.
4bed00a44 Fix function name format according to google style
e2e3d84b2 Added multi WAL log testing to recovery tests.
5219226d3 Merge branch 'ms_win_port' of http://vstfbing:8080/tfs/Bing/_git/repo.RocksDB into ms_win_port
ef4b87f1b Commit both PR and internal code review changes
95f4c2bcb Conditional use of 3rd-party libraries
4f56632b1 Fix occasional failure in compaction_job_test
411c8e3d1 Build fail fix
b7a2369fb Revert "Replace std::priority_queue in MergingIterator with custom heap"
c0b23dd5b Enabling trivial move in universal compaction
d8e3e766f Fixed a bug in test ThreadStatusSingleCompaction
57d216ea6 Remove assert(current_ == CurrentReverse()) in MergingIterator::Prev()
59b50dcef Update HISTORY.md for Listener
4ce5be425 fixed leaking log::Writers
685582a0b Revert two diffs related to DBIter::FindPrevUserKey()
e12b40399 Initialize threads later in constructor
58d7ab3c6 Added tests for ExpandWhileOverlapping()
155ce60da Fix compaction_job_test
b6655a679 Replace std::priority_queue in MergingIterator with custom heap
e25ee32e3 Arena needs mman header for mmap
d2f0912bd Merge the latest changes from github/master
35cd75c37 Introduce InfoLogLevel::HEADER_LEVEL
acee2b08a Fixed endless loop in DBIter::FindPrevUserKey()
218487d8d [wal changes 1/3] fixed unbounded wal growth in some workloads
feb99c31a Merge remote-tracking branch 'origin' into ms_win_port
e70115e71 Fix unity build by removing anonymous namespace
4159f5b87 Prepare 3.12
a69bc91e3 Multithreaded backup and restore in BackupEngineImpl
9dbde7277 Merge remote-tracking branch 'origin' into ms_win_port
03d433ee6 [RocksJava] Fixed test failures
326da912d Add string.h to Histogram as we init the array out of curly braces
ca2fe2c1b Address GCC compilation issues
19e13a595 Fix header inclusion
18285c1e2 Windows Port from Microsoft
c00948d5e [RocksJava] Fix test failure of compactRangeToLevel
05e283196 Allocate LevelFileIteratorState and LevelFileNumIterator from DB iterator's arena
436ed904d Add rpath option to production builds for 4.8.1 toolchain
b0f1927db Increasing timeout for drop writes.
ec70fea4c Fix a comparison in DBIter::FindPrevUserKey()
501591c42 Make column_family_test runnable in ROCKSDB_LITE
91cb82f34 Merge branch 'master' of github.com:facebook/rocksdb
09f5a4b48 set -e in fb_compile_mongo.sh
6199cba99 Fix race in unit test.
0a019d74a Use malloc_usable_size() for accounting block cache size
4cbc4e6f8 Call merge operators with empty values
619167ee6 Fix mac compile
472e64d39 Improve fb_compile_mongo.sh
c9cd404bc Make flush check for shutdown
4fb09c687 Updating SeekToLast with upper bound
dadc42976 Reproducible MongoRocks compile with FB toolchain
62a8fd154 Make stringappend_test runnable in ROCKSDB_LITE
48da7a9ca Improve the comment for BYTES_READ in statistics.
72cab8895 Block redis_test in ROCKSDB_LITE
dec2c9f56 Make table_properties_collector_test runnable in ROCKSDB_LITE
0b1ffe2e1 Remove -Wl,--no-as-needed flag when making shared_lib in OSX and IOS
674b1181c Bottommost level compaction option
782a1590f Implement a table-level row cache
de85e4cad Introduce WAL recovery consistency levels
530534fce Fix trivial move merge
7015fd81c Add read_nanos to IOStatsContext.
7160f5d80 Fix broken gflags link
dda74111a add setMaxTableFilesSize Options unit test
d62b6ed83 add setMaxTableFilesSize to JNI interface
e1d3c7dbe Fixing valgrind error in checkpoint_test
3bdec09cb Remove ldb_tests.py from make check until it is working again.
15325bf55 First version of rocksdb_dump and rocksdb_undump.
04251e1e3 Add wal files to Checkpoint for multiple column families.
18cc5018b Fix memory leaks in PinnedUsageTest
bf03f59c1 Disable CompressLevelCompaction() if Zlib is not supported
df719d496 Make autovector_test runnable in ROCKSDB_LITE
4d6d47688 Block geodb_test in ROCKSDB_LITE
71b438c4a Remove unused target --- compactor_test
eade498bd Block utilities/write_batch_with_index in ROCKSDB_LITE
760e9a94d Fail DB::Open() when the requested compression is not available
69bb210d5 Add Cache.GetPinnedUsageUsage()
4eabbdb7e Skip bottommost level compaction if possible
4b8bb62f0 Don't dump DBOptions for each column family
176f0bedc Merge branch 'master' of github.com:facebook/rocksdb
bb1c74ce1 Fixed a bug of CompactionStats in multi-level universal compaction case
a66b8157d Merge branch 'master' of github.com:facebook/rocksdb
f06be62fd Replace %llu with format macros in ParsedInternalKey::DebugString())
2dc3910b5 Add --benchmark_write_rate_limit option to db_bench
12e030a99 Use CompactRangeOptions for CompactRange
c89369f57 Move dockerbuild.sh to build_tools/
4716ab4d1 Merge pull request #638 from HolodovAlexander/master
25d600569 Clean up InstallSuperVersion
1369f015e Only initialize the ThreadStatusData when necessary.
1a08d0beb Block c_test in ROCKSDB_LITE
40f562e74 Allow GetApproximateSize() to include mem table size if it is skip list memtable
d59d90bb1 db_bench periodically writes QPS to CSV file
46296cc86 Cygwin build not to use -fPIC
bee8d033f Removed two unused macros in iostats_context
5fec96387 Fixed false alarm of size comparison in compaction_job_stats_test
cccd2199a Revert skip bottommost compaction
20f2b5425 Skip bottom most level compaction if no compaction filter
7842920be Slow down writes by bytes written
a84df655f Don't let two L0->L1 compactions run in parallel
d6ce0f7c6 Add largest sequence to FlushJobInfo
ab455ce49 fix clang build
3eddd1abe Add Env::GetThreadID(), which returns the ID of the current thread.
73faa3d41 Handling edge cases for ReFitLevel
bffaf0a8b Merge pull request #631 from mkhq/patch-1
821cff114 Re-generate WriteEntry on WBWIIterator::Entry()
8b7be1808 Updated OS X instructions, replace homebrew/dupes with homebrew/versions
d03f11090 Link all libraries when building shared libraries
75222d130 Revert "Fix compile"
47f1e7212 Merge pull request #630 from rdallman/c-wb-logdata
51440f83e Fix compile
4949ef08d Re-generate WriteEntry on WBWIIterator::Entry()
735df6655 C: add WriteBatch.PutLogData support
e409d3d74 Make "make all" work for CYGWIN
62c3a9579 Add test for iteration+mutation of WBWI
d9b3338eb Add Yahoo's blog post about Sherpa to USERS.md
75d7075a8 Print info message about files need compaction for debuging purpose
406a5682e Fix hang when closing a DB after doing loads with WAL disabled.
d8c8f08c1 GetSnapshot() and ReleaseSnapshot() to move new and free out of DB mutex
643bbbf08 Use nullptr for default compaction_filter_factory
21f2492ac Fix CYGWin release build
f02ce0c65 Fix ASAN errors in c_test
133130a4f Merge pull request #625 from rdallman/c-slice-parts-support
de4d172d0 Merge pull request #622 from rdallman/c-multiget
6df589b44 Add TablePropertiesCollector::NeedCompact() to suggest DB to further compact output files
2e764f06e [API Change] Improve EventListener::OnFlushCompleted interface
7322c7401 Revert incorrect commit
31e60e2a7 Unlock mutex in ReFitLevel
7647df8f9 Fixed the tsan failure in util/compaction_job_stats_impl.cc
b2785472c Fix compile
3ce3bb3da Allowing L0 -> L1 trivial move on sorted data
bb808eadd Changed the CompactionJobStats::output_key_prefix type from char[] to string.
0b3172d07 Add EventListener::OnTableFileDeletion()
211a195d4 C: add MultiGet support
5dc174e11 C: add support for WriteBatch SliceParts params
2d0b9e5f0 Fix compile on darwin
3af668ed1 Fix DBTest.MigrateToDynamicLevelMaxBytesBase slowness with valgrind
408cc4b8e Revert "Merge pull request #621 from rdallman/c-slice-parts-support"
78382d4ba Merge pull request #621 from rdallman/c-slice-parts-support
ca8b85ac0 better document max_write_buffer_number_to_maintain
0483dab2a Remove a TODO that has been done
8afafc278 Fix compile warning in db/db_impl
fe5c6321c Allow EventListener::OnCompactionCompleted to return CompactionJobStats.
3083ed212 Fixed heap-use-after-free error in compaction_job_test.cc
8d8d4e45b Fixed ROCKSDB_LITE compile error due to the missing of TableFileCreationInfo
ab946af08 Fix a compile warning in listener_test.cc
fc8382127 Add EventListener::OnTableFileCreated()
898e803fc Add a stats counter for DB_WRITE back which was mistakenly removed.
ac81130fa Fix Bug: CompactRange() doesn't change to correct level caused by using wrong level
ec7a94436 more times in perf_context and iostats_context
4266d4fd9 Allow users to migrate to options.level_compaction_dynamic_level_bytes=true using CompactRange()
d333820ba Removed DBImpl::notifying_events_
495ce6018 Fixed compile warning in compact_files_example.cc
2ecac9f96 add rocksdb::WritableFileWrapper similar to rocksdb::EnvWrapper
a187e66ad Merge pull request #617 from rdallman/wb-merge-sliceparts
16c197627 Fixed db_stress
4c181f08b Fix compile on darwin
bc7a7a400 fix LITE build
832271f6b Fixed a compile warning in db_stress in NDEBUG mode.
dc9d70de6 Optimistic Transactions
d5a0c0e69 Fixed a compile warning in db_stress
ebfdb3c7f Fixed a compile error in ROCKSDB_LITE
9ffc8ba02 Include EventListener in stress test.
a3da59022 Decrease number of jobs in make release
21cd6b7ad C: add support for WriteBatch SliceParts params
a0635ba3f WriteBatch.Merge w/ SliceParts support
c81535103 Support saving history in memtable_list
ec4ff4e99 Rename EventLoggerHelpers EventHelpers
672dda9b3 [API Change] Move listeners from ColumnFamilyOptions to DBOptions
3ab8ffd4d Compaction now conditionally boosts the size of deletion entries.
a81ac2412 Merge pull request #615 from rdallman/master
6d299b70b Fixed a bug in EventLoggerHelpers::LogTableFileCreation
a0580205c Removed an unused private variable in db_impl.h
328ad902a update an import path to fit in with the rest of the kids
9c38ce1d0 C: extra bbto / noop slice transform
8d26799fe Merge pull request #614 from arschles/docker
32198343f fix typo in c_simple_example
6116ccc23 moving dockerfile to root
d90cee9fd adding docker build script and dockerfile
ea6d3a8ac Don't skip last level when calculating compaction stats
5c224d1b7 Fixed two bugs on logging file deletion.
dc81efe41 Change the log-level of DB summary and options from INFO_LEVEL to WARN_LEVEL
687214f87 Ensure ColumnFamilyOptions.num_levels >= 2 when level compaction is used.
2abb59268 Avoid logging under mutex in DBImpl::WriteLevel0TableForRecovery().
309a9d076 Run tests sequentally if J=1
7fee8775a Allow EventLogger to directly log from a JSONWriter.
7a3577519 Don't artificially inflate L0 score
4cb4d546c Set stats_dump_period_sec to 600 by default
e2c1d4b57 [Public API Change] Make DB::GetDbIdentity() be const function.
eaf61ba9f Minor text correction
f16c0b289 Merge pull request #613 from DerekSchenk/DerekSchenk-patch-issue-606
d1a978ae3 Rename JSONWritter to JSONWriter
3e0817541 Add LDFLAGS to Java static library
812c461c9 Dump db stats in WARN level
b588505a7 Update HISTORY.md for GetThreadList() update.
944043d68 Add --wal_bytes_per_sync for db_bench and more IO stats
d5de04d20 Update history for 3.11
08b6b3796 FORCE_GIT_SHA
04feaeebb Fix comparison between signed and usigned integers
4a855c079 Add an option wal_bytes_per_sync to control sync_file_range for WAL files
b0fdda4ff Allow flushes to run in parallel with manual compaction
74f3832d8 Fixed compile errors due to some gcc does not have std::map::emplace
0c8017dba Remove duplicated code
fb5bdbf98 DBTest.DynamicLevelMaxBytesCompactRange: make sure L0 is not empty before running compact range
6fa708512 CompactRange skips levels 1 to base_level -1 for dynamic level base size
84a9c6a53 add comment
eeb44366b C api: human-readable statistics
3f0867c0f Allow GetThreadList to report Flush properties.
a66f643e9 Use a better way to initialize ThreadStatus::kNumOperationProperties.
7413306d9 Take a chance on a random file when choosing compaction
8c52788f0 Use version defined in Makefile in rocksdb_build_git_sha
5aad88129 DBTest.DynamicLevelMaxBytesBase2: remove an unnecesary check
ec43a8b9f Universal Compaction with multiple levels won't allocate up to output size
714fcc067 Make ThreadStatus::InterpretOperationProperties take const uint64_t*
bc68bd5a1 db_bench to support rate limiter
df1f87a88 Fixed compile error in db/column_family.cc
14431e971 Fixed a bug in EventListener::OnCompactionCompleted().
dbd95b753 Add more table properties to EventLogger
b5881762b Reset parent_index and base_index when picking files marked for compaction
711465cce API to fetch from both a WriteBatchWithIndex and the db
3996fff8a Fix clang build - add override
d97813906 SuggestCompactRange() is manual compaction
2fe24e4b5 Don't treat warnings as error when building rocksdbjavastatic
beda81c53 rm -rf in make clean
50eab9cf3 Fix BackupEngine
962f8ba33 Bump to 3.11
37bbd3185 Merge pull request #601 from adamretter/jdb-bench
77a5a543a Allow GetThreadList() to report basic compaction operation properties.
65fe1cfbb Cleanup CompactionJob
df4130ad8 fix crashes in stats and compaction filter for db_ttl_impl
7ea769487 Fix flakiness in column_family_test
a2c4cc756 Don't treat warnings as error when building release
9aa011fa3 Optimize GetRange Function
36a740889 Fix UNLIKELY parenthesis
9bdbaab94 Update USERS.md
2ab7065af build: avoid unused-variable warning
88044340c Add Size-GB column to benchmark reports
d2346c2cf Fix hang with large write batches and column families.
b6b100fe0 Remove iter_refresh_interval_us
197f01b7b Bugfix remove deprecated option use which was removed in previous commit 019ecd19329ee895284e9b040df0ffe4c08b35d8
aa094e8ea Fix conversion from nano-seconds to milli-seconds and seconds
dddceefe5 Fix clang build
d4540654e Optimize GetApproximateSizes() to use lesser CPU cycles.
fd96b5540 Making GetOptions() comment better (#597)
7246ad34d Don't compact bottommost level in SuggestCompactRange
7f47ba0e2 Fix possible SIGSEGV in CompactRange (github issue #596)
aba005c44 Merge pull request #585 from fyrz/RocksJava-RemoveDeprecatedStuff
d6f39c5ae Helper function to time Merges
a087f80e9 Add scripts to run leveldb benchmark
1bb4928da Include bunch of more events into EventLogger
3db81d535 Fix memory leak in cache_test introduced in the previous commit
4961a9622 Fix build
93ab1473d Merge pull request #593 from charsyam/feature/type-1
6ede020dc fix typos
3d1af4ae6 Don't preinstall jemalloc in Travis
242f9b4c2 Fix CLANG build issue introduced in previous commit
794ccfde8 Task 6532943: Rocksdb - SetCapacity() can dynamically change cache capacity if feasible
98a44559d Build for CYGWIN
d01bbb53a Fix CompactRange for universal compaction with num_levels > 1
e003d3864 Abstract out SetMaxPossibleForUserKey() and SetMinPossibleForUserKey
fd7a35731 Enable open source users to use jemalloc (github issue #438)
aa14670b2 Add an assertion in CompactionPicker
2dc421df4 Implement DB::PromoteL0 method
9bf40b64d Print max score in level summary
397b6588b options.paranoid_file_checks to read all rows after writing to a file.
283a04296 Set --seed per test
618d07b06 Making PreShutdown tests more reliable.
0a91bca5d test: avoid vuln-inducing use of temporary directory
6e359419f Add rpath for production builds
78dbd087d Improve benchmark scripts
6a5ffee0c Fix gflags Makefile
108a927f0 Merge pull request #589 from coderplay/patch-1
a58fd7427 Update USERS.md
d85d08c7b One last fix to Makefile
2db96dca1 Fix make install when there is no shared lib
7d136994c Get rid of error output
79c1b021a Fix Makefile
742fa9e31 Fix compile with two gflags
79c21ec0c skip ioctl-using tests when not supported
6059bdf86 Add experimental API MarkForCompaction()
acf8a4141 maint: use ASSERT_TRUE, not ASSERT_EQ(true; same for false
b5400f90f Kill dead code
48b0a045d Speed up reduce_levels_test
00c2afcd3 Fix bug in ExpandWhileOverlapping()
019ecd193 [RocksJava] Remove deprecated methods
98ef21d2f Merge pull request #584 from pshareghi/rocksdb-3.10-falloch
5b7131c75 [RocksJava] Removed deprecated skipLogErrorOnRecovery methods.
566f65271 [RocksJava] Removed deprecated ColumnFamilyDescriptor methods
6997aa0b6 Merge pull request #582 from fyrz/RocksJava-Fix-RateLimiter
582c4b0f7 [RocksJava] Fix RateLimiter Tests in 3.10
6cfb2150d Merge pull request #581 from vladb38/patch-3
d71e8f7ad Update USERS.md
debaf85ef Bug of trivial move of dynamic level
12d7d3d28 Fix and Improve DBTest.DynamicLevelCompressionPerLevel2
a1271c6c6 Fix build break introduced by new SyncPoint interface change
fcb206b66 SyncPoint to allow a callback with an argument and use it to get DBTest.DynamicLevelCompressionPerLevel2 more straight-forward
281db8bb6 Temporarily disable test CompactFilesOnLevelCompaction
e8808b912 Added falloc.h in build_detect_platform
1983fadcb assert(sorted) in vector rep
9da874801 Get benchmark.sh loads to run faster
9b983befa Fix flakiness of WalManagerTest
d41a565a4 Don't do O(N^2) operations in debug mode for vector memtable
08be1803e Fix bad performance in debug mode
0a0501c8d Add Xfunc to makefile
e7ad14926 Fix flakiness in FIFOCompaction test (github issue #573)
abb405227 Kill benchharness
894e9f745 Update Patent Grant.
590fadc40 Fix compile warning on CLANG
47b874398 Make Compaction class easier to use
753dd1fdd Fix valgrind issues in memtable_list_test
697380f3d Repairer documentation improvement.
2f66d7f92 Add LinkedIn back to USERS.md
0feeee643 Fix memtable_list_test
7b9581bc3 Fixed xfunc related compile errors in ROCKSDB_LITE
fabc11569 MemTableList tests
9741dec0e Fix a compile error in ROCKSDB_LITE in db/db_impl.cc
465b25ca9 "make commit-prereq" doesn't really build ROCKSDB_LITE
d2a056241 Fix a compilation error in ROCKSDB_LITE in db/internal_stats.h
316ec80bf fault_injection_test: add a test case to cover log syncing after a log roll
ed229a0de Fixes for readcache-flashcache
91df4e969 Remove use of whole-archive to include jemalloc
84c5bd7eb Add thread-safety documentation to MemTable and related classes
ee9bdd38a Script to check whether RocksDB can read DB generated by previous releases and vice versa
2b019a151 Enabling checksum in repair db as it should have been.
b1bbdd791 Create EnvOptions using sanitized DB Options
edbb08b5d Fix Makefile
199313dc3 build: create .o files specifically for java-related targets
b118238a5 Trivial move to cover multiple input levels
e7adfe690 Fix formatting of USERS.md
4e7543dcf Add USERS.md
58346b9e2 Log writer record format doc.
db6569cd4 Fix the compilation error in flashcache.cc on Mac
cba592001 build: don't use a glob for java/rocksjni/*
c66483c13 Fix github issue #563
de22c7bd1 Integrate Jenkins with Phabricator
f12614070 Fix TSAN build error of D36447
824e64634 Adding another NewFlashcacheAwareEnv function to support pre-opened fd
5e067a7b1 Clean up compression logging
e3ee98b38 run 'make check's rules (and even subtests) in parallel
a45e7581b Avoid naming conflict of EntryType
3be82bc89 Add p99.9 and p99.99 response time to benchmark report, add new summary report
953a885eb A new call back to TablePropertiesCollector to allow users know the entry is add, delete or merge
d2a92c13b avoid returning a number-of-active-keys estimate of nearly 2^64
a7ac6cef1 Fix level size overflow for options_.level_compaction_dynamic_level_bytes=true
089509b84 db_test: clean up sync points in test cleaning up
afbafeaea Disallow trivial move if compression level is different
d0695f3e2 Fix crash caused by opening an empty DB in readonly mode
51c8133a7 Fix make unity build compiler warning about "stats" shadowing global variable
df71c6b9e Script to trigger jenkins test
38a01ed1b Update COMMIT.md
76d63b452 Fix one non-determinism of DBTest.DynamicCompactionOptions
b23bbaa82 Universal Compactions with Small Files
2511b7d94 Makefile minor cleanup
1bd70fb54 Add --stats_interval_seconds to db_bench
fd3dbef22 Clean up old log files in background threads
99ec2412e Make the benchmark scripts configurable and add tests
2158e0f83 Fix clang build
d61cb0b9d db_bench can now disable flashcache for background threads
1c47c433b build: always attempt to update util/build_version.cc
e018892bb Formalize the DB properties string definitions.
cfa576402 Make auto_sanity_test always use the db_sanity_test.cc of the newer commit.
e9fddb7a2 Merge pull request #560 from xiaoxichen/patch-1
bcd8a71a2 Fix interger overflow on i386 arch
030859eb5 Dump compression info on startup
3539e0644 Merge pull request #558 from aamihailov/master
a3e4b3248 fix compilation error (same as fix #284)
ff1ff7c62 TSAN: avoid new link failure with -pg
39d508e34 Add a missing section title in HISTORY.md
727684bf9 Fixed a typo in RocksDBSample.java
ca25e86ef build: cause the "check" rule to depend on $(PROGRAMS)
986bdc680 Merge branch 'master' of github.com:facebook/rocksdb
1e57f2bf2 Fix build
2d417e52d Update HISTORY.md for 3.10.0
cbd6a2073 Merge branch 'master' of github.com:facebook/rocksdb
211ca26ae Fixing build issue
2495f9396 Merge pull request #555 from pshareghi/staticLZ4
f06de5f23 Merge pull request #556 from fyrz/RocksJava-Maven-Fix
cd987c383 Fix compile error when NROCKSDB_THREAD_STATUS is not used.
8f104ae5e [RocksJava] Maven build fix
3d1a924ff Adding stats for the merge and filter operation
afc51649e Merge pull request #546 from fyrz/RocksJava-MemEnv
4806cc126 Added static lz4 support for roccksjavastatic
6284eef4c Merge pull request #545 from fyrz/RocksJava-Level-Compression
248c063ba Report elapsed time in micros in ThreadStatus instead of start time.
315abac94 Undeprecate GetLiveFiles()
a057bb2a8 Improve ThreadStatusSingleCompaction
868968c8e Merge branch 'master' of github.com:facebook/rocksdb
689391406 Make SSTDumpTest.GetProperties less noisy
8d8656243 Merge pull request #551 from fyrz/RocksJava-JavaDoc-Fix
f8c505b23 Merge pull request #550 from fyrz/RocksJava-Sample-Fix
46443bfa9 [RocksJava] Add missing JavaDoc annotations
864b7e88f [RocksJava] Java sample bugfix
2b2394cbb [RocksJava] DBBenchmark option for RocksMemEnv
fd8804f97 [RocksJava] Expose MemEnv in RocksJava Summary: In 3.10 the C++ code was extended with a MemEnv implementation. This is now also available in RocksJava.
004b89fba [RocksJava] Add compression per level to API
51da3aab4 Merge pull request #536 from fyrz/RocksJava-32-Bit-adjustment
b7e88cfb5 Merge pull request #543 from fyrz/RocksJava-Logger-Comment
38d286f14 Clean-up WAL directory before running db_test
28bc6de98 rocksdb: print status error message when (ASSERT|EXPECT)_OK fails
9405b5ef8 rocksdb: Remove #include "util/string_util.h" from util/testharness.h
220d0dff7 rocksdb: Remove #include "util/random.h" from util/testharness.h
b088c83e6 Don't delete files when column family is dropped
17ae3fcbc rocksdb: initial util/testharness clean up
39f4271be [RocksJava] Enhanced Logger comment
5615e23d8 [RocksJava] 32-Bit adjustments
836bcc2f0 Merge pull request #532 from fyrz/RocksJava-LevelCompactionDynamicLevelBytes
67d995808 rocksdb: fix make unity
52e0f3353 Clean up compactions_in_progress_
6b626ff24 rocksdb: change db_test::MultiThreadedDBTest as value parameterized test.
9720ea4de A build option to run through all check-in requirements.
0831a3599 Add a DB Property For Number of Deletions in Memtables
f7ed65464 Fix RocksJava test failure due to deprecation of table_cache_remove_scan_count_limit
51301b869 Enable dynamic changing of rate limiter's bytes_per_second
652db51a3 Fix compilation error in rocksjni/write_batch_test.cc
dfccc7b4e Add readwhilemerging benchmark
c345d1ee8 [RocksJava] Integrated changes for D34857
12350115d [RocksJava] Added LevelCompactionDynamicLevelBytes to Options
230e68727 Fix TSAN failue in env_test
155d468c5 Using chrono as a fallback
81345b90f Create an abstract interface for write batches
46214df4a Only run db_test in Travis
c88ff4ca7 Deprecate removeScanCountLimit in NewLRUCache
b4b69e4f7 rocksdb: switch to gtest
413e35273 Merge pull request #540 from dalgaaf/wip-da-fix-elif
969aa806b util/xfunc.h: fix #elif check for NDEBUG
87c7d49d6 util/env_posix.cc: fix #elif check for __MACH__
c86e5d7b9 stack_trace.cc: fix #elif check for OS_MACOSX
98c37fda5 Remove unused parameter in CancelAllBackgroundWork
9fd6edf81 rocksdb: Replace ASSERT* with EXPECT* in functions that does not return void value
d4d42c02e Fixed clang build in env.h
b2b308652 Speed up rocksDB close call.
a7aba2ef6 rocksdb: Add gtest
95344346a rocksdb: Small refactoring before migrating to gtest
bd4963e64 rocksdb: update reference to the latest version of clang dev in fbcode
56337faf3 Fix compaction IO stats to handle large file counts
eafa1bfc3 Merge pull request #529 from fyrz/RocksJava-Logger
ac03c4d51 Merge pull request #522 from fyrz/RocksJava-Optimize-Filters-For-Hits
c6967a1a5 Make RecordIn/RecordOut human readable
c8da67032 Stop printing per-level stall times.
04778a94c [RocksJava] OptimizeFiltersForHits
57f2a00c6 RocksJava - JNI Logger callback
814627af3 [RocksJava] Change log level at runtime in custom logger
a3bd4142f [RocksJava] Logging JNI callback
58878f1c6 Switch to use_existing_db=1 for updaterandom and mergerandom
12134139e Fixed the unit-test issue in PreShutdownCompactionMiddle
fd1b3f385 Fix the issue in PreShutdownMultipleCompaction
56c4a9c76 Fix compile warning in thread_status_util.h on Mac
417367c42 Fix SIGSEGV when not using cache
e25ff039c Prevent slowdowns and stalls in PreShutdown tests
f69071265 Speed up db_bench shutdown
c1b3cde18 Improve the robustness of ThreadStatusSingleCompaction
8c12426c9 Fix the deadlock issue in ThreadStatusSingleCompaction.
b16ead531 DBTest.DynamicLevelCompressionPerLevel should not run without snappy support
a5e60bafc Fix a typo / test failure in ThreadStatusSingleCompaction
cb2c91850 Don't run some tests is snappy is not present
c594b0e89 Allow GetThreadList() to report operation stage.
2623b2cf6 Include chrono
52d8347a9 EventLogger
756532daf Merge pull request #524 from fyrz/RocksJava-Test-Fix
47a2b3a40 Merge pull request #534 from fyrz/RocksJava-Fix-BrokenJacocoReport
9d22a1f13 Allow negative Wnew
2c84303aa Merge pull request #528 from fyrz/RocksJava-NativeLibraryLoader
2dc636f62 [RocksJava] Fix broken jacoco report
f210b0f6c [RocksJava] Fix NativeLibraryLoader
3ebebfccd Prevent xxhash symbols from polluting global namespace
53996149d Removing unnecessary kInlineSize
b411d0603 Prevent stalls in preshutdown tests
1d43bc41f Fixing segmentation fault in db_bench
e9de8b65a Change the way options.compression_per_level is used when options.level_compaction_dynamic_level_bytes=true
2b785d76b Fixed a bug where CompactFiles won't delete obsolete files until flush.
6f5579868 Fixed a compile error in db_bench in mac.
05d92efa7 Add convenience.cc to src.mk
2884b100b db_bench: Better way to randomize repeated read keys in -read_random_exp_range
284be570c Provide a mechanism to inform Rocksdb that it is shutting down
2ddf53b2c Get OptimizeFilterForHits work on Mac
480b28476 Fix make_new_version.sh
89597bb66 Allow GetThreadList() to report the start time of the current operation.
37921b499 db_bench: Add Option -read_random_exp_range to allow read skewness.
485ac0dbd Add rate_limiter to string options
e126e0da5 Single threaded tests -> sync=0 Multi threaded tests -> sync=1 by default unless DB_BENCH_NO_SYNC is defined
dc4532c49 Add --thread_status_per_interval to db_bench
34c75e984 fix-up patch: avoid new link error
ebc647de8 build: fix missing dependency problems
492f6d27e Fix a segfault in fbson under Mac OS X compiler
0d13bbe27 RocksJava] Fix ColumnFamily tests
67533809f [RocksJava] Fixed CompactionTest
1b7b997b8 [RocksJava] Remove MaxValue from Statistics
f862b3812 [RocksJava] Fix cleanup in tests
a01b59259 [RocksJava] DefaultColumnFamily Memory Fix
22c73d15b [RocksJava] Fix ColumnFamily tests
694988b62 Fix a bug in stall time counter.  Improve its output format.
b8d23cdcb Revert chrono use
db0373934 options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
f29b33c73 Add functionality to pre-fetch blocks specified by a key range to BlockBasedTable implementation.
c4bd03a97 Fix typo in log message
3cf7f353d Instrument memtable seeks
216a9e16f Fix compile
b9ff6b050 Fix a bug in ReadOnlyBackupEngine
afa8156af adding stdlib to fbson
f9c14a42e Fix compile on Mac
a9f0e2b5b Fix compile
e7c434c36 Add columnfamily option  optimize_filters_for_hits to optimize for key hits only
ba9d1737a RocksDB on FreeBSD support
4ba119df5 rocksdb: Update path to dev clang in fbcode
8984e5f84 Fix race in sync point.
03b432d4b rocksdb: Fix uninitialized use error
ccef2a766 Merge pull request #518 from fyrz/RocksJava-Native-Library-Loader-Fix
9fcf1a7b0 [RocksJava] RocksJava Testcases
03bbf718c Return fbson
62247ffa3 rocksdb: Add missing override
1e06a4068 Support builds for MongoDB+RocksDB
b74ad6632 Merge pull request #508 from fyrz/RocksJava-Final-Patch
d9f4875e5 Disable pre-fetching of index and filter blocks for sst_dump_tool.
182b4ceac Limit key range to number of keys, not number of writes
a360bb61b Merge pull request #516 from fyrz/RocksJava-Update-Statistics-To-3.10
8c7684474 [RocksJava] Updated TickerTypes and Histogram to 3.10
7f0c77cb3 [RocksJava] Integrated changes from D33417
819e787bb [RocksJava] Final usage correction
5139e678b Upgrade compiler in Travis
4ade89962 Fix compile error on MacOS.
ace3d8506 Revert "Unused managed iterator"
7b8f348e5 Attempt at fixing travis issue
d85993998 Merge pull request #506 from fyrz/RocksJava-Raw-Use
30e93c9b9 Merge pull request #505 from fyrz/RocksJava-Redundant-Modifier
217854dc4 Introduce DISABLE_WARNING_AS_ERROR in Makefile
bd339a979 Unused managed iterator
174a79c99 LevelDb include guard replaced with #pragma once
6fdda8ac4 rocksdb: changes to support 'make analyze' in Jenkins
96d989f70 catch config errors with L0 file count triggers
62f7a1be4 rocksdb: Fixed 'Dead assignment' and 'Dead initialization' scan-build warnings
5636c4a80 Verbose build in travis
a047409ae Fixed a bug in the test case
4f514a53d build: enable more compiler warnings
a2b911b63 inputs: restore "const" attribute removed by D33759
1b4082581 mark as unused some variables with cpp-derived names
c6d54b503 fix erroneous assert: cast kBlockSize (of type unsigned int) to "int"
aa5d8e6d9 table_test.cc: add missing 5th arg in TestArgs initializer
c37937a9c maint: remove extraneous "const" attribute from return type
9283c7afd build: remove always-true assertions
06a766de5 Adding Flush to AutoRollLogger
92416fa7f Fix mac build
96ab15d30 GetOptionsFromString + fixes to block_based_table_options
73711f956 rocksdb: Fix scan-build bug 'Memory leak' in db/db_bench.cc
98870c7b9 rocksdb: Fix scan-build memory warning in table/block_based_table_reader.cc
a42324e37 build: do not relink every single binary just for a timestamp
d45a6a400 Add rocksdb.num-live-versions: number of live versions
11581b741 build: abbreviate AR command, too
b8ac71ba1 Revert "Fbson to Json"
7ce1b2c19 Fbson to Json
7d817268b Managed iterator
b4b8c25a5 build: factor out AM_LINK command (trivial)
dc885c6e9 build: make "make" output readable by default
a37b46ae1 build: fix Makefile inconsistencies (trivial)
55277c328 build: remove unused rules: rocksdb_shell, DBClientProxy_test
3ad6b794c rocksdb: Fix 'Division by zero' scan-build warning
12753130e Remove ThreadStatusMultiCompaction test
f0c36da6e Add thread_status_util_debug.cc back
daebb1f91 build: running "make" with no arguments must not "uninstall"
e60bc99fe Allow GetThreadList to reflect flush activity.
b9a0213cd build: fix unportable Makefile syntax
4e4b85784 rocksdb: Fix scan-build 'Called C++ object pointer is null' and 'Dereference of null pointer' bugs
b3fd16226 build: remove unportable use of sed in favor of $(CXX)'s -MT
e7ea51a8e Introduce job_id for flush and compaction
6a0e737d9 [RocksJava] Raw use of parametrized class
439701270 [RocksJava] Redundant access-modifier in interfaces
2d62e8051 Merge pull request #504 from fyrz/RocksJava-Flush-Correction
eaf39568e [RocksJava] FlushOptions Correction
5d1151deb Added simple monitoring script to monitor overusage of memory in db_bench
5f00af457 DBTest.DestroyDBMetaDatabase: create DB directories if not exists
68af7811e Remember whole key/prefix filtering on/off in SST file
fd5970b45 Merge pull request #503 from weiweisd/master
513ad866b modify double type euqal compare in json_document.cc
933973dc2 Merge pull request #495 from fyrz/RocksJava-CF-Name-Byte-Handling
5e8e453d5 [RocksJava] Integrated changes from D33165
677d02427 [RocksJava] CF Name shall handle bytes correctly
d1cafc089 Merge pull request #1 from facebook/master
6d6305dd7 Perf Context to report DB mutex waiting time
863009b5a Fix deleting obsolete files #2
1851f977c Added RocksDB stats GET_HIT_L0 and GET_HIT_L1
91ac3b206 Print DB pointer when opening a DB
bee4e5124 Merge pull request #492 from fyrz/logger-logv-virtual
cfe8837e4 Switch logv with loglevel to virtual
aaceef363 Fix formatting
ee4aa9a0e Merge pull request #481 from mkevac/backupable
82faa377a added simple example for db restore from backup
d090330c8 fixed c_simple_example and added some comments
965130830 renamed backup to backup_and_restore in c_test for clarity
bbb52b21f Merge pull request #483 from adamretter/restructure-java-build
7e50ed8c2 Added some more wrappers and wrote a test for backup in C
218c3ecea Fix std::cout data race
8f679c290 Merge branch 'master' of github.com:facebook/rocksdb
da9cbce73 Add Header to logging to  capture application level information
2a979822b Fix deleting obsolete files
8e83a9d31 Add a missing field for STATE_MUTEX_WAIT to global_state_table
6f1013035 Fix DestroyDB
7de4e99a8 Revert "Fix wal_dir not getting cleaned"
9a52e06a0 Add GetID to ColumnFamilyHandle
181191a1e Add a counter for collecting the wait time on db mutex.
f36d394ae Fix wal_dir not getting cleaned
53ae09c39 db_test: fix a data race in SpecialEnv
fe9f69119 Fix fault_injestion_test
b37f5ffc7 Put db_test back to TESTS in Makefile
108470e96 Fix stack trace on mac
3e53760fc Fix compaction_picker_test
e39f4f6cf Fix data race #3
e63140d52 Get() to use prefix bloom filter when filter is not block based
678503ebc Add utility functions for interpreting ThreadStatus
4d98e2935 rocksdb: Enable scan-build static analysis
756e1f151 Remove unused util/thread_event_info.h
dad98dd4a Changes for supporting cross functional tests for inplace_update
9898f6398 Divide test DBIteratorTest.DBIterator to smaller tests
829363b44 Options::PrepareForBulkLoad() to increase parallelism of flushes
b04408c47 Fix unity build
8d3819369 NewIteratorWithBase() for default column family
2c2d5ab7e Fix compile warning in util/xfunc.h
0b8dec717 Cross functional test infrastructure for RocksDB.
868bfa403 Merge pull request #488 from ekg/master
9900f3821 Merge pull request #484 from fyrz/RocksJava-Release-MD-change
e6eaf938c remove old debugging message (#487)
f33f3955e Moved Java Samples main classes into samples/src/main/java
ad325517f Update test lib versions and maven plugin versions
d6187d07b Maven can now build a standard project layout
157768890 Moved Java Benchmark main classes into benchmark/src/main/java
dd8d5471e Adjustment to NativeLibraryLoader to allow native library to be loaded from either java.library.path or from extracting from the Jar. Means that the test in the build do not need to rely on the Jar, useful when creating similar builds (and executing tests) from Maven
353db6dae Moved Java main classes into src/main/java
98cb501bc Moved Java test classes into src/test/java
7479a62a7 Release.md - Remove version change instrcution
4a4e4279f Update HISTORY-JAVA.md
384cb6619 Merge pull request #480 from fyrz/RocksJava-Deprecate-SkipLogError
ca52a67cf [RocksJava] Deprecate setSkipLogErrorOnRecovery
114d21878 Merge pull request #479 from fyrz/RocksJava-Snapshot-Sequence-Number
cb5c3159f [RocksJava] Snapshot - GetSequenceNumber
ea189b320 Merge pull request #474 from fyrz/RocksJava-GetUpdatesSince
391f85fc8 [RocksJava] Incorporated changes for D32151
68cd93b87 [RocksJava] GetUpdatesSince support
caedd40dd [RocksJava] Adjusted auto pointer
b39006e3d [RocksJava] enable/disable File deletions
9a456fba2 [RocksJava] GetUpdatesSince support
d3a736761 Merge pull request #482 from fyrz/RocksJava-TTL-Fix
939bb3659 [RocksJava] Fix ColumnFamily name alloc in TTL DB
86e2a1eee Allow creating backups from C
db9ed5fdb Unaddressed comment in previous diff. Change only in code comments.
5917de0ba CappedFixTransform: return fixed length prefix, or full key if key is shorter than the fixed length
6c6037f60 Expose Snapshot's SequenceNumber
2fd8f750a Compile MemEnv with standard RocksDB library
173c52a97 Fix build on older compilers -- emplace() is not available
d07fec3bd make DBTest.SharedWriteBuffer to pass MockEnv
4bdf38b16 Disable FlushSchedule when running TSAN
e84299c76 Fix bug recently introduced in MemFile::Lock()
e5aab4c2b Fix data race in HashLinkList
2113ecd3c Merge pull request #472 from fyrz/RocksJava-Cleanup
10af17f3d fault_injection_test: add a unit test to allow parallel compactions and multiple levels
0c4d1053d Fix data race #5
cc0d8be01 [RocksJava] Integrated review comments (D32145)
5257c9c42 Merge pull request #452 from robertabcd/backupable-mem
560ed402b [minor] fprintf to stderr instead of stdout in test
551a41df3 Merge pull request #476 from alabid/alabid/add-to-simple-example
d2a2b058f fault_injection_test: to support file closed after being deleted
f8f040ccc Updated .gitignore to ignore *~ files and example object files
e8bf2310a Remove blob store from the codebase
ea7d0b943 Added WriteBatch block to simple_example.cc
d6c7300cc Fixed a compile warning in clang in db/listener_test.cc
f9758e012 Add compaction listener.
e919ecedf SuperVersion::Unref() to use sequential consistency to decrease ref counting
4c49fedaf Use ustricter consistency in thread local operations
1b43ab58d fault_injection_test: add more logging and makes synchronization slightly stronger
ca2b00277 [RocksJava] Cleanup portal.h & tests
f8dc5c459 [RocksJava] Add missing test to Makefile
b3c133148 [RocksJava] Removed todo comment in portal.h
7ffcc457f [RocksJava] Cleanup portal.h
c4fb83441 Update the comment for the removal of mac-install-gflags.sh
be8f0b12e Rename DBImpl::log_dir_unsynced_ to log_dir_synced_
c1de6c42a fault_injection_test: add a test case to drop random number of unsynced data
d888c9574 Sync WAL Directory and DB Path if different from DB directory
58f34edfc Fix valgrind
f1c886247 Fix data race #1
b08b2fe73 Merge pull request #471 from fyrz/RocksJava-Fix-NativeLibraryLoader
e61f38e5a [RocksJava] Fix native library loader
26b50783d Fix assert in histogramData
42189612c Fix data race #2
f5a839835 Fix archive WAL race conditions
43ec4e68b fault_injection_test: bring back 3 iteration runs
c2e8e8c1c Fix two namings in fault_injection_test.cc
b4c13a868 fault_injection_test: improvements and add new tests
a52dd0024 Fix ASAN failure with backupable DB
910186c27 Return the build with 4.8.1
401d4205e Add thread sanitizer
b068f0a67 Upgrade our compilers
a76d92862 Merge pull request #466 from fyrz/RocksJava-Support-ReadOptions-Iterator
bef7821f0 [RocksJava] ReadOptions support in Iterators
3b494a610 Make options_test runnable on ROCKSDB_LITE
912c52e82 Merge pull request #465 from fyrz/RocksJava-BlockBasedTable-FormatVersion
cd4c07197 Update HISTORY.md for GetThreadStatus() support on compaction.
46a7048dc Reduce false alarm in ThreadStatusMultipleCompaction test
aed028698 Merge pull request #462 from fyrz/RocksJava-JNI-allocation-correction
e5df90f5d Fix comment (minor)
dd53428f8 Incorporated review comments
908258a4f [RocksJava] BlockBasedTableConfig 3.10
2efe22849 [RocksJava] Incorporated changes for D31809
4e48753b7 Sync manifest file when initializing it
e204a5a16 [RocksJava] ColumnFamily name JNI correction
96264784d [RocksJava] ColumnFamily name JNI correction
ae82849bc Fix build failure
423dee841 Abort db_bench if Get() returns error
206237d12 DBImpl::CheckConsistency() shouldn't create path name with double "/"
5e98e5324 Merge pull request #458 from fyrz/RocksJava-TTLDB-Support
4ffe0be41 [RocksJava] Integrated changes for D31449
e82856754 [RocksJava] Integrated changes from D31449
859c54a03 [RocksJava] TTL-Support
5ff8aec4d [RocksJava] TTL Support
ca47da9e6 [RocksJava] TTL-Support
1190ebe5a Merge pull request #461 from fyrz/RocksJava-DirectSlice-Fix
ea25ff715 [RocksJava] Integrated proposed simplificiation
d68e83c35 [RocksJava] DirectSlice String termination fix
0ddf5f73e memenv: normalize file path
4d9d5955a Merge pull request #464 from fyrz/RocksJava-Various-Fixes
ceaea2b72 Adding prefix_extractor string config parameter
3d628f8f2 Update format_version comment
155bec4cf fallocate also tests FALLOC_FL_KEEP_SIZE
c75c02e7a [RocksJava] WriteBatchWithIndexTest fix
c787fb50b [RocksJava] JavaDoc errors in Java8
b229f970d Remove Compaction::ReleaseInputs().
f2ddb8b45 Fix for bug where GeoDB accesses key after next modification of iterator
d10f1de2b Ported LevelDB's fault_injection_test
2bb059007 Change db_stress to work with format_version == 2
9ab5adfc5 New BlockBasedTable version -- better compressed block format
2355931c6 Merge pull request #450 from adamretter/writebatch-with-index
3d246c89c Abstract duplicate code on key and value slice objects into generic methods
2d0dd8db3 Implement WBWIRocksIterator for WriteBatchWithIndex in the Java API
de678b288 Abstractions for common iterator behaviour
e01acb3a0 Test for WriteBatchWithIndex#newIterator()
56f24941a Simplify the Java API by permitting WriteBatchWithIndex to be provided straight to RocksDB#write
95d5f9848 Test for RocksDB#write(WriteBatchWithIndex)
ef5b34dee Implement WriteBatchWithIndex in the Java API
c6e554561 Abstractions for common write batch behaviour
be905491b Test for WriteBatchWithIndex#newIteratorWithBase(org.rocksdb.RocksIterator)
2241e3f4d Extract the interface for a RocksIterator
a8cfa7ace Extract the interface for a WriteBatch
45e43b81d Adds support for db->DefaultColumnFamily() to the Java API
516a04267 Add LZ4 compression to sanity test
2ccc54301 Merge pull request #460 from neutronsharc/master
2a7bd0ea4 Remove duplicated method declarations in C header.
bb128bfec More accurate message for compaction applied to a different version
96b8240bc Support footer versions bigger than 1
53f615df6 Fix clang build
02b30202c Merge pull request #455 from Andersbakken/stdlib_fix
2159484dd Remove two unnecessary blank lines in db/db_test.cc
e7dd88c57 Merge pull request #441 from fyrz/RocksJava-ColumnFamilyDescriptor-Alignment
d2c018fd5 Make ThreadStatusMultipleCompaction more robust.
bf9aa4dfc Improve GetThreadStatus to avoid false alarm in some case.
c91cdd59c Allow GetThreadList() to indicate a thread is doing Compaction.
402c1152a Fix c_simple_example
a9ea65d65 Build with clang 3.5 on Linux.
23ad5f401 [RocksJava] Incorporated changes for D30525
0aab1005f [RocksJava] ColumnFamilyDescriptor alignment with listColumnFamilies
15d2abbec Fix build issues
abb9b95ff Move compression functions from port/ to util/
9132e52ea DB Stats Dump to print total stall time
93b35c299 Merge pull request #453 from fyrz/SimpleCExampleSigSegv
628a67b00 Reduce memory footprint in backupable db.
ef3901642 Fixed memory issue in c_simple_example
b89d58dfa :%s/build_config/make_config
242b9769c Memtablerep Benchmark
73ee4feba Add comments about properties supported by DB::GetProperty() and DB::GetIntProperty()
2dca48f55 Merge pull request #451 from StanislavGlebik/document_db_improvement
4b57d9a82 Fixed negative numbers comparison in DocumentDB
9ef59a09a VersionSet::AddLiveFiles() to assert current version is included.
4d16a9a63 VersionBuilder to optimize for applying a later edit deleting files added by previous edits
7731d51c8 Simplify column family concurrency
07aa4e0e3 Fix compaction summary log for trivial move
9d5bd411b benchmark.sh won't run through all tests properly if one specifies wal_dir to be different than db directory.
62ad0a9b1 Deprecating skip_log_error_on_recovery
fa0b126c0 Fix corruption_test -- if status is not OK, return status -- during recovery
d7b4bb62a Fail DB::Open() on WAL corruption
9619081d9 Merge pull request #449 from robertabcd/improve-backupable
49376bfe8 Fix errors when using -Wshorten-64-to-32.
a8c5564a9 Do not issue extra GetFileSize() calls when loading BackupMeta.
caa1fd0e0 Improve performance when loading BackupMeta.
e9ca35815 Fix CLANG build for db_bench
bf287b76e Add structures for exposing thread events and operations.
a801c1fb0 db_bench --num_hot_column_families to be default off
2067058a6 Dump routine to BlockBasedTableReader (valgrind)
ddc81440d db_bench to add an option as number of hot column families to add to
a944afd35 Fixed a compile error in db/db_impl.cc on ROCKSDB_LITE
7ea7bdf04 Dump routine to BlockBasedTableReader
ae508df90 Clean up compile for c_simple_example
b62300961 Fix compile of compact_file_example
ded26605f Merge pull request #444 from adamretter/java-api-fix
98490bccf Fix the build on Mac OS X
4d9972974 Merge pull request #443 from behanna/master
5045c4394 add support for nested BlockBasedTableOptions in config string
d232cb156 Fix the build with -DNDEBUG.
45bab305f Move GetThreadList() feature under Env.
4fd26f287 Only execute flush from compaction if max_background_flushes = 0
0acc73881 Speed up FindObsoleteFiles()
d8c4ce6b5 Merge pull request #442 from alabid/alabid/fix-example-typo
949bd71fd fix really trivial typo
f8999fcf3 Fix a SIGSEGV in BackgroundFlush
ade4034a9 MultiGet for DBWithTTL
fdb6be4e2 Rewritten system for scheduling background work
a3001b1d3 Remove -mtune=native because it's redundant
e27c84522 Merge pull request #437 from fyrz/RocksJava-SliceTests-Fixes
1fed1282a [RocksJava] Incorporated changes D30081
5b9ceef01 [RocksJava] JavaDoc correction
5fbba60b6 [RocksJava] Incorporated changes D30081
b0230d7e0 [RocksJava] Incorporate additions for D30081
b015ed0ca [RocksJava] Slice / DirectSlice improvements
4d422db01 Merge pull request #430 from adamretter/increase-parallelism
04c4e4969 Merge pull request #411 from fyrz/RocksJava-RangeCompaction
62d19b7b5 Merge pull request #427 from haneefmubarak/c-examples
28424d734 style fixes in c example
7198ed5a2 Handle errors during pthread calls
91c58752f error detection and memory leaks in c example
25f70a5ab Avoid unnecessary unlock and lock mutex when notifying events.
7661e5a76 Move the file copy out of the mutex.
17e84f215 Rudimentary test cases for setIncreaseParallelism
eda0dcdd9 Exposed IncreasedParallelism option to Java API as setIncreasedParallelism
efc94ceb2 [RocksJava] Incorporated changes for D29283
69188ff44 [RocksJava] CompactRange support
48adce77c [RocksJava] CompactRange support
153f4f071 RocksDB: Allow Level-Style Compaction to Place Files in Different Paths
06eed650a Optimize default compile to compilation platform by default
cef6f8439 Added 'dump_live_files' command to ldb tool.
7ab1526c0 Add an assert and avoid std::sort(autovector) to investigate an ASAN issue
74b3fb6d9 Fix Mac compile errors on util/cache_test.cc
d7a486668 Improve scalability of DB::GetSnapshot()
ee95cae9a Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
0ab0242f3 VersionBuilder to use unordered set and map to store added and deleted files
e93f044d9 add range scan test to benchmark script
cb82d7b08 Fix #434
046ba7d47 Fix calculation of max_total_wal_size in db_options_.max_total_wal_size == 0 case
1b7fbb9e8 Update HISTORY.md for release 3.9
635c61fd3 Fix problem with create_if_missing option when wal_dir is used
2871bc7bc Merge pull request #422 from fyrz/RocksJava-Quality-Improvements
8c5781666 Add -fno-exceptions flag to ROCKSDB_LITE.
1f04066ca Add DBProperty to return number of snapshots and time for oldest snapshot
6436ba6b0 Provide mechanism to restart tests from previous error
d84b2bade Replace exception by abort() in dummy HdfsEnv implementation.
9260e1ad7 Bump version to 3.9
8f4e1c1c9 Remove the compability check on log2 OS_ANDROID as it's already blocked by ROCKSDB_LITE
c4a7423c1 Replace runtime_error exception by abort() in thread_local
a94d54aa4 Remove the use of exception in WriteBatch::Handler
a5d4fc0a2 Fix compile warning in db_stress
1a8f4821a Replace exception by assertion in autovector
97c194088 Fix compile warning in db_stress.cc on Mac
5f719d720 Replace exception by setting valid_ = false in DBIter::MergeValuesNewToOld()
c0dee851c Improve formatting, add missing newlines
815f638cd Fix java build
32a0a0384 Add Moved(GB) to Compaction IO stats
a14b7873e Enforce write buffer memory limit across column families
3e684aa68 Integrated changes from D29571
37d73d597 Fix linters
a15169f2e Fixed a Lint problem
b7f9e644c [RocksJava] Quality improvements
e002a6122 [RocksJava] Comparator tests for CF
335e6ad5c [RocksJava] Remove obsolete dbFolder cleanup
b036804ac RocksJava - FindBugs issues
9a632b4a9 Merge pull request #429 from fyrz/RocksJava-MacOSX-strip-fix
b42667506 [RocksJava] MacOSX strip support
e463cb0bc Merge pull request #424 from eile/master
91d898163 Tweak Makefile for building on BG/Q
c6f31a289 minor memory leak in C example
703ef66a8 Merge pull request #426 from fyrz/RocksJava-Restore-PrecisionFix
ac4ed1e30 fix examples/makefile for C example
d7f5ccb0c add c example to makefile and fix "make clean"
9c34d5e36 fix type in C simple example
0a9a7e753 added C version of simple_example
bcf908689 Block Universal and FIFO compactions in ROCKSDB_LITE
67cb7ca75 [RocksJava] Fixed MacOS build of RocksJava
b8136a7d2 Merge pull request #398 from fyrz/RocksJava-CreateCheckPoint
533592a27 Merge pull request #401 from fyrz/RocksJava-Sigsegv-MergeOperatorName
73d72ed5c Block ReadOnlyDB in ROCKSDB_LITE
e47f0fa9e Merge pull request #425 from adamretter/macosx-clean-fix
ff0cb90d1 Do not delete Java Fatal Error Log, developers may still want these for reference
2a792cd30 There will also be a librocksdbjni-osx.jnilib.dSYM folder on MacOSX builds to be deleted
beb74c14c Fix travis-build error
a486352e0 Merge pull request #423 from zerebubuth/c_iterate_upper_bound
26109d487 Store upper bound `Slice` with the same lifetime as the `ReadOptions` so that we can provide a pointer to it.
a97314219 Fix compile error in ROCKSDB_LITE
9d5019327 Replace log2 by implementing Log2 in options_builder
805bac6d2 Add test for upper bounds on iterators using C interface.
f193deea3 [RocksJava] Addressed comments in D28971
94f70a86b [RocksJava] Incoroporated changes for D29013
a280af2a5 [RocksJava] Sigsegv fix for MergerOperatorByName
fcc2dfd9f [RocksJava] Support for stored snapshots
274ba6270 Block internal_stats in ROCKSDB_LITE
4f2e8bab5 Merge pull request #421 from fyrz/RocksJava-PrecisionFix
c4765dc10 [RocksJava] Fix precision problem in rocksjni
14788e181 Merge pull request #420 from rdallman/add-wal
88dd8d889 c api: add max wal total to opts
7e608e2fe Block plain_table_index.cc in ROCKSDB_LITE
13de000f0 Add rocksdb::ToString() to address cases where std::to_string is not available.
90ee85f8e Improve listener_test to avoid possible false alarm
2946e37a0 remove unreliable test in db/cuckoo_table_db_test.cc
9c7ca65d2 free builders in VersionSet::DumpManifest
7530c75ab Merge pull request #413 from saghmrossi/master
d699d7034 Make RocksDB compile without gflags
325722149 Fixes valgrind error in GetSnapshotLink. Free checkpoint now.
ada3d7873 Merge pull request #415 from fyrz/RocksJava-Makefile
569853ed1 Fix leak when create_missing_column_families=true on ThreadStatus
c4b65f70f [RocksJava] Makefile correction
141018016 Make arena use hugepage if possible
3a40c427b Fix db_bench on CLANG mode
9222a2d02 Fixed iOS build caused by GetThreadList feature.
aa31fc506 Improve listener_test by ensuring flushes are completed before assert.
7ec71f101 Provide default implementation of LinkFile, don't break the build
cd278584c Clean up StringSplit
d84069995 Fix mac compile
4f882924d Merge pull request #404 from fyrz/RocksJava-Backup-Restore-3.8
4b63fcbff Add enable_thread_tracking to DBOptions
bafce6197 first rdb commit
9e285d423 Added CompatibleOptions for compatibility with LevelDB Options
353307758 Add IOS_CROSS_COMPILE to macro guard for GetThreadList feature.
eecdebe65 Fixed the destruction order of static variables in ThreadStatusImpl.
004f416b7 Moved checkpoint to utilities
beabc6879 Fixed ~ThreadStatusImpl().
faa8d32be [RocksJava] Integrated changes from D29019.
3d78c7a8c [RocksJava] Lint adjustments
d7529b2de [RocksJava] Cleanup Backupable implementations
fa703efb2 [RocksJava] Improved BackupableDBTest
24fdc4741 [RocksJava] Backupable/Restorable DB update 3.8.0
9972f969e [RocksJava] BackupableDBOptions alginment + 3.8
fbc42a093 Fixed -Werror=unused-but-set-variable in thread_status_impl
a564be715 Fix asan error in thread_status_impl.cc
7165d1886 Fix clang compile error
d0c5f28a5 Introduce GetThreadList API
1fd1aecb3 Merge pull request #409 from fyrz/RocksJava-Make-cleanup
91c8dcefc [RocksJava] Strip library in publish
e7fcaa4d9 [RocksJava] JavaDoc is executed too often
2cd1794e4 [RocksJava] Make cleanup - Clean Target
be005e17b fix clang compilation
5e69f19c4 Merge pull request #405 from fyrz/RocksJava-Convenient-Options
55a344872 Merge pull request #408 from fyrz/Missing-include
9e9a83baf Missing header in build on CentOS
91ccc8ebe [RocksJava] Integrated changes in D29025
5249d0db5 [RocksJava] Convenience methods for Options
8d3f8f969 remove all remaining references to cfd->options()
1e4a45aac remove cfd->options() in DBImpl::NotifyOnFlushCompleted
517c28994 Options helper supports k, m, g, and t unit suffixes
c46c2be8d Merge pull request #397 from fyrz/RocksJava-GetIntProperty
8efd4bb42 [RocksJava] Improved comments in RocksDB class
5529c1ad1 [RocksJava] GetIntProperty in RocksDB
db59eeb61 Merge pull request #406 from fyrz/Build-Fix
e97f014b9 [RocksJava] JavaDoc corrections - Java8
98e59f981 Fixed a bug which could hide non-ok status in CompactionJob::Run()
ec24bd4e6 Merge pull request #402 from adamretter/bugfix-native-library-loader
d3c4a0f4a Improve the comment in InfoLogLevelTest.java
a77e97c53 Merge pull request #396 from fyrz/RocksJava-LogLevel
585c759cf Make sure to use the correct Java classloader for loading the RocksDB Native Library
c3915abba Minor tidyup and use Java 7 for file copying
a122a42bb Merge pull request #399 from fyrz/RocksJava-Version-to-3.8
b8d5e3f08 [RocksJava] MVN Build reads version from version.h
23295b74b Clean job_context
0ce38fe98 Fix signed/unsigned compile
e7960c03a Don't parallelize the build in travis
84af2ff8d Clean job context in DeleteFile
8a1bcc39c [RocksJava] Bump version to 3.8 in rocksjni.pom
5c04acda0 Explicitly clean JobContext
4947a0674 [RocksJava] Incorporated review comments D28947
07cd3c42a [RocksJava] LogLevel support in Options
26dc5da96 Fix compaction_job_test
5f583d2a9 Merge pull request #394 from lalinsky/cuckoo-c
353303a76 Merge pull request #380 from fyrz/RocksJava-Junit-Framework
3f9c95a51 [RocksJava] Minor lint correction
e46450da6 [RocksJava] Rebased + integrated CF tests
cd82beb0c [RocksJava] Merged in latest changes.
b6abab8b7 [RocksJava] Merged & rebased to HEAD
74057d6d2 [RocksJava] Improved tests within RocksJava
628e39e97 [RocksJava] Integrated review comments from D28209
a4b28c1ae [RocksJava] Extended Testcases
36f3a0bb8 [RocksJava] Integrated review comments from adamretter in D28209
b09268695 [RocksJava] Extended testcases
9bec23c41 [RocksJava] Test-framework integration
f617135d5 [RocksJava] Testcase improvements
1fe7a4c62 [RocksJava] Test-framework integration
04ca7481d Fix build
6c1b040cc Provide openable snapshots
9be338cf9 CompactionJobTest
c9fd03ec5 Update docs for NewAdaptiveTableFactory
e6c3cc657 Add very basic tests to make sure the C cuckoo table options compile and run
7fe247080 Update HISTORY.md for RocksJava
c44a29278 Add cuckoo table options to the C interface
136b8583b Merge pull request #395 from lalinsky/fix-env-test
94fa542f8 Update HISTROY.md for 3.8 release
a177742a9 Make db_stress built for ROCKSDB_LITE
746cfaac5 Relax the block count check on deallocation in env_test
f822129b3 Add a unit test for behavior when merge operator and compaction filter co-exist.
4161de92a Fix SIGSEGV
373c665ed Fix broken test in 31b02d.
772bc97f1 No CompactFiles in ROCKSDB_LITE
1d1a64f58 Move NeedsCompaction() from VersionStorageInfo to CompactionPicker
cd0980150 Add concurrency to compacting SpatialDB
3c92e5233 Fix include
25f273027 Fix iOS compile with -Wshorten-64-to-32
fa50abb72 Fix bug of reading from empty DB.
31b02dc21 Improve Backup Engine.
1033db29f Merge pull request #390 from fyrz/RocksJava-Cleanup
9a03da773 Merge pull request #375 from fyrz/RocksJava-ColumnFamilyOptions-Extension-3.6
d50c68e3a [RocksJava] JavaDoc cleanup warnings with Java8
079d942ea [RocksJava] Code-cleanup + Java7 warnings removed
9a255b95f [RocksJava] Sample and Default value
9d2ba2136 [RocksJava] Incorporated review comments
fa9cfc65f [RocksJava] Integrated Review comments from yhchiang in D28023
75010d208 [RocksJava] ColumnFamily custom Options API extension
0345c2156 [RocksJava] Extend Options with ColumnFamilyOptions implementation ColumnFamilyOptions implementation with tests [RocksJava] Extended ColumnFamilyTest
975949522 Fixed clang compile error in version_builder_test
581141935 Fixed GetEstimatedActiveKeys
1f621e6ab Fix additional -Wshorten-64-to-32 errros
767777c2b Turn on -Wshorten-64-to-32 and fix all the errors
113796c49 Fix NewFileNumber()
625e162c6 Merge pull request #393 from fyrz/RocksJava-Flush
fc6fcbab9 [RocksJava] Flush functionality
8e5547f64 [RocksJava] Makefile restructured
26f0a78b0 Merge pull request #363 from adamretter/write_batch-iterate
35c8c814e Make ForwardIterator::status() more efficient
d88568c68 Move -Wnon-virtual-dtor to c++ flags
c7ee9c3ab Fix -Wnon-virtual-dtor errors
746252197 Merge pull request #384 from msb-at-yahoo/compaction-filter-2-empty-changed-values
dd726a59e Bump Version Number to 3.8
4a3bd2bad Optimize usage of Status in CompactionJob
bcdb9671c Fix build
d904fbbb0 Addresed comments from code review https://reviews.facebook.net/D27567
eeb9cf6c4 Test for WriteBatchHandler
8fb4751d5 Iterator support for Write Batches
00211f9c5 Fix SIGSEGV in db_stresS
a4a2bfd6b Merge pull request #391 from Liuchang0812/fixmake
856059094 Merge pull request #389 from Liuchang0812/master
01a770637 remove unused target
dc3410463 Merge branch 'fix-example'
e7620536c fix make static_lib error
543df158c Expose sst_dump functionality as library call.
e3d3567b5 Get rid of mutex in CompactionJob's state
344edbb04 Fixed the shadowing in db/compaction.cc and include/rocksdb/db.h
b8b390342 Fixed compile error in db/db_impl.cc
b622ba5d6 Fixed compile error in db/flush_job.cc
642ac9d8a Fixed compile error in db/compaction.cc and db/compaction_picker.cc
68effa034 Fix -Wshadow for tools
844786189 Fixed -WShadow errors in db/db_test.cc and include/rocksdb/metadata.h
28c82ff1b CompactFiles, EventListener and GetDatabaseMetaData
5c9309053 Turn on -Wshadow
31342c400 Fix implicit compare
a0f887c9e Fix compile
53af5d877 Redesign pending_outputs_
ec101cd49 Correctly test both compaction styles in CompactionDeletionTriggerReopen
8d87467bb Make PartialCompactionFailure Test more robust again.
64d302d30 make DropWritesFlush deterministic
cd5c0925a Merge pull request #387 from fyrz/RocksJava-WShadow-Fix
c4bf07c24 [RocksJava] -WShadow improvements
e526b7140 Make PartialCompactionFailure Test more robust.
0c2be0de3 Turn on -Wshadow for travis
5fd33d26f Turn off -Wshadow
9f20395cd Turn -Wshadow back on
c02338a69 update HISOTRY.md for new release
367a3f9cb Improve DBTest.GroupCommitTest: artificially slowdown log writing to trigger group commit
b52b144c7 Merge pull request #386 from EugenePig/java8
ac95ae1b5 Make sure WAL is synced for DB::Write() if write batch is empty
59d549798 suppress JDK8 errors for #385
ea18b944a Add db_bench option --report_file_operations
2ea1219eb Fix RecordIn and RecordDrop stats
e4211d10c Apply InfoLogLevel to the logs in util/env_hdfs.cc
76f6c7c7c CompactionFilterV2: eliminate an often unnecessary allocation.
29a9161f3 Note dynamic options in options.h
fd24ae9d0 SetOptions() to return status and also add it to StackableDB
b1267750f fix the asan check
83bf09144 Bump verison number to 3.7
da5daa061 Replace some ASSERT_TRUE() asserts in DBTest.DynamicMemtableOptions and DynamicCompactionOptions with more specific ones
b0cda4a11 DBTest.DynamicMemtableOptions to use single background compaction
8810850dd Apply InfoLogLevel to the logs in db/compaction_job.cc
71783f652 Merge pull request #377 from fyrz/RocksJava-KeyMayExist
614bbcbe2 Merge pull request #374 from fyrz/RocksJava-DBOptions-Extension-3.6
d8e119663 Apply InfoLogLevel to the logs in db/version_set.cc
2a019f1d0 Apply InfoLogLevel to the logs in db/wal_manager.cc
469d474ba Apply InfoLogLevel to the logs in db/db_impl.cc
ac6afaf9e Enforce naming convention of getters in version_set.h
09899f0b5 DB::Open() to automatically increase thread pool size if it is smaller than max number of parallel compactions or flushes
636e57b52 Fix coverage script
30ca3752b Revamp our build tools
051c67ff5 Merge pull request #378 from baotiao/master
d0e7e49ae Merge pull request #379 from fyrz/RocksJavaBuildFix
94e31ac22 [RocksJava] Extend Options with DBOptions implementation [RocksJava] Included DBOptionsTest and refactored OptionsTest
b060d3006 [RocksJava] Build fix after options refactoring
9fd65e566 add make clean in examples makefile
8e79ce68c Revert "Fix lint errors and coding style of ldb related codes."
45a612f99 Revert "Fix incorrect fixing of lint errors in ldb_cmd.cc"
27129c739 [RocksJava] KeyMayExist w/o ColumnFamilies
86905e3cb Move VersionBuilder logic to a separate .cc file
74eb4fbe9 CompactionJob
8ddddd62d Fix incorrect fixing of lint errors in ldb_cmd.cc
46c14c666 Fix #258. benchmarkharness -- make bm_min_usec uint
72cb7cf20 Add fsync / corrupt simulation to env_mem
0e526eb9d introduce TestMemEnv and use it in db_test
8db24f4b3 exclude mock test file from MOCK_SOURCES
5594d446f unfriend DBImpl and InternalStats from VersionStorageInfo
82e3ae540 fix c_test
bc9f36fd5 Fix lint errors and coding style of ldb related codes.
c645250ee CompactionStats to support larger value of RecordIn and RecordDrop
f7e6c856a Fix BaseReferencedVersionBuilder's destructor order
c76dcb44d fix
b452dede5 fix
29d83cc33 temporarily remove -Wshadow
c1a924b9f Move convenience.h to /include
7e01d1202 Add support for in place update for db_stress
9f7fc3ac4 Turn on -Wshadow
98849a35f Apply InfoLogLevel to the logs in table/block_based_table_reader.cc
4d2ba38b6 Make VersionBuilder unit testable
2b1f23dca Apply InfoLogLevel to the logs in db/db_iter.cc
ccaf1aa7c Merge pull request #372 from fyrz/RocksJava-CF-Merge-Hardening
a29118ffc Merge pull request #355 from fyrz/RocksJava-Options-Refactoring-3.6
85b04ca76 [RocksJava] Review comments - reformatted MergeTest
df7abb4e8 [RocksJava] Integrated code review comments
171be0ed5 Merge with ColumnFamilies & Hardening CFHandle
39464a990 [RocksJava] Options Refactoring 3.6
0f7f3b860 Check InfoLogLevel earlier in Log functions.
73605d917 Apply InfoLogLevel to the logs in util/db_info_dumper.cc
fda592d90 Merge pull request #356 from fyrz/RocksJava-TableOptions-3.6
c73d13bb8 [RocksJava] Integrate review comments from yhchiang
b011e201f Integrated review comments by ankgup87
2c1bd8846 BlockBasedTableConfig & PlainTableConfig enhancements
e770d6175 Merge pull request #371 from dlezama/master
41af0f56b Fix build break because of unsigned/signed mismatch
c5db7f260 Fix CompactionPickerTest.Level1Trigger2
37e9b6370 Apply InfoLogLevel to the logs in utilities/ttl/db_ttl_impl.h
217cc217d Apply InfoLogLevel to the logs in table/meta_blocks.cc
635905481 WalManager
fd95745a5 Fix compile error in table/plain_table_index.cc
c3dd0f75d comparator_db_test to cover more irregular comparators
6afafa369 Apply InfoLogLevel to the logs in utilities/merge_operators/uint64add.cc
e7ad69b9f Apply InfoLogLevel to the logs in table/plain_table_index.cc
bbd9c5345 Apply InfoLogLevel to the logs in table/block_based_table_builder.cc
065766b8d DynamicCompactionOptions: relax the check bound a little
5c82a8837 Add a test in compaction_picker_test to test the max score
86de2007b Add ComparatorDBTest to test non-default comparators
17be187ff dummy var to suppress compiler warning/error
c2999f54b Revert "tmp"
76d1c28e8 Make CompactionPicker more easily tested
01e6f8509 Apply InfoLogLevel to the logs in db/transaction_log_impl.h
082e49ba8 Apply InfoLogLevel to the logs in db/repair.cc
c4b468000 Apply InfoLogLevel to the logs in db/flush_job.cc
34d436b7d Apply InfoLogLevel to the logs in db/column_family.cc
cda9943f9 Apply InfoLogLevel to the logs in db/compaction_picker.cc
7b3a618f9 Apply InfoLogLevel to the logs in db/db_filesnapshot.cc
2d4fe048f remove dead code
9ab013236 tmp
76d54530d minor - remove default value for ChangeFilterOptions() and ChangeCompactionOptions()
44f0ff31c use fallocate(FALLOC_FL_PUNCH_HOLE) to release unused blocks at the end of file
97451f837 add an env var ROCKSDB_TESTS_FROM to control where to start from a list of tests
e130e88bc DBTest: options clean up - part 4
34f3c5a20 DBTest: options clean up - part 3
cdc7230e4 DBTest: options clean up - part 2
5a921b895 DBTest: options clean up - part 1
c9c935923 Move the check to the beginning of the loop in VersionEdit::EncodeTo()
2110e43a5 Remove an unnecessary include file in version_edit.cc
412b7f85b Include atomic in mock_table.h
c08285334 Include all the mocks
abac3d647 TableMock + framework for mock classes
fb3f8ffe5 Improve the robustness of PartialCompactionFailure test again.
60fa7d136 Improve the robustnesss of PartialCompactionFailure test.
3772a3d09 Fix the bug where compaction does not fail when RocksDB can't create a new file.
c49dedbe0 Merge pull request #366 from fyrz/RocksJava-Backup-Restore-Improvements
a39e931e5 FlushProcess
efa2fb33b make LevelFileNumIterator and LevelFileIteratorState anonymous
f7c973069 [RocksJava] Integrated review comments
7e12ae5a2 [RocksJava] - BackupInfos & Restore-/BackupableDB enhancements
eb357af58 unfriend ForwardIterator from VersionSet
f981e0813 unfriend ColumnFamilyData from VersionSet
834c67d77 rename FileLevel to LevelFilesBrief / unfriend CompactedDBImpl
a28b3c438 unfriend UniversalCompactionPicker,LevelCompactionPicker and FIFOCompactionPicker from VersionSet
5187d896b unfriend Compaction and CompactionPicker from VersionSet
75d7e2c37 Merge pull request #369 from fyrz/Small-Fix
db52419cf Merge pull request #290 from vladb38/master
45e756f04 [RocksJava] Minor correction to the previous pull request merge
f94f1a97d Merge pull request #368 from fyrz/RocksJava-Hardening-RocksIterator
b08c39e14 [RocksJava] RocksIterator: Assert for valid RocksDB instance & documentation
b680033e6 Include atomic in env_test
56ef2caaa [RocksJava] - Hardening RocksIterator
c1c68bce4 remove atomic_pointer.h references
7c303f0e7 Include atomic
179c23021 Merge pull request #351 from fyrz/RocksJava_Snapshot_Support
48842ab31 Deprecate AtomicPointer
f37048ad1 Merge pull request #367 from fyrz/FixBrokenJavaBuild
679a9671f RocksJava Fix after MutableCFOptions change.
714c63c58 db_stress for dynamic options
f1841985e dynamic inplace_update options
a04929aa4 fixed conflict in java/Makefile
a1bae76c8 Integrated changes due to review bei ankgup87
b8ce52648 [RocksJava] Support Snapshots
bc3bc4bc2 Merge pull request #357 from fyrz/JavaTest-Fix
42f0eaceb Merge pull request #354 from fyrz/RocksJava-memtables-3.6
965d9d50b Fix timing
001ce64dc Use chrono for timing
240ed0cd7 Fix uninitialized parameter caused by D24513
724fba2b3 Improve the log in Universal Compaction to include more debug information.
720c1c056 fix erro during merge
b794194ad Remove java build from travis
122f98e0b dynamic max_mem_compact_level
1fee591e7 comments for DynamicCompactionOptions test
574028679 dynamic max_sequential_skip_in_iterations
bd4fbaee3 Fixed cross platform build after introducing Java-7 dependencies
1eb545721 Fix incorrectly merged Java - Makefile
9aa9668a8 [RocksJava] Memtables update to 3.6
4b1786e95 Fix SIGSEGV when declaring Arena after ScopedArenaIterator
05b2e60dd Merge pull request #362 from adamretter/travis-jcheck
9383922cc Added java tests to travis build
90f156402 Fix CompactBetweenSnapshots
2717422e2 Merge pull request #361 from adamretter/fix-yosemite-build
3b5fe3a1f Correct the log message in VersionEdit
c584d2b53 Fix for building RocksDB Java on Mac OS X Yosemite
2a8e5203d db_bench: --batch_size used for write benchmarks too
d755e53b8 Printing number of keys in DB Stats
839c376bd fix table_test
0fd985f42 Avoid reloading filter on Get() if cache_index_and_filter_blocks == false
e11a5e776 Improve the comment of util/thread_local.h
6398e6a6a Fix DeleteFile() + enable deleting files oldest files in level 0
0d3145198 Merge pull request #231 from adamretter/master
a6fb7f312 Fix code review comments raised in https://reviews.facebook.net/D22779
c63494fb6 Tests for ComparatorOptions, Comparator and DirectComparator, and by proxy we also exercise Slice and DirectSlice
5e2527411 Fix code style problems identified by lint
25641bfc9 Fix to memory dealocation when creating a slice from a byte buffer
fc12cb83f Add locking to comparator jni callback methods which must be thread-safe
d6fe8dacc Feature - Implement Java API for Comparator and Slice. Allows use of either byte[] or DirectByteBuffer for accessing underlying data.
700f6ec3f Ignore IntelliJ idea project files and ignore java/out folder
5bfb7f5d0 db_bench: seekrandom can specify --seek_nexts to read specific keys after seek.
ff8f74c20 remove checking lower bound of level size
2dd9bfe3a Sanitize block-based table index type and check prefix_extractor
dbcfe27d6 Merge branch 'master' of https://github.com/facebook/rocksdb
6c6691864 Speed up DB::Open() and Version creation by limiting the number of FileMetaData initialization.
5db9e7664 Fix Mac compile error: C++11 forbids default arguments for lambda expressions
f4363fb81 Fix DynamicMemtableOptions test
8f01bf80c Merge pull request #353 from fyrz/FixBloomfilterRocksJava
ee80fb4b4 Total memtables size counter
c12f571d3 Fix mac compile, second try
d2e60f5ce Fix mac compile
274dc81c9 fix build failure
d6c8dba72 Log MutableCFOptions in SetOptions
4d5708aa5 dynamic soft_rate_limit and hard_rate_limit
065a67c4f dynamic disable_auto_compactions
dc50a1a59 make max_write_buffer_number dynamic
6a150c011 ldb: support --fix_prefix_len
bafbc23ba Filters getting disposed by System.gc before EOL
ca250d71a Move logging out of mutex
5cc9adf5b WriteBatchWithIndex's Iterator bug of SeekToFirst() and SeekToLast()
7658bcc1e Merge pull request #352 from fyrz/OptionsTestMergeProblem
1b97934a2 Options correction
2ef3ed86f Integrated feedback from ankgup87
a40ce219b Adding merge functions to RocksDBJava
cc6c883f5 Stop stopping writes on bg_error_
4942f6efb Merge pull request #350 from fyrz/reenable_accidentally_disabled_test
ee28f431d With the last commit a Test was accidentally disabled. This commit solves this.
b5dd7eed6 Merge pull request #319 from fyrz/column_families_java
18004d2f2 [RocksJava] Column family support
5908d08a0 Merge pull request #336 from fyrz/32BitRocksJavaBug
9e5f2a952 Merge pull request #344 from fyrz/java-doc-enhancements
4f5a68725 32-Bit RocksJava resolution for jlong overflows
16d2ebdbc Minor adjustment to prevent two warnings
70294c911 JavaDoc improvements on RocksJava
833357402 WriteBatchWithIndex supports an iterator that merge its change with a base iterator.
4f65fbd19 WriteBatchWithIndex's iterator to support SeekToFirst(), SeekToLast() and Prev()
f441b273a WriteBatchWithIndex to support an option to overwrite rows when operating the same key
3ead857a0 Fixed Mac compile error in util/options_test.cc
5a7618634 Fixed compile error on Mac: default arguments for lambda expressions
cd0d581ff convert Options from string
f18b4a484 minor update to benchmark script
b7d3d6ebc db_bench: set thread pool size according to max_background_flushes
c5f54a8f3 Merge pull request #339 from fyrz/bloomFilterSupportNewFormat
ced612957 Improved JavaDoc
5e43155b3 RocksJava should support not only BlockBased Bloomfilter
1d525891b Update HISTORY for 3.6
88edfd90a SkipListRep::LookaheadIterator
6a443309d Merge pull request #342 from fyrz/java_makefile_fix
4f272408c RocksJava Makefile includes incorrect paths to version.h
f78b832e5 Log RocksDB version
25f6a852e add db_test for changing memtable size
daab6dc51 Merge pull request #318 from criccomini/master
63eade401 Fix error introduced by merge
ba882972f Merge pull request #340 from nbougalis/nullderef
6bcff9dc2 Merge pull request #341 from fyrz/arc-lint
d6169954b Removed code which prevents `arc lint` from working properly.
b87db0715 Avoid dereferencing a null field
1a1b95347 Merge pull request #270 from tdfischer/check-with-unity
e107b6b2a Merge pull request #337 from fyrz/cross-platform-jdb_bench
70e401a2e Merge pull request #338 from fyrz/rm_obs_code_write_batch_with_index_test
1e5a52815 update release readme
d44871e80 fix java doc directory in git ignore
2a4d6e796 merge master to resolve merge conflicts
6b2c1d962 make publish jni jars depend on release jni jars
1c7c76476 Replaced obsolete comparator with builtin variant.
22c64be43 Cross platform fix for Java benchmark shell script.
81828592b Merge pull request #335 from fyrz/version-script-cross-platform
69d4c5123 Cross-platform fix version.sh
fcd13a77a Merge pull request #334 from fyrz/JavaDoc-Cleanup
bfb0246f5 Merge pull request #331 from fyrz/findbug_issues
05204bb11 Lint changes
da8ff9ff8 Fixed Findbugs issues
a5757ff3c Listing of changes
df3373fbf [Java] Fix compile error on DbBenchmark.java
4eb5a40f7 [Java] Fixed link error on library loading on Mac.
56dfd363f Fix a check in database shutdown or Column family drop during flush.
0e516a75d Fix lint errors in java/rocksjni/options.cc
8ea232b9e Add number of records dropped in compaction summary
f4086a88b perf_context.get_from_output_files_time is set for MultiGet() and ReadOnly DB too.
e869fc6a8 remove proper javadoc directory
c1273533b Merge pull request #333 from nbougalis/cleanups
99744e0c4 bump version to 3.6
45d526e22 singular javadoc directory
378f321da merge master to resolve merge conflicts
a1d3f0d2b don't fail if javadocs diretory doesn't exist
c832f1644 add not about updating pom version and rename pom to be unversioned
a213971d8 Don't return (or dereference) dangling pointer
2a1add673 use proper major/minor/micro version rather than hard coding 3.5.0
2d72f7807 update release docs in java
8322cf000 use javadoc instead of javadocs
079a612b5 Fix build_tools/version.sh
2e8012498 add javadoc and sources targets for sonatype
df08a2d03 add single rocksdbjni pom
deefcf476 make fat jar unclassified to satisfy sonatype
fd2545c80 add maven publication target and instructions
1e9af10ef Merge pull request #330 from fyrz/setFilterPolicyForBBTConfig
017354177 FilterTest
d410b39d5 BlockBasedTableConfig Filter policy support RocksJava
0908ddcea Don't keep managing two rocksdb version
d0916f452 add major minor micro version to java jars
d6987216c Merge pull request #327 from dalgaaf/wip-da-SCA-20141001
25888ae00 Merge pull request #329 from fyrz/master
89833e5a8 Fixed signed-unsigned comparison warning in db_test.cc
fcac705f9 Fixed compile warning on Mac caused by unused variables.
b3343fdea resolution for java build problem introduced by 5ec53f3edf62bec1b690ce12fb21a6c52203f3c8
187b29938 ForwardIterator: update prev_key_ only if prefix hasn't changed
5ec53f3ed make compaction related options changeable
d122e7bcf Update INSTALL.md
9d6f38086 backupable_db_test.cc: pass const string param by reference
8ff0b4095 document_db_test.cc: pass const string param by reference
177caca42 ttl/ttl_test.cc: pass const string param by reference
4a171882d db/version_set.cc: remove unnecessary checks
bf3bfd044 util/cache_test.cc: use static_cast over C-Style cast
86e29f033 document_db.cc: remove unused variable
28a6e3158 table/block_based_table_builder.cc: remove unused variable
091153493 db/db_test.cc: remove unused variable
6b6cedbb1 table/format.cc: reduce scope of some variables
55652043c table/cuckoo_table_reader.cc: pass func parameter by reference
5abd8add7 db/deletefile_test.cc: remove unused variable
d6483af87 db/db_test.cc: reduce scope of some variables
44cca0cd8 db/db_iter.cc: remove unused variable
986dad025 Merge pull request #324 from dalgaaf/wip-da-SCA-20140930
3a0d498a3 rebase master
8ee75dca2 db/memtable.cc: remove unused variable merge_result
0fd8bbca5 db/db_impl.cc: reduce scope of prefix_initialized
676ff7b1f compaction_picker.cc: remove check for >=0 for unsigned
e55aea551 document_db.cc: fix assert
d517c8364 in_table_factory.cc: use correct format specifier
b14037556 ttl/ttl_test.cc: prefer prefix ++operator for non-primitive types
43c789c8f spatialdb/spatial_db.cc: use !empty() instead of 'size() > 0'
0de452ee9 document_db.cc: pass const parameter by reference
4cc8643ba util/ldb_cmd.cc: prefer prefix ++operator for non-primitive types
af8c2b2d9 util/signal_test.cc: suppress intentional null pointer deref
33580fa39 db/db_impl.cc: fix object handling, remove double lines
873f1356a db_ttl_impl.h: pass func parameter by reference
855845714 ldb_cmd_execute_result.h: perform init in initialization list
063471bf7 table/table_test.cc: pass func parameter by reference
93548ce8f table/cuckoo_table_reader.cc: pass func parameter by ref
b8b7117e9 db/version_set.cc: use !empty() instead of 'size() > 0'
8ce050b51 table/bloom_block.*: pass func parameter by reference
53910ddb1 db_test.cc: pass parameter by reference
68ca53416 corruption_test.cc: pass parameter by reference
726ac5bca shrink vagrant commands to single line
a2f98ef61 fix tabs in Makefile
7506198da cuckoo_table_db_test.cc: add flush after delete
1f963305a Print MB per second compaction throughput separately for reads and writes
ffe3d490d Add an instruction about SSE in INSTALL.md
0b923f0f9 add centos 5.6 build instead of ubuntu.
ee1f3ccb0 Package generation for Ubuntu and CentOS
f0f795549 Fixing comile errors on OS X
99fb613e5 remove 2 space linter
b2d64a486 Fix linters, second try
747523d24 Print per column family metrics in db_bench
56ebd4087 Fix arc lint (should fix #238)
637f89179 Merge pull request #321 from eonnen/master
827e31c74 Make test use a compatible type in the size checks.
fd5d80d55 CompactedDB: log using the correct info_log
2faf49d5f use GetContext to replace callback function pointer
6a64ea617 add note about java 7
983d2de2d Add AUTHORS file. Fix #203
c4519c777 fix mis-named jar in JNI loader
abd70c5e1 Merge pull request #316 from fyrz/ReverseBytewiseComparator
2dc6f62bb handle kDelete type in cuckoo builder
8b8011a68 Changed name of ReverseBytewiseComparator based on review comment
b8e26615a since we're not sharing folders with the vm, copy built .so files and jars back to host system.
389edb6b1 universal compaction picker: use double for potential overflow
4e735bb7f Rsync files to VM rather than sync folders, since sync folders was causing clock skew and confusig make.
82a8f43cc Document RELEASE.mdgit status
9db13987b Update RocksDB's Java bindings to support multiple native RocksDB builds in the same Jar file. Cross build RocksDB for linux32 and linux64 using Vagrant. Build a cross-platform fat jar that contains osx, linux32, and linux64 RocksDB static builds.
534048426 Built-in comparator(s) in RocksJava
d439451fa delay initialization of cuckoo table iterator
94997eab5 reduce memory usage of cuckoo table builder
c6275956e improve memory efficiency of cuckoo reader
581442d44 option to choose module when calculating CuckooTable hash
fbd2dafc9 CompactedDBImpl::MultiGet() for better CuckooTable performance
3c6800610 CompactedDBImpl
f7375f39f Fix double deletes
21ddcf6e4 Remove allow_thread_local
fb4a492cc Merge pull request #311 from ankgup87/master
611e286b9 Merge branch 'master' of https://github.com/facebook/rocksdb
0103b4498 Merge branch 'master' of ssh://github.com/ankgup87/rocksdb
1dfb7bb98 Add block based table config options
cdaf44f9a Enlarge log size cap when printing file summary
7cc1ed7f0 Merge pull request #309 from naveenatceg/staticbuild
ba6d660f6 Resolving merge conflict
51eeaf65e Addressing review comments
fd7d3fe60 Addressing review comments (adding a env variable to override temp directory)
cf7ace886 Addressing review comments
0a29ce539 re-enable BlockBasedTable::SetupForCompaction()
55af37075 Remove TODO for checking index checksums
3d74f0997 Fix compile
53b003995 Fix release compile
d0de413f4 WriteBatchWithIndex to allow different Comparators for different column families
57a32f147 change target_file_size_base to uint64_t
5e6aee432 dont create backup_input if compaction filter v2 is not used
49b5f94c5 Merge pull request #306 from Liuchang0812/fix_cast
787cb4db2 remove cast, replace %llu with % PRIu64
a7574d4fa Update logging.cc
7e0dcb953 Update logging.cc
57fa3cc5b Merge pull request #304 from Liuchang0812/fix-check
cd44522a9 Merge pull request #305 from Liuchang0812/fix-logging
6a031b6a8 remove unused variable
4436f17bd fixed #303: replace %ld with % PRId64
7a1bd057f Merge pull request #302 from ankgup87/master
423e52cd4 Merge branch 'master' of https://github.com/facebook/rocksdb
bfeef94d3 Add rate limiter
ed9a2df8c fix unity build
32f2532a0 Print compression_size_percent as a signed int
976caca09 Skip AllocateTest if fallocate() is not supported in the file system
3b897cddd Enable no-fbcode RocksDB build
f44594743 RocksDB: Format uint64 using PRIu64 in db_impl.cc
e17bc65c7 Merge pull request #299 from ankgup87/master
b93797abc Fix build
adae3ca1f [Java] Fix JNI link error caused by the removal of options.db_stats_log_interval
90b8c07b4 Fix unit tests errors
51af7c326 CuckooTable: add one option to allow identity function for the first hash function
035043559 Fixed a signed-unsigned comparison in spatial_db.cc -- issue #293
2fb1fea30 Fix syncronization issues
6f0964e37 Only run make unity on travis instead of make check
9ed1b49a2 Build unity build on make check
ff7689561 Remove some unnecessary constructors
feadb9df5 fix cuckoo table builder test
3c232e164 Fix mac compile
54cada92b Run make format on PR #249
27b22f13a Merge pull request #249 from tdfischer/decompression-refactoring
fb6456b00 Replace naked calls to operator new and delete (Fixes #222)
5600c8f6e cuckoo table: return estimated size - 1
a062e1f2c SetOptions() for memtable related options
e4eca6a1e Options conversion function for convenience
a7c209452 Merge pull request #292 from saghmrossi/master
4d0523452 Merge branch 'master' of github.com:saghmrossi/rocksdb
60a4aa175 Test use_mmap_reads
94e43a1df [Java] Fixed 32-bit overflowing issue when converting jlong to size_t
f9eaaa66e added include for inttypes.h to fix nonworking printf statements
f090575e4 Replaced "built on on earlier work" by "built on earlier work" in README.md
faad439ac Fix #284
49aacd8d2 Fix make install
acb9348ff [Java] Include WriteBatch into RocksDBSample.java, fix how DbBenchmark.java handles WriteBatch.
4a27a2f19 Don't sync manifest when disableDataSync = true
9b8480d93 Merge pull request #287 from yinqiwen/rate-limiter-crash-fix
28be16b1d fix rate limiter crash #286
04ce1b25f Fix #284
add22e351 standardize scripts to run RocksDB benchmarks
dee91c259 WriteThread
540a257f2 Fix WAL synced
24f034bf4 Merge pull request #282 from Chilledheart/develop
49fe329e5 Fix build issue under macosx
ebb5c65e6 Add make install
0352a9fa9 add_wrapped_bloom_test
9c0e66ce9 Don't run background jobs (flush, compactions) when bg_error_ is set
a9639bda8 Fix valgrind test
d1f24dc7e Relax FlushSchedule test
3d9e6f775 Push model for flushing memtables
059e584dd [unit test] CompactRange should fail if we don't have space
dd641b211 fix RocksDB java build
53404d9fb add_qps_info_in cache bench
a52cecb56 Fix Mac compile
092f97e21 Fix comments and typos
6cc12860f Added a few statistics for BackupableDB
0a42295a2 Fix SimpleWriteTimeoutTest
06d986252 Always pass MergeContext as pointer, not reference
d343c3fe4 Improve db recovery
6bb7e3ef2 Merger test
88841bd00 Explicitly cast char to signed char in Hash()
52311463e MemTableOptions
1d284db21 Addressing review comments
55114e7f4 Some updates for SpatialDB
171d4ff4a remove TailingIterator reference in db_impl.h
9b0f7ffa1 rename version_set options_ to db_options_ to avoid confusion
2d57828d0 Check stop level trigger-0 before slowdown level-0 trigger
659d2d50c move compaction_filter to immutable_options
048560a64 reduce references to cfd->options() in DBImpl
011241bb9 DB::Flush() Do not wait for background threads when there is nothing in mem table
a2bb7c3c3 Push- instead of pull-model for managing Write stalls
0af157f9b Implement full filter for block based table.
9360cc690 Fix valgrind issue
02d5bff39 Merge pull request #277 from wankai/master
88a2f44f9 fix comments
7c16e3922 Merge pull request #276 from wankai/master
823773837 replace hard-coded number with named variable
db8ca520b Merge pull request #273 from nbougalis/static-analysis
b7b031f42 Merge pull request #274 from wankai/master
4c2b1f097 Merge remote-tracking branch 'upstream/master'
a5d286307 typo improvement
9f8aa0939 Don't leak data returned by opendir
d1cfb71ec Remove unused member(s)
bfee319fb sizeof(int*) where sizeof(int) was intended
d40c1f742 Add missing break statement
2e97c3898 Avoid off-by-one error when using readlink
40ddc3d6c add cache bench
9f1c80b55 Drop column family from write thread
8de151bb9 Add db_bench with lots of column families to regression tests
c9e419ccb rename options_ to db_options_ in DBImpl to avoid confusion
5cd0576ff Fix compaction bug in Cuckoo Table Builder. Use kvs_.size() instead of num_entries in FileSize() method.
0fbb3facc fixed memory leak in unit test DBIteratorBoundTest
adcd2532c fix asan check
4092b7a0b Merge pull request #272 from project-zerus/patch-1
bb6ae0f80 fix more compile warnings
6d3144118 Merge pull request #271 from nbougalis/cleanups
0cd0ec4fe Plug memory leak during index creation
4329d74e0 Fix swapped variable names to accurately reflect usage
45a5e3ede Remove path with arena==nullptr from NewInternalIterator
5665e5e28 introduce ImmutableOptions
e0b99d4f5 created a new ReadOptions parameter 'iterate_upper_bound'
51ea88900 Fix travis builds
a4816269f Relax backupable rate limiting test
f7f973d35 Merge pull request #269 from huahang/patch-2
ef5b38472 fix a few compile warnings
2fd3806c8 Merge pull request #263 from wankai/master
1785114a6 delete unused Comparator
1b1d9619f update HISTORY.md
703c3eacd comments about the BlockBasedTableOptions migration in Options
4b5ad8865 Merge pull request #260 from wankai/master
19cc588b7 change to filter_block std::unique_ptr support RAII
9b976e34f Merge pull request #259 from wankai/master
5d25a4693 Merge remote-tracking branch 'upstream/master'
dff2b1a8f typo improvement
343e98a7d Reverting import change
ddb8039e3 RocksDB static build Make file changes to download and build the dependencies .Load the shared library when RocksDB is initialized
REVERT: 1fdd726a8 Hotfix RocksDB 3.5
REVERT: d67500a59 Add `make install` to Makefile in 3.5.fb.
REVERT: 4cb631aa0 update HISTORY.md
REVERT: cfd0946be comments about the BlockBasedTableOptions migration in Options

git-subtree-dir: src/rocksdb2
git-subtree-split: aead4041720f40e305e7f9d432875749d91d5a2d
This commit is contained in:
Brad Chase
2017-12-02 13:44:49 -05:00
parent c168d54495
commit d0dac38ea8
1303 changed files with 307707 additions and 62115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.arcconfig
View File

@@ -1,10 +0,0 @@
{
"project_id" : "rocksdb",
"conduit_uri" : "https://reviews.facebook.net/",
"copyright_holder" : "Facebook",
"load" : [
"linters"
],
"lint.engine" : "FacebookFbcodeLintEngine",
"lint.engine.single.linter" : "FbcodeCppLinter"
}

45
.gitignore vendored
View File

@@ -1,5 +1,4 @@
TARGETS
build_config.mk
make_config.mk
*.a
*.arc
@@ -20,16 +19,56 @@ build_config.mk
*.d-e
*.o-*
*.swp
*~
*.vcxproj
*.vcxproj.filters
*.sln
*.cmake
CMakeCache.txt
CMakeFiles/
build/
ldb
manifest_dump
sst_dump
blob_dump
column_aware_encoding_exp
util/build_version.cc
build_tools/VALGRIND_LOGS/
coverage/COVERAGE_REPORT
.gdbhistory
.phutil_module_cache
.gdb_history
package/
unity.a
tags
etags
rocksdb_dump
rocksdb_undump
db_test2
java/out
java/target
java/test-libs
java/*.log
java/include/org_rocksdb_*.h
.idea/
*.iml
rocksdb.cc
rocksdb.h
unity.cc
java/crossbuild/.vagrant
.vagrant/
java/**/*.asc
java/javadoc
scan_build_report/
t
LOG
db_logs/
tp2/
fbcode/
fbcode
buckifier/*.pyc

88
.travis.yml
View File

@@ -1,20 +1,72 @@
sudo: false
dist: trusty
language: cpp
compiler: gcc
before_install:
# As of this writing (10 May 2014) the Travis build environment is Ubuntu 12.04,
# which needs the following ugly dependency incantations to build RocksDB:
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
- sudo apt-get update -qq
- sudo apt-get install -y -qq gcc-4.8 g++-4.8 zlib1g-dev libbz2-dev libsnappy-dev libjemalloc-dev
- sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 50
- sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 50
- wget https://gflags.googlecode.com/files/libgflags0_2.0-1_amd64.deb
- sudo dpkg -i libgflags0_2.0-1_amd64.deb
- wget https://gflags.googlecode.com/files/libgflags-dev_2.0-1_amd64.deb
- sudo dpkg -i libgflags-dev_2.0-1_amd64.deb
# Lousy hack to disable use and testing of fallocate, which doesn't behave quite
# as EnvPosixTest::AllocateTest expects within the Travis OpenVZ environment.
- sed -i "s/fallocate(/HACK_NO_fallocate(/" build_tools/build_detect_platform
script: make check -j8
os:
- linux
- osx
compiler:
- clang
- gcc
osx_image: xcode8.3
jdk:
- oraclejdk7
cache:
- ccache
- apt
addons:
apt:
packages: ['zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev', 'mingw-w64']
env:
- TEST_GROUP=platform_dependent # 16-18 minutes
- TEST_GROUP=1 # 33-35 minutes
- TEST_GROUP=2 # 30-32 minutes
# Run java tests
- JOB_NAME=java_test # 4-11 minutes
# Build ROCKSDB_LITE
- JOB_NAME=lite_build # 3-4 minutes
# Build examples
- JOB_NAME=examples # 5-7 minutes
- JOB_NAME=cmake # 3-5 minutes
- JOB_NAME=cmake-mingw # 3 minutes
matrix:
exclude:
- os: osx
env: TEST_GROUP=1
- os: osx
env: TEST_GROUP=2
- os : osx
env: JOB_NAME=cmake-mingw
- os : linux
compiler: clang
- os : osx
compiler: gcc
# https://docs.travis-ci.com/user/caching/#ccache-cache
install:
- if [ "${TRAVIS_OS_NAME}" == osx ]; then
brew install ccache;
PATH=$PATH:/usr/local/opt/ccache/libexec;
fi
before_script:
# Increase the maximum number of open file descriptors, since some tests use
# more FDs than the default limit.
- ulimit -n 8192
script:
- ${CXX} --version
- if [ "${TEST_GROUP}" == 'platform_dependent' ]; then ccache -C && OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some; fi
- if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
- if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
- if [ "${JOB_NAME}" == 'java_test' ]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
- if [ "${JOB_NAME}" == 'lite_build' ]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib tools; fi
- if [ "${JOB_NAME}" == 'examples' ]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
- if [ "${JOB_NAME}" == 'cmake' ]; then mkdir build && cd build && cmake .. && make -j4 rocksdb; fi
- if [ "${JOB_NAME}" == 'cmake-mingw' ]; then mkdir build && cd build && cmake .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb; fi
notifications:
email: false
email:
- leveldb@fb.com
webhooks:
- https://buildtimetrend.herokuapp.com/travis

11
AUTHORS Normal file
View File

@@ -0,0 +1,11 @@
Facebook Inc.
Facebook Engineering Team
Google Inc.
# Initial version authors:
Jeffrey Dean <jeff@google.com>
Sanjay Ghemawat <sanjay@google.com>
# Partial list of contributors:
Kevin Regan <kevin.d.regan@gmail.com>
Johan Bilien <jobi@litl.com>

927
CMakeLists.txt Normal file
View File

@@ -0,0 +1,927 @@
# Prerequisites for Windows:
# This cmake build is for Windows 64-bit only.
#
# Prerequisites:
# You must have at least Visual Studio 2015 Update 3. Start the Developer Command Prompt window that is a part of Visual Studio installation.
# Run the build commands from within the Developer Command Prompt window to have paths to the compiler and runtime libraries set.
# You must have git.exe in your %PATH% environment variable.
#
# To build Rocksdb for Windows is as easy as 1-2-3-4-5:
#
# 1. Update paths to third-party libraries in thirdparty.inc file
# 2. Create a new directory for build artifacts
# mkdir build
# cd build
# 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries.
# See thirdparty.inc for more information.
# sample command: cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
# 4. Then build the project in debug mode (you may want to add /m[:<N>] flag to run msbuild in <N> parallel threads
# or simply /m ot use all avail cores)
# msbuild rocksdb.sln
#
# rocksdb.sln build features exclusions of test only code in Release. If you build ALL_BUILD then everything
# will be attempted but test only code does not build in Release mode.
#
# 5. And release mode (/m[:<N>] is also supported)
# msbuild rocksdb.sln /p:Configuration=Release
#
# Linux:
#
# 1. Install a recent toolchain such as devtoolset-3 if you're on a older distro. C++11 required.
# 2. mkdir build; cd build
# 3. cmake ..
# 4. make -j
cmake_minimum_required(VERSION 2.6)
project(rocksdb)
if(POLICY CMP0042)
cmake_policy(SET CMP0042 NEW)
endif()
list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/")
option(WITH_JEMALLOC "build with JeMalloc" OFF)
if(MSVC)
include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
else()
if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
# FreeBSD has jemaloc as default malloc
# but it does not have all the jemalloc files in include/...
set(WITH_JEMALLOC ON)
else()
if(WITH_JEMALLOC)
find_package(JeMalloc REQUIRED)
add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE)
include_directories(${JEMALLOC_INCLUDE_DIR})
endif()
endif()
option(WITH_SNAPPY "build with SNAPPY" OFF)
if(WITH_SNAPPY)
find_package(snappy REQUIRED)
add_definitions(-DSNAPPY)
include_directories(${SNAPPY_INCLUDE_DIR})
list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES})
endif()
option(WITH_ZLIB "build with zlib" OFF)
if(WITH_ZLIB)
find_package(zlib REQUIRED)
add_definitions(-DZLIB)
include_directories(${ZLIB_INCLUDE_DIR})
list(APPEND THIRDPARTY_LIBS ${ZLIB_LIBRARIES})
endif()
option(WITH_BZ2 "build with bzip2" OFF)
if(WITH_BZ2)
find_package(bzip2 REQUIRED)
add_definitions(-DBZIP2)
include_directories(${BZIP2_INCLUDE_DIR})
list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES})
endif()
option(WITH_LZ4 "build with lz4" OFF)
if(WITH_LZ4)
find_package(lz4 REQUIRED)
add_definitions(-DLZ4)
include_directories(${LZ4_INCLUDE_DIR})
list(APPEND THIRDPARTY_LIBS ${LZ4_LIBRARIES})
endif()
option(WITH_ZSTD "build with zstd" OFF)
if(WITH_ZSTD)
find_package(zstd REQUIRED)
add_definitions(-DZSTD)
include_directories(${ZSTD_INCLUDE_DIR})
list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARIES})
endif()
endif()
if(WIN32)
execute_process(COMMAND powershell -noprofile -Command "Get-Date -format MM_dd_yyyy" OUTPUT_VARIABLE DATE)
execute_process(COMMAND powershell -noprofile -Command "Get-Date -format HH:mm:ss" OUTPUT_VARIABLE TIME)
string(REGEX REPLACE "(..)_(..)_..(..).*" "\\1/\\2/\\3" DATE "${DATE}")
string(REGEX REPLACE "(..):(.....).*" " \\1:\\2" TIME "${TIME}")
set(GIT_DATE_TIME "${DATE} ${TIME}")
else()
execute_process(COMMAND date "+%Y/%m/%d %H:%M:%S" OUTPUT_VARIABLE DATETIME)
string(REGEX REPLACE "\n" "" DATETIME ${DATETIME})
set(GIT_DATE_TIME "${DATETIME}")
endif()
find_package(Git)
if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
if(WIN32)
execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
else()
execute_process(COMMAND ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
endif()
else()
set(GIT_SHA 0)
endif()
string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}")
if(NOT WIN32)
execute_process(COMMAND
"./build_tools/version.sh" "full"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
OUTPUT_VARIABLE ROCKSDB_VERSION
)
string(STRIP "${ROCKSDB_VERSION}" ROCKSDB_VERSION)
execute_process(COMMAND
"./build_tools/version.sh" "major"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
OUTPUT_VARIABLE ROCKSDB_VERSION_MAJOR
)
string(STRIP "${ROCKSDB_VERSION_MAJOR}" ROCKSDB_VERSION_MAJOR)
endif()
option(WITH_MD_LIBRARY "build with MD" ON)
if(WIN32 AND MSVC)
if(WITH_MD_LIBRARY)
set(RUNTIME_LIBRARY "MD")
else()
set(RUNTIME_LIBRARY "MT")
endif()
endif()
set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
add_library(build_version OBJECT ${BUILD_VERSION_CC})
target_include_directories(build_version PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/util)
if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /wd4127 /wd4800 /wd4996 /wd4351")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers")
if(MINGW)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
include(CheckCXXCompilerFlag)
CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
if(HAVE_OMIT_LEAF_FRAME_POINTER)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
endif()
endif()
endif()
option(PORTABLE "build a portable binary" OFF)
option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
if(PORTABLE)
# MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h
# is available, it is available by default.
if(FORCE_SSE42 AND NOT MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
endif()
else()
if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
endif()
endif()
set(CMAKE_REQUIRED_FLAGS ${CMAKE_CXX_FLAGS})
include(CheckCXXSourceCompiles)
CHECK_CXX_SOURCE_COMPILES("
#include <cstdint>
#include <nmmintrin.h>
int main() {
volatile uint32_t x = _mm_crc32_u32(0, 0);
}
" HAVE_SSE42)
if(HAVE_SSE42)
add_definitions(-DHAVE_SSE42)
elseif(FORCE_SSE42)
message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
endif()
CHECK_CXX_SOURCE_COMPILES("
#if defined(_MSC_VER) && !defined(__thread)
#define __thread __declspec(thread)
#endif
int main() {
static __thread int tls;
}
" HAVE_THREAD_LOCAL)
if(HAVE_THREAD_LOCAL)
add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
endif()
option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON)
if(FAIL_ON_WARNINGS)
if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX")
else() # assume GCC
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
endif()
endif()
option(WITH_ASAN "build with ASAN" OFF)
if(WITH_ASAN)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address")
if(WITH_JEMALLOC)
message(FATAL "ASAN does not work well with JeMalloc")
endif()
endif()
option(WITH_TSAN "build with TSAN" OFF)
if(WITH_TSAN)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread -pie")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fPIC")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread -fPIC")
if(WITH_JEMALLOC)
message(FATAL "TSAN does not work well with JeMalloc")
endif()
endif()
option(WITH_UBSAN "build with UBSAN" OFF)
if(WITH_UBSAN)
add_definitions(-DROCKSDB_UBSAN_RUN)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined")
if(WITH_JEMALLOC)
message(FATAL "UBSAN does not work well with JeMalloc")
endif()
endif()
# Used to run CI build and tests so we can run faster
set(OPTIMIZE_DEBUG_DEFAULT 0) # Debug build is unoptimized by default use -DOPTDBG=1 to optimize
if(DEFINED OPTDBG)
set(OPTIMIZE_DEBUG ${OPTDBG})
else()
set(OPTIMIZE_DEBUG ${OPTIMIZE_DEBUG_DEFAULT})
endif()
if(MSVC)
if((${OPTIMIZE_DEBUG} EQUAL 1))
message(STATUS "Debug optimization is enabled")
set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /${RUNTIME_LIBRARY}d")
else()
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /${RUNTIME_LIBRARY}d")
endif()
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
endif()
if(CMAKE_COMPILER_IS_GNUCXX)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-memcmp")
endif()
option(ROCKSDB_LITE "Build RocksDBLite version" OFF)
if(ROCKSDB_LITE)
add_definitions(-DROCKSDB_LITE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
endif()
if(CMAKE_SYSTEM_NAME MATCHES "Cygwin")
add_definitions(-fno-builtin-memcmp -DCYGWIN)
elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
add_definitions(-DOS_MACOSX)
if(CMAKE_SYSTEM_PROCESSOR MATCHES arm)
add_definitions(-DIOS_CROSS_COMPILE -DROCKSDB_LITE)
# no debug info for IOS, that will make our library big
add_definitions(-DNDEBUG)
endif()
elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
add_definitions(-DOS_LINUX)
elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")
add_definitions(-DOS_SOLARIS)
elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
add_definitions(-DOS_FREEBSD)
elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
add_definitions(-DOS_NETBSD)
elseif(CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
add_definitions(-DOS_OPENBSD)
elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly")
add_definitions(-DOS_DRAGONFLYBSD)
elseif(CMAKE_SYSTEM_NAME MATCHES "Android")
add_definitions(-DOS_ANDROID)
elseif(CMAKE_SYSTEM_NAME MATCHES "Windows")
add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX)
if(MINGW)
add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_VISTA)
endif()
endif()
if(NOT WIN32)
add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX)
endif()
option(WITH_FALLOCATE "build with fallocate" ON)
if(WITH_FALLOCATE)
set(CMAKE_REQUIRED_FLAGS ${CMAKE_C_FLAGS})
include(CheckCSourceCompiles)
CHECK_C_SOURCE_COMPILES("
#include <fcntl.h>
#include <linux/falloc.h>
int main() {
int fd = open(\"/dev/null\", 0);
fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
}
" HAVE_FALLOCATE)
if(HAVE_FALLOCATE)
add_definitions(-DROCKSDB_FALLOCATE_PRESENT)
endif()
endif()
include(CheckFunctionExists)
CHECK_FUNCTION_EXISTS(malloc_usable_size HAVE_MALLOC_USABLE_SIZE)
if(HAVE_MALLOC_USABLE_SIZE)
add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE)
endif()
include_directories(${PROJECT_SOURCE_DIR})
include_directories(${PROJECT_SOURCE_DIR}/include)
include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
find_package(Threads REQUIRED)
add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest)
# Main library source code
set(SOURCES
cache/clock_cache.cc
cache/lru_cache.cc
cache/sharded_cache.cc
db/builder.cc
db/c.cc
db/column_family.cc
db/compacted_db_impl.cc
db/compaction.cc
db/compaction_iterator.cc
db/compaction_job.cc
db/compaction_picker.cc
db/compaction_picker_universal.cc
db/convenience.cc
db/db_filesnapshot.cc
db/db_impl.cc
db/db_impl_write.cc
db/db_impl_compaction_flush.cc
db/db_impl_files.cc
db/db_impl_open.cc
db/db_impl_debug.cc
db/db_impl_experimental.cc
db/db_impl_readonly.cc
db/db_info_dumper.cc
db/db_iter.cc
db/dbformat.cc
db/event_helpers.cc
db/experimental.cc
db/external_sst_file_ingestion_job.cc
db/file_indexer.cc
db/flush_job.cc
db/flush_scheduler.cc
db/forward_iterator.cc
db/internal_stats.cc
db/log_reader.cc
db/log_writer.cc
db/malloc_stats.cc
db/managed_iterator.cc
db/memtable.cc
db/memtable_list.cc
db/merge_helper.cc
db/merge_operator.cc
db/range_del_aggregator.cc
db/repair.cc
db/snapshot_impl.cc
db/table_cache.cc
db/table_properties_collector.cc
db/transaction_log_impl.cc
db/version_builder.cc
db/version_edit.cc
db/version_set.cc
db/wal_manager.cc
db/write_batch.cc
db/write_batch_base.cc
db/write_controller.cc
db/write_thread.cc
env/env.cc
env/env_chroot.cc
env/env_encryption.cc
env/env_hdfs.cc
env/mock_env.cc
memtable/alloc_tracker.cc
memtable/hash_cuckoo_rep.cc
memtable/hash_linklist_rep.cc
memtable/hash_skiplist_rep.cc
memtable/skiplistrep.cc
memtable/vectorrep.cc
memtable/write_buffer_manager.cc
monitoring/histogram.cc
monitoring/histogram_windowing.cc
monitoring/instrumented_mutex.cc
monitoring/iostats_context.cc
monitoring/perf_context.cc
monitoring/perf_level.cc
monitoring/statistics.cc
monitoring/thread_status_impl.cc
monitoring/thread_status_updater.cc
monitoring/thread_status_util.cc
monitoring/thread_status_util_debug.cc
options/cf_options.cc
options/db_options.cc
options/options.cc
options/options_helper.cc
options/options_parser.cc
options/options_sanity_check.cc
port/stack_trace.cc
table/adaptive_table_factory.cc
table/block.cc
table/block_based_filter_block.cc
table/block_based_table_builder.cc
table/block_based_table_factory.cc
table/block_based_table_reader.cc
table/block_builder.cc
table/block_prefix_index.cc
table/bloom_block.cc
table/cuckoo_table_builder.cc
table/cuckoo_table_factory.cc
table/cuckoo_table_reader.cc
table/flush_block_policy.cc
table/format.cc
table/full_filter_block.cc
table/get_context.cc
table/index_builder.cc
table/iterator.cc
table/merging_iterator.cc
table/meta_blocks.cc
table/partitioned_filter_block.cc
table/persistent_cache_helper.cc
table/plain_table_builder.cc
table/plain_table_factory.cc
table/plain_table_index.cc
table/plain_table_key_coding.cc
table/plain_table_reader.cc
table/sst_file_writer.cc
table/table_properties.cc
table/two_level_iterator.cc
tools/db_bench_tool.cc
tools/dump/db_dump_tool.cc
tools/ldb_cmd.cc
tools/ldb_tool.cc
tools/sst_dump_tool.cc
util/arena.cc
util/auto_roll_logger.cc
util/bloom.cc
util/coding.cc
util/compaction_job_stats_impl.cc
util/comparator.cc
util/concurrent_arena.cc
util/crc32c.cc
util/delete_scheduler.cc
util/dynamic_bloom.cc
util/event_logger.cc
util/file_reader_writer.cc
util/file_util.cc
util/filename.cc
util/filter_policy.cc
util/hash.cc
util/log_buffer.cc
util/murmurhash.cc
util/random.cc
util/rate_limiter.cc
util/slice.cc
util/sst_file_manager_impl.cc
util/status.cc
util/status_message.cc
util/string_util.cc
util/sync_point.cc
util/testutil.cc
util/thread_local.cc
util/threadpool_imp.cc
util/transaction_test_util.cc
util/xxhash.cc
utilities/backupable/backupable_db.cc
utilities/blob_db/blob_db.cc
utilities/blob_db/blob_db_impl.cc
utilities/blob_db/blob_dump_tool.cc
utilities/blob_db/blob_file.cc
utilities/blob_db/blob_log_reader.cc
utilities/blob_db/blob_log_writer.cc
utilities/blob_db/blob_log_format.cc
utilities/blob_db/ttl_extractor.cc
utilities/cassandra/cassandra_compaction_filter.cc
utilities/cassandra/format.cc
utilities/cassandra/merge_operator.cc
utilities/checkpoint/checkpoint_impl.cc
utilities/col_buf_decoder.cc
utilities/col_buf_encoder.cc
utilities/column_aware_encoding_util.cc
utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
utilities/date_tiered/date_tiered_db_impl.cc
utilities/debug.cc
utilities/document/document_db.cc
utilities/document/json_document.cc
utilities/document/json_document_builder.cc
utilities/env_mirror.cc
utilities/env_timed.cc
utilities/geodb/geodb_impl.cc
utilities/leveldb_options/leveldb_options.cc
utilities/lua/rocks_lua_compaction_filter.cc
utilities/memory/memory_util.cc
utilities/merge_operators/max.cc
utilities/merge_operators/put.cc
utilities/merge_operators/string_append/stringappend.cc
utilities/merge_operators/string_append/stringappend2.cc
utilities/merge_operators/uint64add.cc
utilities/option_change_migration/option_change_migration.cc
utilities/options/options_util.cc
utilities/persistent_cache/block_cache_tier.cc
utilities/persistent_cache/block_cache_tier_file.cc
utilities/persistent_cache/block_cache_tier_metadata.cc
utilities/persistent_cache/persistent_cache_tier.cc
utilities/persistent_cache/volatile_tier_impl.cc
utilities/redis/redis_lists.cc
utilities/simulator_cache/sim_cache.cc
utilities/spatialdb/spatial_db.cc
utilities/table_properties_collectors/compact_on_deletion_collector.cc
utilities/transactions/optimistic_transaction_db_impl.cc
utilities/transactions/optimistic_transaction.cc
utilities/transactions/transaction_base.cc
utilities/transactions/pessimistic_transaction_db.cc
utilities/transactions/transaction_db_mutex_impl.cc
utilities/transactions/pessimistic_transaction.cc
utilities/transactions/transaction_lock_mgr.cc
utilities/transactions/transaction_util.cc
utilities/transactions/write_prepared_txn.cc
utilities/ttl/db_ttl_impl.cc
utilities/write_batch_with_index/write_batch_with_index.cc
utilities/write_batch_with_index/write_batch_with_index_internal.cc
$<TARGET_OBJECTS:build_version>)
if(WIN32)
list(APPEND SOURCES
port/win/io_win.cc
port/win/env_win.cc
port/win/env_default.cc
port/win/port_win.cc
port/win/win_logger.cc
port/win/win_thread.cc
port/win/xpress_win.cc)
if(WITH_JEMALLOC)
list(APPEND SOURCES
port/win/win_jemalloc.cc)
endif()
else()
list(APPEND SOURCES
port/port_posix.cc
env/env_posix.cc
env/io_posix.cc)
endif()
set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})
set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
if(WIN32)
set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
else()
set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
target_link_libraries(${ROCKSDB_SHARED_LIB}
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
LINKER_LANGUAGE CXX
VERSION ${ROCKSDB_VERSION}
SOVERSION ${ROCKSDB_VERSION_MAJOR}
CXX_STANDARD 11
OUTPUT_NAME "rocksdb")
endif()
option(WITH_LIBRADOS "Build with librados" OFF)
if(WITH_LIBRADOS)
list(APPEND SOURCES
utilities/env_librados.cc)
list(APPEND THIRDPARTY_LIBS rados)
endif()
add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
target_link_libraries(${ROCKSDB_STATIC_LIB}
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
if(WIN32)
add_library(${ROCKSDB_IMPORT_LIB} SHARED ${SOURCES})
target_link_libraries(${ROCKSDB_IMPORT_LIB}
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
COMPILE_DEFINITIONS "ROCKSDB_DLL;ROCKSDB_LIBRARY_EXPORTS")
if(MSVC)
set_target_properties(${ROCKSDB_STATIC_LIB} PROPERTIES
COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_STATIC_LIB}.pdb")
set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_IMPORT_LIB}.pdb")
endif()
endif()
option(WITH_JNI "build with JNI" OFF)
if(WITH_JNI OR JNI)
message(STATUS "JNI library is enabled")
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
else()
message(STATUS "JNI library is disabled")
endif()
# Installation and packaging
if(WIN32)
option(ROCKSDB_INSTALL_ON_WINDOWS "Enable install target on Windows" OFF)
endif()
if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
# Change default installation prefix on Linux to /usr
set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "Install path prefix, prepended onto install directories." FORCE)
endif()
endif()
include(GNUInstallDirs)
include(CMakePackageConfigHelpers)
set(package_config_destination ${CMAKE_INSTALL_LIBDIR}/cmake/rocksdb)
configure_package_config_file(
${CMAKE_SOURCE_DIR}/cmake/RocksDBConfig.cmake.in RocksDBConfig.cmake
INSTALL_DESTINATION ${package_config_destination}
)
write_basic_package_version_file(
RocksDBConfigVersion.cmake
VERSION ${ROCKSDB_VERSION}
COMPATIBILITY SameMajorVersion
)
install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
install(
TARGETS ${ROCKSDB_STATIC_LIB}
EXPORT RocksDBTargets
COMPONENT devel
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
)
install(
TARGETS ${ROCKSDB_SHARED_LIB}
EXPORT RocksDBTargets
COMPONENT runtime
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
)
install(
EXPORT RocksDBTargets
COMPONENT devel
DESTINATION ${package_config_destination}
NAMESPACE RocksDB::
)
install(
FILES
${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfig.cmake
${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfigVersion.cmake
COMPONENT devel
DESTINATION ${package_config_destination}
)
endif()
option(WITH_TESTS "build with tests" ON)
if(WITH_TESTS)
set(TESTS
cache/cache_test.cc
cache/lru_cache_test.cc
db/column_family_test.cc
db/compact_files_test.cc
db/compaction_iterator_test.cc
db/compaction_job_stats_test.cc
db/compaction_job_test.cc
db/compaction_picker_test.cc
db/comparator_db_test.cc
db/corruption_test.cc
db/cuckoo_table_db_test.cc
db/db_basic_test.cc
db/db_blob_index_test.cc
db/db_block_cache_test.cc
db/db_bloom_filter_test.cc
db/db_compaction_filter_test.cc
db/db_compaction_test.cc
db/db_dynamic_level_test.cc
db/db_flush_test.cc
db/db_inplace_update_test.cc
db/db_io_failure_test.cc
db/db_iter_test.cc
db/db_iterator_test.cc
db/db_log_iter_test.cc
db/db_memtable_test.cc
db/db_merge_operator_test.cc
db/db_options_test.cc
db/db_properties_test.cc
db/db_range_del_test.cc
db/db_sst_test.cc
db/db_statistics_test.cc
db/db_table_properties_test.cc
db/db_tailing_iter_test.cc
db/db_test.cc
db/db_test2.cc
db/db_universal_compaction_test.cc
db/db_wal_test.cc
db/db_write_test.cc
db/dbformat_test.cc
db/deletefile_test.cc
db/external_sst_file_basic_test.cc
db/external_sst_file_test.cc
db/fault_injection_test.cc
db/file_indexer_test.cc
db/filename_test.cc
db/flush_job_test.cc
db/listener_test.cc
db/log_test.cc
db/manual_compaction_test.cc
db/memtable_list_test.cc
db/merge_helper_test.cc
db/merge_test.cc
db/options_file_test.cc
db/perf_context_test.cc
db/plain_table_db_test.cc
db/prefix_test.cc
db/repair_test.cc
db/table_properties_collector_test.cc
db/version_builder_test.cc
db/version_edit_test.cc
db/version_set_test.cc
db/wal_manager_test.cc
db/write_batch_test.cc
db/write_callback_test.cc
db/write_controller_test.cc
env/env_basic_test.cc
env/env_test.cc
env/mock_env_test.cc
memtable/inlineskiplist_test.cc
memtable/skiplist_test.cc
memtable/write_buffer_manager_test.cc
monitoring/histogram_test.cc
monitoring/iostats_context_test.cc
monitoring/statistics_test.cc
options/options_settable_test.cc
options/options_test.cc
table/block_based_filter_block_test.cc
table/block_test.cc
table/cleanable_test.cc
table/cuckoo_table_builder_test.cc
table/cuckoo_table_reader_test.cc
table/full_filter_block_test.cc
table/merger_test.cc
table/table_test.cc
tools/ldb_cmd_test.cc
tools/reduce_levels_test.cc
tools/sst_dump_test.cc
util/arena_test.cc
util/auto_roll_logger_test.cc
util/autovector_test.cc
util/bloom_test.cc
util/coding_test.cc
util/crc32c_test.cc
util/delete_scheduler_test.cc
util/dynamic_bloom_test.cc
util/event_logger_test.cc
util/file_reader_writer_test.cc
util/filelock_test.cc
util/hash_test.cc
util/heap_test.cc
util/rate_limiter_test.cc
util/slice_transform_test.cc
util/timer_queue_test.cc
util/thread_list_test.cc
util/thread_local_test.cc
utilities/backupable/backupable_db_test.cc
utilities/blob_db/blob_db_test.cc
utilities/cassandra/cassandra_functional_test.cc
utilities/cassandra/cassandra_format_test.cc
utilities/cassandra/cassandra_row_merge_test.cc
utilities/cassandra/cassandra_serialize_test.cc
utilities/checkpoint/checkpoint_test.cc
utilities/column_aware_encoding_test.cc
utilities/date_tiered/date_tiered_test.cc
utilities/document/document_db_test.cc
utilities/document/json_document_test.cc
utilities/geodb/geodb_test.cc
utilities/lua/rocks_lua_test.cc
utilities/memory/memory_test.cc
utilities/merge_operators/string_append/stringappend_test.cc
utilities/object_registry_test.cc
utilities/option_change_migration/option_change_migration_test.cc
utilities/options/options_util_test.cc
utilities/persistent_cache/hash_table_test.cc
utilities/persistent_cache/persistent_cache_test.cc
utilities/redis/redis_lists_test.cc
utilities/spatialdb/spatial_db_test.cc
utilities/simulator_cache/sim_cache_test.cc
utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
utilities/transactions/optimistic_transaction_test.cc
utilities/transactions/transaction_test.cc
utilities/ttl/ttl_test.cc
utilities/write_batch_with_index/write_batch_with_index_test.cc
)
if(WITH_LIBRADOS)
list(APPEND TESTS utilities/env_librados_test.cc)
endif()
set(BENCHMARKS
cache/cache_bench.cc
memtable/memtablerep_bench.cc
tools/db_bench.cc
table/table_reader_bench.cc
utilities/column_aware_encoding_exp.cc
utilities/persistent_cache/hash_table_bench.cc)
add_library(testharness OBJECT util/testharness.cc)
foreach(sourcefile ${BENCHMARKS})
get_filename_component(exename ${sourcefile} NAME_WE)
add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
$<TARGET_OBJECTS:testharness>)
target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
endforeach(sourcefile ${BENCHMARKS})
# For test util library that is build only in DEBUG mode
# and linked to tests. Add test only code that is not #ifdefed for Release here.
set(TESTUTIL_SOURCE
db/db_test_util.cc
monitoring/thread_status_updater_debug.cc
table/mock_table.cc
util/fault_injection_test_env.cc
utilities/cassandra/test_utils.cc
)
# test utilities are only build in debug
enable_testing()
add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
if(MSVC)
set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
endif()
set_target_properties(${TESTUTILLIB}
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
)
# Tests are excluded from Release builds
set(TEST_EXES ${TESTS})
foreach(sourcefile ${TEST_EXES})
get_filename_component(exename ${sourcefile} NAME_WE)
add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
$<TARGET_OBJECTS:testharness>)
set_target_properties(${exename}${ARTIFACT_SUFFIX}
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
)
target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
if(NOT "${exename}" MATCHES "db_sanity_test")
add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
endif()
endforeach(sourcefile ${TEST_EXES})
# C executables must link to a shared object
set(C_TESTS db/c_test.c)
set(C_TEST_EXES ${C_TESTS})
foreach(sourcefile ${C_TEST_EXES})
string(REPLACE ".c" "" exename ${sourcefile})
string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
set_target_properties(${exename}${ARTIFACT_SUFFIX}
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
)
target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_IMPORT_LIB} testutillib${ARTIFACT_SUFFIX})
add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
endforeach(sourcefile ${C_TEST_EXES})
endif()
option(WITH_TOOLS "build with tools" ON)
if(WITH_TOOLS)
add_subdirectory(tools)
endif()

5
CONTRIBUTING.md
View File

@@ -12,8 +12,3 @@ Complete your CLA here: <https://code.facebook.com/cla>
If you prefer to sign a paper copy, we can send you a PDF. Send us an
e-mail or create a new github issue to request the CLA in PDF format.
## License
By contributing to RocksDB, you agree that your contributions will be
licensed under the [BSD License](LICENSE).

339
COPYING Normal file
View File

@@ -0,0 +1,339 @@
GNU GENERAL PUBLIC LICENSE
Version 2, June 1991
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
License is intended to guarantee your freedom to share and change free
software--to make sure the software is free for all its users. This
General Public License applies to most of the Free Software
Foundation's software and to any other program whose authors commit to
using it. (Some other Free Software Foundation software is covered by
the GNU Lesser General Public License instead.) You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
this service if you wish), that you receive source code or can get it
if you want it, that you can change the software or use pieces of it
in new free programs; and that you know you can do these things.
To protect your rights, we need to make restrictions that forbid
anyone to deny you these rights or to ask you to surrender the rights.
These restrictions translate to certain responsibilities for you if you
distribute copies of the software, or if you modify it.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must give the recipients all the rights that
you have. You must make sure that they, too, receive or can get the
source code. And you must show them these terms so they know their
rights.
We protect your rights with two steps: (1) copyright the software, and
(2) offer you this license which gives you legal permission to copy,
distribute and/or modify the software.
Also, for each author's protection and ours, we want to make certain
that everyone understands that there is no warranty for this free
software. If the software is modified by someone else and passed on, we
want its recipients to know that what they have is not the original, so
that any problems introduced by others will not reflect on the original
authors' reputations.
Finally, any free program is threatened constantly by software
patents. We wish to avoid the danger that redistributors of a free
program will individually obtain patent licenses, in effect making the
program proprietary. To prevent this, we have made it clear that any
patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and
modification follow.
GNU GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License applies to any program or other work which contains
a notice placed by the copyright holder saying it may be distributed
under the terms of this General Public License. The "Program", below,
refers to any such program or work, and a "work based on the Program"
means either the Program or any derivative work under copyright law:
that is to say, a work containing the Program or a portion of it,
either verbatim or with modifications and/or translated into another
language. (Hereinafter, translation is included without limitation in
the term "modification".) Each licensee is addressed as "you".
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.
1. You may copy and distribute verbatim copies of the Program's
source code as you receive it, in any medium, provided that you
conspicuously and appropriately publish on each copy an appropriate
copyright notice and disclaimer of warranty; keep intact all the
notices that refer to this License and to the absence of any warranty;
and give any other recipients of the Program a copy of this License
along with the Program.
You may charge a fee for the physical act of transferring a copy, and
you may at your option offer warranty protection in exchange for a fee.
2. You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a) You must cause the modified files to carry prominent notices
stating that you changed the files and the date of any change.
b) You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
c) If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.
In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3. You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:
a) Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,
b) Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,
c) Accompany it with the information you received as to the offer
to distribute corresponding source code. (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for
making modifications to it. For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable. However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.
If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.
4. You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.
5. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Program or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.
6. Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.
7. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all. For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.
If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
8. If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded. In such case, this License incorporates
the limitation as if written in the body of this License.
9. The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the Program
specifies a version number of this License which applies to it and "any
later version", you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation. If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.
10. If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission. For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this. Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.
NO WARRANTY
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
REPAIR OR CORRECTION.
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
Also add information on how to contact you by electronic and paper mail.
If the program is interactive, make it output a short notice like this
when it starts in an interactive mode:
Gnomovision version 69, Copyright (C) year name of author
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, the commands you use may
be called something other than `show w' and `show c'; they could even be
mouse-clicks or menu items--whatever suits your program.
You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the program, if
necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
`Gnomovision' (which makes passes at compilers) written by James Hacker.
<signature of Ty Coon>, 1 April 1989
Ty Coon, President of Vice
This General Public License does not permit incorporating your program into
proprietary programs. If your program is a subroutine library, you may
consider it more useful to permit linking proprietary applications with the
library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License.

24
DEFAULT_OPTIONS_HISTORY.md Normal file
View File

@@ -0,0 +1,24 @@
# RocksDB default options change log
## Unreleased
* delayed_write_rate takes the rate given by rate_limiter if not specified.
## 5.2
* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
## 5.0 (11/17/2016)
* Options::allow_concurrent_memtable_write and Options::enable_write_thread_adaptive_yield are now true by default
* Options.level0_stop_writes_trigger default value changes from 24 to 32.
## 4.8.0 (5/2/2016)
* options.max_open_files changes from 5000 to -1. It improves performance, but users need to set file descriptor limit to be large enough and watch memory usage for index and bloom filters.
* options.base_background_compactions changes from max_background_compactions to 1. When users set higher max_background_compactions but the write throughput is not high, the writes are less spiky to disks.
* options.wal_recovery_mode changes from kTolerateCorruptedTailRecords to kPointInTimeRecovery. Avoid some false positive when file system or hardware reorder the writes for file data and metadata.
## 4.7.0 (4/8/2016)
* options.write_buffer_size changes from 4MB to 64MB.
* options.target_file_size_base changes from 2MB to 64MB.
* options.max_bytes_for_level_base changes from 10MB to 256MB.
* options.soft_pending_compaction_bytes_limit changes from 0 (disabled) to 64GB.
* options.hard_pending_compaction_bytes_limit changes from 0 (disabled) to 256GB.
* table_cache_numshardbits changes from 4 to 6.
* max_file_opening_threads changes from 1 to 16.

16
DUMP_FORMAT.md Normal file
View File

@@ -0,0 +1,16 @@
## RocksDB dump format
The version 1 RocksDB dump format is fairly simple:
1) The dump starts with the magic 8 byte identifier "ROCKDUMP"
2) The magic is followed by an 8 byte big-endian version which is 0x00000001.
3) Next are arbitrarily sized chunks of bytes prepended by 4 byte little endian number indicating how large each chunk is.
4) The first chunk is special and is a json string indicating some things about the creation of this dump. It contains the following keys:
* database-path: The path of the database this dump was created from.
* hostname: The hostname of the machine where the dump was created.
* creation-time: Unix seconds since epoc when this dump was created.
5) Following the info dump the slices paired into are key/value pairs.

454
HISTORY.md
View File

@@ -1,12 +1,453 @@
# Rocksdb Change Log
## 5.8.7 (11/28/2017)
### Bug Fixes
* Fix IOError on WAL write doesn't propagate to write group follower
### Unreleased
## 5.8.6 (11/20/2017)
### Bug Fixes
* Fixed aligned_alloc issues with Windows.
----- Past Releases -----
## 5.8.1 (10/23/2017)
### New Features
* Add a new db property "rocksdb.estimate-oldest-key-time" to return oldest data timestamp. The property is available only for FIFO compaction with compaction_options_fifo.allow_compaction = false.
## 5.8.0 (08/30/2017)
### Public API Change
* Users of `Statistics::getHistogramString()` will see fewer histogram buckets and different bucket endpoints.
* `Slice::compare` and BytewiseComparator `Compare` no longer accept `Slice`s containing nullptr.
* `Transaction::Get` and `Transaction::GetForUpdate` variants with `PinnableSlice` added.
### New Features
* Add Iterator::Refresh(), which allows users to update the iterator state so that they can avoid some initialization costs of recreating iterators.
* Replace dynamic_cast<> (except unit test) so people can choose to build with RTTI off. With make, release mode is by default built with -fno-rtti and debug mode is built without it. Users can override it by setting USE_RTTI=0 or 1.
* Universal compactions including the bottom level can be executed in a dedicated thread pool. This alleviates head-of-line blocking in the compaction queue, which cause write stalling, particularly in multi-instance use cases. Users can enable this feature via `Env::SetBackgroundThreads(N, Env::Priority::BOTTOM)`, where `N > 0`.
* Allow merge operator to be called even with a single merge operand during compactions, by appropriately overriding `MergeOperator::AllowSingleOperand`.
* Add `DB::VerifyChecksum()`, which verifies the checksums in all SST files in a running DB.
* Block-based table support for disabling checksums by setting `BlockBasedTableOptions::checksum = kNoChecksum`.
### Bug Fixes
* Fix wrong latencies in `rocksdb.db.get.micros`, `rocksdb.db.write.micros`, and `rocksdb.sst.read.micros`.
* Fix incorrect dropping of deletions during intra-L0 compaction.
* Fix transient reappearance of keys covered by range deletions when memtable prefix bloom filter is enabled.
* Fix potentially wrong file smallest key when range deletions separated by snapshot are written together.
## 5.7.0 (07/13/2017)
### Public API Change
* DB property "rocksdb.sstables" now prints keys in hex form.
### New Features
* Measure estimated number of reads per file. The information can be accessed through DB::GetColumnFamilyMetaData or "rocksdb.sstables" DB property.
* RateLimiter support for throttling background reads, or throttling the sum of background reads and writes. This can give more predictable I/O usage when compaction reads more data than it writes, e.g., due to lots of deletions.
* [Experimental] FIFO compaction with TTL support. It can be enabled by setting CompactionOptionsFIFO.ttl > 0.
* Introduce `EventListener::OnBackgroundError()` callback. Users can implement it to be notified of errors causing the DB to enter read-only mode, and optionally override them.
* Partitioned Index/Filters exiting the experimental mode. To enable partitioned indexes set index_type to kTwoLevelIndexSearch and to further enable partitioned filters set partition_filters to true. To configure the partition size set metadata_block_size.
### Bug Fixes
* Fix discarding empty compaction output files when `DeleteRange()` is used together with subcompactions.
## 5.6.0 (06/06/2017)
### Public API Change
* Scheduling flushes and compactions in the same thread pool is no longer supported by setting `max_background_flushes=0`. Instead, users can achieve this by configuring their high-pri thread pool to have zero threads.
* Replace `Options::max_background_flushes`, `Options::max_background_compactions`, and `Options::base_background_compactions` all with `Options::max_background_jobs`, which automatically decides how many threads to allocate towards flush/compaction.
* options.delayed_write_rate by default take the value of options.rate_limiter rate.
* Replace global variable `IOStatsContext iostats_context` with `IOStatsContext* get_iostats_context()`; replace global variable `PerfContext perf_context` with `PerfContext* get_perf_context()`.
### New Features
* Change ticker/histogram statistics implementations to use core-local storage. This improves aggregation speed compared to our previous thread-local approach, particularly for applications with many threads.
* Users can pass a cache object to write buffer manager, so that they can cap memory usage for memtable and block cache using one single limit.
* Flush will be triggered when 7/8 of the limit introduced by write_buffer_manager or db_write_buffer_size is triggered, so that the hard threshold is hard to hit.
* Introduce WriteOptions.low_pri. If it is true, low priority writes will be throttled if the compaction is behind.
* `DB::IngestExternalFile()` now supports ingesting files into a database containing range deletions.
### Bug Fixes
* Shouldn't ignore return value of fsync() in flush.
## 5.5.0 (05/17/2017)
### New Features
* FIFO compaction to support Intra L0 compaction too with CompactionOptionsFIFO.allow_compaction=true.
* DB::ResetStats() to reset internal stats.
* Statistics::Reset() to reset user stats.
* ldb add option --try_load_options, which will open DB with its own option file.
* Introduce WriteBatch::PopSavePoint to pop the most recent save point explicitly.
* Support dynamically change `max_open_files` option via SetDBOptions()
* Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families.
* Add debugging function `GetAllKeyVersions` to see internal versions of a range of keys.
* Support file ingestion with universal compaction style
* Support file ingestion behind with option `allow_ingest_behind`
* New option enable_pipelined_write which may improve write throughput in case writing from multiple threads and WAL enabled.
### Bug Fixes
* Fix the bug that Direct I/O uses direct reads for non-SST file
## 5.4.0 (04/11/2017)
### Public API Change
* random_access_max_buffer_size no longer has any effect
* Removed Env::EnableReadAhead(), Env::ShouldForwardRawRequest()
* Support dynamically change `stats_dump_period_sec` option via SetDBOptions().
* Added ReadOptions::max_skippable_internal_keys to set a threshold to fail a request as incomplete when too many keys are being skipped when using iterators.
* DB::Get in place of std::string accepts PinnableSlice, which avoids the extra memcpy of value to std::string in most of cases.
* PinnableSlice releases the pinned resources that contain the value when it is destructed or when ::Reset() is called on it.
* The old API that accepts std::string, although discouraged, is still supported.
* Replace Options::use_direct_writes with Options::use_direct_io_for_flush_and_compaction. Read Direct IO wiki for details.
* Added CompactionEventListener and EventListener::OnFlushBegin interfaces.
### New Features
* Memtable flush can be avoided during checkpoint creation if total log file size is smaller than a threshold specified by the user.
* Introduce level-based L0->L0 compactions to reduce file count, so write delays are incurred less often.
* (Experimental) Partitioning filters which creates an index on the partitions. The feature can be enabled by setting partition_filters when using kFullFilter. Currently the feature also requires two-level indexing to be enabled. Number of partitions is the same as the number of partitions for indexes, which is controlled by metadata_block_size.
## 5.3.0 (03/08/2017)
### Public API Change
* Remove disableDataSync option.
* Remove timeout_hint_us option from WriteOptions. The option has been deprecated and has no effect since 3.13.0.
* Remove option min_partial_merge_operands. Partial merge operands will always be merged in flush or compaction if there are more than one.
* Remove option verify_checksums_in_compaction. Compaction will always verify checksum.
### Bug Fixes
* Fix the bug that iterator may skip keys
## 5.2.0 (02/08/2017)
### Public API Change
* NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explict provide one.
* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
* Options::use_direct_writes and Options::use_direct_reads are now ready to use.
* (Experimental) Two-level indexing that partition the index and creates a 2nd level index on the partitions. The feature can be enabled by setting kTwoLevelIndexSearch as IndexType and configuring index_per_partition.
### New Features
* Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations.
* Added new function GetApproximateMemTableStats that approximates both number of records and size of memtables.
* Add Direct I/O mode for SST file I/O
### Bug Fixes
* RangeSync() should work if ROCKSDB_FALLOCATE_PRESENT is not set
* Fix wrong results in a data race case in Get()
* Some fixes related to 2PC.
* Fix bugs of data corruption in direct I/O
## 5.1.0 (01/13/2017)
* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
* Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully.
* BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env.
### Bug Fixes
* Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions.
* When file copying is needed when creating checkpoints or bulk loading files, fsync the file after the file copying.
## 5.0.0 (11/17/2016)
### Public API Change
* Options::max_bytes_for_level_multiplier is now a double along with all getters and setters.
* Support dynamically change `delayed_write_rate` and `max_total_wal_size` options via SetDBOptions().
* Introduce DB::DeleteRange for optimized deletion of large ranges of contiguous keys.
* Support dynamically change `delayed_write_rate` option via SetDBOptions().
* Options::allow_concurrent_memtable_write and Options::enable_write_thread_adaptive_yield are now true by default.
* Remove Tickers::SEQUENCE_NUMBER to avoid confusion if statistics object is shared among RocksDB instance. Alternatively DB::GetLatestSequenceNumber() can be used to get the same value.
* Options.level0_stop_writes_trigger default value changes from 24 to 32.
* New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys.
* Removed flashcache support.
* DB::AddFile() is deprecated and is replaced with DB::IngestExternalFile(). DB::IngestExternalFile() remove all the restrictions that existed for DB::AddFile.
### New Features
* Add avoid_flush_during_shutdown option, which speeds up DB shutdown by not flushing unpersisted data (i.e. with disableWAL = true). Unpersisted data will be lost. The options is dynamically changeable via SetDBOptions().
* Add memtable_insert_with_hint_prefix_extractor option. The option is mean to reduce CPU usage for inserting keys into memtable, if keys can be group by prefix and insert for each prefix are sequential or almost sequential. See include/rocksdb/options.h for more details.
* Add LuaCompactionFilter in utilities. This allows developers to write compaction filters in Lua. To use this feature, LUA_PATH needs to be set to the root directory of Lua.
* No longer populate "LATEST_BACKUP" file in backup directory, which formerly contained the number of the latest backup. The latest backup can be determined by finding the highest numbered file in the "meta/" subdirectory.
## 4.13.0 (10/18/2016)
### Public API Change
* DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference.
* Added Statistics::getAndResetTickerCount().
### New Features
* Add DB::SetDBOptions() to dynamic change base_background_compactions and max_background_compactions.
* Added Iterator::SeekForPrev(). This new API will seek to the last key that less than or equal to the target key.
## 4.12.0 (9/12/2016)
### Public API Change
* CancelAllBackgroundWork() flushes all memtables for databases containing writes that have bypassed the WAL (writes issued with WriteOptions::disableWAL=true) before shutting down background threads.
* Merge options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor into max_compaction_bytes.
* Remove ImmutableCFOptions.
* Add a compression type ZSTD, which can work with ZSTD 0.8.0 or up. Still keep ZSTDNotFinal for compatibility reasons.
### New Features
* Introduce NewClockCache, which is based on CLOCK algorithm with better concurrent performance in some cases. It can be used to replace the default LRU-based block cache and table cache. To use it, RocksDB need to be linked with TBB lib.
* Change ticker/histogram statistics implementations to accumulate data in thread-local storage, which improves CPU performance by reducing cache coherency costs. Callers of CreateDBStatistics do not need to change anything to use this feature.
* Block cache mid-point insertion, where index and filter block are inserted into LRU block cache with higher priority. The feature can be enabled by setting BlockBasedTableOptions::cache_index_and_filter_blocks_with_high_priority to true and high_pri_pool_ratio > 0 when creating NewLRUCache.
## 4.11.0 (8/1/2016)
### Public API Change
* options.memtable_prefix_bloom_huge_page_tlb_size => memtable_huge_page_size. When it is set, RocksDB will try to allocate memory from huge page for memtable too, rather than just memtable bloom filter.
### New Features
* A tool to migrate DB after options change. See include/rocksdb/utilities/option_change_migration.h.
* Add ReadOptions.background_purge_on_iterator_cleanup. If true, we avoid file deletion when destorying iterators.
## 4.10.0 (7/5/2016)
### Public API Change
* options.memtable_prefix_bloom_bits changes to options.memtable_prefix_bloom_bits_ratio and deprecate options.memtable_prefix_bloom_probes
* enum type CompressionType and PerfLevel changes from char to unsigned char. Value of all PerfLevel shift by one.
* Deprecate options.filter_deletes.
### New Features
* Add avoid_flush_during_recovery option.
* Add a read option background_purge_on_iterator_cleanup to avoid deleting files in foreground when destroying iterators. Instead, a job is scheduled in high priority queue and would be executed in a separate background thread.
* RepairDB support for column families. RepairDB now associates data with non-default column families using information embedded in the SST/WAL files (4.7 or later). For data written by 4.6 or earlier, RepairDB associates it with the default column family.
* Add options.write_buffer_manager which allows users to control total memtable sizes across multiple DB instances.
## 4.9.0 (6/9/2016)
### Public API changes
* Add bottommost_compression option, This option can be used to set a specific compression algorithm for the bottommost level (Last level containing files in the DB).
* Introduce CompactionJobInfo::compression, This field state the compression algorithm used to generate the output files of the compaction.
* Deprecate BlockBaseTableOptions.hash_index_allow_collision=false
* Deprecate options builder (GetOptions()).
### New Features
* Introduce NewSimCache() in rocksdb/utilities/sim_cache.h. This function creates a block cache that is able to give simulation results (mainly hit rate) of simulating block behavior with a configurable cache size.
## 4.8.0 (5/2/2016)
### Public API Change
* Allow preset compression dictionary for improved compression of block-based tables. This is supported for zlib, zstd, and lz4. The compression dictionary's size is configurable via CompressionOptions::max_dict_bytes.
* Delete deprecated classes for creating backups (BackupableDB) and restoring from backups (RestoreBackupableDB). Now, BackupEngine should be used for creating backups, and BackupEngineReadOnly should be used for restorations. For more details, see https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F
* Expose estimate of per-level compression ratio via DB property: "rocksdb.compression-ratio-at-levelN".
* Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
### New Features
* Add ReadOptions::readahead_size. If non-zero, NewIterator will create a new table reader which performs reads of the given size.
## 4.7.0 (4/8/2016)
### Public API Change
* rename options compaction_measure_io_stats to report_bg_io_stats and include flush too.
* Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options.
## 4.6.0 (3/10/2016)
### Public API Changes
* Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier.
* Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signature of Cache::Insert() is updated accordingly.
* Tickers [NUMBER_DB_NEXT, NUMBER_DB_PREV, NUMBER_DB_NEXT_FOUND, NUMBER_DB_PREV_FOUND, ITER_BYTES_READ] are not updated immediately. The are updated when the Iterator is deleted.
* Add monotonically increasing counter (DB property "rocksdb.current-super-version-number") that increments upon any change to the LSM tree.
### New Features
* Add CompactionPri::kMinOverlappingRatio, a compaction picking mode friendly to write amplification.
* Deprecate Iterator::IsKeyPinned() and replace it with Iterator::GetProperty() with prop_name="rocksdb.iterator.is.key.pinned"
## 4.5.0 (2/5/2016)
### Public API Changes
* Add a new perf context level between kEnableCount and kEnableTime. Level 2 now does not include timers for mutexes.
* Statistics of mutex operation durations will not be measured by default. If you want to have them enabled, you need to set Statistics::stats_level_ to kAll.
* DBOptions::delete_scheduler and NewDeleteScheduler() are removed, please use DBOptions::sst_file_manager and NewSstFileManager() instead
### New Features
* ldb tool now supports operations to non-default column families.
* Add kPersistedTier to ReadTier. This option allows Get and MultiGet to read only the persited data and skip mem-tables if writes were done with disableWAL = true.
* Add DBOptions::sst_file_manager. Use NewSstFileManager() in include/rocksdb/sst_file_manager.h to create a SstFileManager that can be used to track the total size of SST files and control the SST files deletion rate.
## 4.4.0 (1/14/2016)
### Public API Changes
* Change names in CompactionPri and add a new one.
* Deprecate options.soft_rate_limit and add options.soft_pending_compaction_bytes_limit.
* If options.max_write_buffer_number > 3, writes will be slowed down when writing to the last write buffer to delay a full stop.
* Introduce CompactionJobInfo::compaction_reason, this field include the reason to trigger the compaction.
* After slow down is triggered, if estimated pending compaction bytes keep increasing, slowdown more.
* Increase default options.delayed_write_rate to 2MB/s.
* Added a new parameter --path to ldb tool. --path accepts the name of either MANIFEST, SST or a WAL file. Either --db or --path can be used when calling ldb.
## 4.3.0 (12/8/2015)
### New Features
* CompactionFilter has new member function called IgnoreSnapshots which allows CompactionFilter to be called even if there are snapshots later than the key.
* RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions.
* Introduce LoadLatestOptions() in rocksdb/utilities/options_util.h. This function can construct the latest DBOptions / ColumnFamilyOptions used by the specified RocksDB intance.
* Introduce CheckOptionsCompatibility() in rocksdb/utilities/options_util.h. This function checks whether the input set of options is able to open the specified DB successfully.
### Public API Changes
* When options.db_write_buffer_size triggers, only the column family with the largest column family size will be flushed, not all the column families.
## 4.2.0 (11/9/2015)
### New Features
* Introduce CreateLoggerFromOptions(), this function create a Logger for provided DBOptions.
* Add GetAggregatedIntProperty(), which returns the sum of the GetIntProperty of all the column families.
* Add MemoryUtil in rocksdb/utilities/memory.h. It currently offers a way to get the memory usage by type from a list rocksdb instances.
### Public API Changes
* CompactionFilter::Context includes information of Column Family ID
* The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower.
* TablePropertiesCollectorFactory::CreateTablePropertiesCollector() now takes an option Context, containing the information of column family ID for the file being written.
* Remove DefaultCompactionFilterFactory.
## 4.1.0 (10/8/2015)
### New Features
* Added single delete operation as a more efficient way to delete keys that have not been overwritten.
* Added experimental AddFile() to DB interface that allow users to add files created by SstFileWriter into an empty Database, see include/rocksdb/sst_file_writer.h and DB::AddFile() for more info.
* Added support for opening SST files with .ldb suffix which enables opening LevelDB databases.
* CompactionFilter now supports filtering of merge operands and merge results.
### Public API Changes
* Added SingleDelete() to the DB interface.
* Added AddFile() to DB interface.
* Added SstFileWriter class.
* CompactionFilter has a new method FilterMergeOperand() that RocksDB applies to every merge operand during compaction to decide whether to filter the operand.
* We removed CompactionFilterV2 interfaces from include/rocksdb/compaction_filter.h. The functionality was deprecated already in version 3.13.
## 4.0.0 (9/9/2015)
### New Features
* Added support for transactions. See include/rocksdb/utilities/transaction.h for more info.
* DB::GetProperty() now accepts "rocksdb.aggregated-table-properties" and "rocksdb.aggregated-table-properties-at-levelN", in which case it returns aggregated table properties of the target column family, or the aggregated table properties of the specified level N if the "at-level" version is used.
* Add compression option kZSTDNotFinalCompression for people to experiment ZSTD although its format is not finalized.
* We removed the need for LATEST_BACKUP file in BackupEngine. We still keep writing it when we create new backups (because of backward compatibility), but we don't read it anymore.
### Public API Changes
* Removed class Env::RandomRWFile and Env::NewRandomRWFile().
* Renamed DBOptions.num_subcompactions to DBOptions.max_subcompactions to make the name better match the actual functionality of the option.
* Added Equal() method to the Comparator interface that can optionally be overwritten in cases where equality comparisons can be done more efficiently than three-way comparisons.
* Previous 'experimental' OptimisticTransaction class has been replaced by Transaction class.
## 3.13.0 (8/6/2015)
### New Features
* RollbackToSavePoint() in WriteBatch/WriteBatchWithIndex
* Add NewCompactOnDeletionCollectorFactory() in utilities/table_properties_collectors, which allows rocksdb to mark a SST file as need-compaction when it observes at least D deletion entries in any N consecutive entries in that SST file. Note that this feature depends on an experimental NeedCompact() API --- the result of this API will not persist after DB restart.
* Add DBOptions::delete_scheduler. Use NewDeleteScheduler() in include/rocksdb/delete_scheduler.h to create a DeleteScheduler that can be shared among multiple RocksDB instances to control the file deletion rate of SST files that exist in the first db_path.
### Public API Changes
* Deprecated WriteOptions::timeout_hint_us. We no longer support write timeout. If you really need this option, talk to us and we might consider returning it.
* Deprecated purge_redundant_kvs_while_flush option.
* Removed BackupEngine::NewBackupEngine() and NewReadOnlyBackupEngine() that were deprecated in RocksDB 3.8. Please use BackupEngine::Open() instead.
* Deprecated Compaction Filter V2. We are not aware of any existing use-cases. If you use this filter, your compile will break with RocksDB 3.13. Please let us know if you use it and we'll put it back in RocksDB 3.14.
* Env::FileExists now returns a Status instead of a boolean
* Add statistics::getHistogramString() to print detailed distribution of a histogram metric.
* Add DBOptions::skip_stats_update_on_db_open. When it is on, DB::Open() will run faster as it skips the random reads required for loading necessary stats from SST files to optimize compaction.
## 3.12.0 (7/2/2015)
### New Features
* Added experimental support for optimistic transactions. See include/rocksdb/utilities/optimistic_transaction.h for more info.
* Added a new way to report QPS from db_bench (check out --report_file and --report_interval_seconds)
* Added a cache for individual rows. See DBOptions::row_cache for more info.
* Several new features on EventListener (see include/rocksdb/listener.h):
- OnCompationCompleted() now returns per-compaction job statistics, defined in include/rocksdb/compaction_job_stats.h.
- Added OnTableFileCreated() and OnTableFileDeleted().
* Add compaction_options_universal.enable_trivial_move to true, to allow trivial move while performing universal compaction. Trivial move will happen only when all the input files are non overlapping.
### Public API changes
* EventListener::OnFlushCompleted() now passes FlushJobInfo instead of a list of parameters.
* DB::GetDbIdentity() is now a const function. If this function is overridden in your application, be sure to also make GetDbIdentity() const to avoid compile error.
* Move listeners from ColumnFamilyOptions to DBOptions.
* Add max_write_buffer_number_to_maintain option
* DB::CompactRange()'s parameter reduce_level is changed to change_level, to allow users to move levels to lower levels if allowed. It can be used to migrate a DB from options.level_compaction_dynamic_level_bytes=false to options.level_compaction_dynamic_level_bytes.true.
* Change default value for options.compaction_filter_factory and options.compaction_filter_factory_v2 to nullptr instead of DefaultCompactionFilterFactory and DefaultCompactionFilterFactoryV2.
* If CancelAllBackgroundWork is called without doing a flush after doing loads with WAL disabled, the changes which haven't been flushed before the call to CancelAllBackgroundWork will be lost.
* WBWIIterator::Entry() now returns WriteEntry instead of `const WriteEntry&`
* options.hard_rate_limit is deprecated.
* When options.soft_rate_limit or options.level0_slowdown_writes_trigger is triggered, the way to slow down writes is changed to: write rate to DB is limited to to options.delayed_write_rate.
* DB::GetApproximateSizes() adds a parameter to allow the estimation to include data in mem table, with default to be not to include. It is now only supported in skip list mem table.
* DB::CompactRange() now accept CompactRangeOptions instead of multiple parameters. CompactRangeOptions is defined in include/rocksdb/options.h.
* CompactRange() will now skip bottommost level compaction for level based compaction if there is no compaction filter, bottommost_level_compaction is introduced in CompactRangeOptions to control when it's possible to skip bottommost level compaction. This mean that if you want the compaction to produce a single file you need to set bottommost_level_compaction to BottommostLevelCompaction::kForce.
* Add Cache.GetPinnedUsage() to get the size of memory occupied by entries that are in use by the system.
* DB:Open() will fail if the compression specified in Options is not linked with the binary. If you see this failure, recompile RocksDB with compression libraries present on your system. Also, previously our default compression was snappy. This behavior is now changed. Now, the default compression is snappy only if it's available on the system. If it isn't we change the default to kNoCompression.
* We changed how we account for memory used in block cache. Previously, we only counted the sum of block sizes currently present in block cache. Now, we count the actual memory usage of the blocks. For example, a block of size 4.5KB will use 8KB memory with jemalloc. This might decrease your memory usage and possibly decrease performance. Increase block cache size if you see this happening after an upgrade.
* Add BackupEngineImpl.options_.max_background_operations to specify the maximum number of operations that may be performed in parallel. Add support for parallelized backup and restore.
* Add DB::SyncWAL() that does a WAL sync without blocking writers.
## 3.11.0 (5/19/2015)
### New Features
* Added a new API Cache::SetCapacity(size_t capacity) to dynamically change the maximum configured capacity of the cache. If the new capacity is less than the existing cache usage, the implementation will try to lower the usage by evicting the necessary number of elements following a strict LRU policy.
* Added an experimental API for handling flashcache devices (blacklists background threads from caching their reads) -- NewFlashcacheAwareEnv
* If universal compaction is used and options.num_levels > 1, compact files are tried to be stored in none-L0 with smaller files based on options.target_file_size_base. The limitation of DB size when using universal compaction is greatly mitigated by using more levels. You can set num_levels = 1 to make universal compaction behave as before. If you set num_levels > 1 and want to roll back to a previous version, you need to compact all files to a big file in level 0 (by setting target_file_size_base to be large and CompactRange(<cf_handle>, nullptr, nullptr, true, 0) and reopen the DB with the same version to rewrite the manifest, and then you can open it using previous releases.
* More information about rocksdb background threads are available in Env::GetThreadList(), including the number of bytes read / written by a compaction job, mem-table size and current number of bytes written by a flush job and many more. Check include/rocksdb/thread_status.h for more detail.
### Public API changes
* TablePropertiesCollector::AddUserKey() is added to replace TablePropertiesCollector::Add(). AddUserKey() exposes key type, sequence number and file size up to now to users.
* DBOptions::bytes_per_sync used to apply to both WAL and table files. As of 3.11 it applies only to table files. If you want to use this option to sync WAL in the background, please use wal_bytes_per_sync
## 3.10.0 (3/24/2015)
### New Features
* GetThreadStatus() is now able to report detailed thread status, including:
- Thread Operation including flush and compaction.
- The stage of the current thread operation.
- The elapsed time in micros since the current thread operation started.
More information can be found in include/rocksdb/thread_status.h. In addition, when running db_bench with --thread_status_per_interval, db_bench will also report thread status periodically.
* Changed the LRU caching algorithm so that referenced blocks (by iterators) are never evicted. This change made parameter removeScanCountLimit obsolete. Because of that NewLRUCache doesn't take three arguments anymore. table_cache_remove_scan_limit option is also removed
* By default we now optimize the compilation for the compilation platform (using -march=native). If you want to build portable binary, use 'PORTABLE=1' before the make command.
* We now allow level-compaction to place files in different paths by
specifying them in db_paths along with the target_size.
Lower numbered levels will be placed earlier in the db_paths and higher
numbered levels will be placed later in the db_paths vector.
* Potentially big performance improvements if you're using RocksDB with lots of column families (100-1000)
* Added BlockBasedTableOptions.format_version option, which allows user to specify which version of block based table he wants. As a general guideline, newer versions have more features, but might not be readable by older versions of RocksDB.
* Added new block based table format (version 2), which you can enable by setting BlockBasedTableOptions.format_version = 2. This format changes how we encode size information in compressed blocks and should help with memory allocations if you're using Zlib or BZip2 compressions.
* MemEnv (env that stores data in memory) is now available in default library build. You can create it by calling NewMemEnv().
* Add SliceTransform.SameResultWhenAppended() to help users determine it is safe to apply prefix bloom/hash.
* Block based table now makes use of prefix bloom filter if it is a full fulter.
* Block based table remembers whether a whole key or prefix based bloom filter is supported in SST files. Do a sanity check when reading the file with users' configuration.
* Fixed a bug in ReadOnlyBackupEngine that deleted corrupted backups in some cases, even though the engine was ReadOnly
* options.level_compaction_dynamic_level_bytes, a feature to allow RocksDB to pick dynamic base of bytes for levels. With this feature turned on, we will automatically adjust max bytes for each level. The goal of this feature is to have lower bound on size amplification. For more details, see comments in options.h.
* Added an abstract base class WriteBatchBase for write batches
* Fixed a bug where we start deleting files of a dropped column families even if there are still live references to it
### Public API changes
* Deprecated skip_log_error_on_recovery and table_cache_remove_scan_count_limit options.
* Logger method logv with log level parameter is now virtual
### RocksJava
* Added compression per level API.
* MemEnv is now available in RocksJava via RocksMemEnv class.
* lz4 compression is now included in rocksjava static library when running `make rocksdbjavastatic`.
* Overflowing a size_t when setting rocksdb options now throws an IllegalArgumentException, which removes the necessity for a developer to catch these Exceptions explicitly.
## 3.9.0 (12/8/2014)
### New Features
* Add rocksdb::GetThreadList(), which in the future will return the current status of all
rocksdb-related threads. We will have more code instruments in the following RocksDB
releases.
* Change convert function in rocksdb/utilities/convenience.h to return Status instead of boolean.
Also add support for nested options in convert function
### Public API changes
* New API to create a checkpoint added. Given a directory name, creates a new
database which is an image of the existing database.
* New API LinkFile added to Env. If you implement your own Env class, an
implementation of the API LinkFile will have to be provided.
* MemTableRep takes MemTableAllocator instead of Arena
### Improvements
* RocksDBLite library now becomes smaller and will be compiled with -fno-exceptions flag.
## 3.8.0 (11/14/2014)
### Public API changes
* BackupEngine::NewBackupEngine() was deprecated; please use BackupEngine::Open() from now on.
* BackupableDB/RestoreBackupableDB have new GarbageCollect() methods, which will clean up files from corrupt and obsolete backups.
* BackupableDB/RestoreBackupableDB have new GetCorruptedBackups() methods which list corrupt backups.
### Cleanup
* Bunch of code cleanup, some extra warnings turned on (-Wshadow, -Wshorten-64-to-32, -Wnon-virtual-dtor)
### New features
* CompactFiles and EventListener, although they are still in experimental state
* Full ColumnFamily support in RocksJava.
## 3.7.0 (11/6/2014)
### Public API changes
* Introduce SetOptions() API to allow adjusting a subset of options dynamically online
* Introduce 4 new convenient functions for converting Options from string: GetColumnFamilyOptionsFromMap(), GetColumnFamilyOptionsFromString(), GetDBOptionsFromMap(), GetDBOptionsFromString()
* Remove WriteBatchWithIndex.Delete() overloads using SliceParts
* When opening a DB, if options.max_background_compactions is larger than the existing low pri pool of options.env, it will enlarge it. Similarly, options.max_background_flushes is larger than the existing high pri pool of options.env, it will enlarge it.
## 3.6.0 (10/7/2014)
### Disk format changes
* If you're using RocksDB on ARM platforms and you're using default bloom filter, there is a disk format change you need to be aware of. There are three steps you need to do when you convert to new release: 1. turn off filter policy, 2. compact the whole database, 3. turn on filter policy
### Behavior changes
* We have refactored our system of stalling writes. Any stall-related statistics' meanings are changed. Instead of per-write stall counts, we now count stalls per-epoch, where epochs are periods between flushes and compactions. You'll find more information in our Tuning Perf Guide once we release RocksDB 3.6.
* When disableDataSync=true, we no longer sync the MANIFEST file.
* Add identity_as_first_hash property to CuckooTable. SST file needs to be rebuilt to be opened by reader properly.
### Public API changes
* Change target_file_size_base type to uint64_t from int.
* Remove allow_thread_local. This feature was proved to be stable, so we are turning it always-on.
## 3.5.0 (9/3/2014)
### New Features
* Add include/utilities/write_batch_with_index.h, providing a utilitiy class to query data out of WriteBatch when building it.
* Add include/utilities/write_batch_with_index.h, providing a utility class to query data out of WriteBatch when building it.
* Move BlockBasedTable related options to BlockBasedTableOptions from Options. Change corresponding JNI interface. Options affected include:
no_block_cache, block_cache, block_cache_compressed, block_size, block_size_deviation, block_restart_interval, filter_policy, whole_key_filtering. filter_policy is changed to shared_ptr from a raw pointer.
* Remove deprecated options: disable_seek_compaction and db_stats_log_interval
@@ -20,6 +461,7 @@
* Support Multiple DB paths in universal style compactions
* Add feature of storing plain table index and bloom filter in SST file.
* CompactRange() will never output compacted files to level 0. This used to be the case when all the compaction input files were at level 0.
* Added iterate_upper_bound to define the extent upto which the forward iterator will return entries. This will prevent iterating over delete markers and overwritten entries for edge cases where you want to break out the iterator anyways. This may improve performance in case there are a large number of delete markers or overwritten entries.
### Public API changes
* DBOptions.db_paths now is a vector of a DBPath structure which indicates both of path and target size
@@ -34,7 +476,7 @@
### New Features
* Added JSON API prototype.
* HashLinklist reduces performance outlier caused by skewed bucket by switching data in the bucket from linked list to skip list. Add parameter threshold_use_skiplist in NewHashLinkListRepFactory().
* RocksDB is now able to reclaim storage space more effectively during the compaction process. This is done by compensating the size of each deletion entry by the 2X average value size, which makes compaction to be triggerred by deletion entries more easily.
* RocksDB is now able to reclaim storage space more effectively during the compaction process. This is done by compensating the size of each deletion entry by the 2X average value size, which makes compaction to be triggered by deletion entries more easily.
* Add TimeOut API to write. Now WriteOptions have a variable called timeout_hint_us. With timeout_hint_us set to non-zero, any write associated with this timeout_hint_us may be aborted when it runs longer than the specified timeout_hint_us, and it is guaranteed that any write completes earlier than the specified time-out will not be aborted due to the time-out condition.
* Add a rate_limiter option, which controls total throughput of flush and compaction. The throughput is specified in bytes/sec. Flush always has precedence over compaction when available bandwidth is constrained.
@@ -49,11 +491,11 @@
2) It added some complexity to the important code-paths,
3) None of our internal customers were really using it.
Because of that, Options::disable_seek_compaction is now obsolete. It is still a parameter in Options, so it does not break the build, but it does not have any effect. We plan to completely remove it at some point, so we ask users to please remove this option from your code base.
* Add two paramters to NewHashLinkListRepFactory() for logging on too many entries in a hash bucket when flushing.
* Add two parameters to NewHashLinkListRepFactory() for logging on too many entries in a hash bucket when flushing.
* Added new option BlockBasedTableOptions::hash_index_allow_collision. When enabled, prefix hash index for block-based table will not store prefix and allow hash collision, reducing memory consumption.
### New Features
* PlainTable now supports a new key encoding: for keys of the same prefix, the prefix is only written once. It can be enabled through encoding_type paramter of NewPlainTableFactory()
* PlainTable now supports a new key encoding: for keys of the same prefix, the prefix is only written once. It can be enabled through encoding_type parameter of NewPlainTableFactory()
* Add AdaptiveTableFactory, which is used to convert from a DB of PlainTable to BlockBasedTabe, or vise versa. It can be created using NewAdaptiveTableFactory()
### Performance Improvements

122
INSTALL.md
View File

@@ -1,37 +1,52 @@
## Compilation
**Important**: If you plan to run RocksDB in production, don't compile using default
`make` or `make all`. That will compile RocksDB in debug mode, which is much slower
than release mode.
RocksDB's library should be able to compile without any dependency installed,
although we recommend installing some compression libraries (see below).
We do depend on newer gcc with C++11 support.
We do depend on newer gcc/clang with C++11 support.
There are few options when compiling RocksDB:
* [recommended] `make static_lib` will compile librocksdb.a, RocksDB static library.
* [recommended] `make static_lib` will compile librocksdb.a, RocksDB static library. Compiles static library in release mode.
* `make shared_lib` will compile librocksdb.so, RocksDB shared library.
* `make shared_lib` will compile librocksdb.so, RocksDB shared library. Compiles shared library in release mode.
* `make check` will compile and run all the unit tests
* `make check` will compile and run all the unit tests. `make check` will compile RocksDB in debug mode.
* `make all` will compile our static library, and all our tools and unit tests. Our tools
depend on gflags. You will need to have gflags installed to run `make all`.
depend on gflags. You will need to have gflags installed to run `make all`. This will compile RocksDB in debug mode. Don't
use binaries compiled by `make all` in production.
* By default the binary we produce is optimized for the platform you're compiling on
(`-march=native` or the equivalent). SSE4.2 will thus be enabled automatically if your
CPU supports it. To print a warning if your CPU does not support SSE4.2, build with
`USE_SSE=1 make static_lib` or, if using CMake, `cmake -DFORCE_SSE42=ON`. If you want
to build a portable binary, add `PORTABLE=1` before your make commands, like this:
`PORTABLE=1 make static_lib`.
## Dependencies
* You can link RocksDB with following compression libraries:
- [zlib](http://www.zlib.net/) - a library for data compression.
- [bzip2](http://www.bzip.org/) - a library for data compression.
- [snappy](https://code.google.com/p/snappy/) - a library for fast
- [lz4](https://github.com/lz4/lz4) - a library for extremely fast data compression.
- [snappy](http://google.github.io/snappy/) - a library for fast
data compression.
- [zstandard](http://www.zstd.net) - Fast real-time compression
algorithm.
* All our tools depend on:
- [gflags](https://code.google.com/p/gflags/) - a library that handles
- [gflags](https://gflags.github.io/gflags/) - a library that handles
command line flags processing. You can compile rocksdb library even
if you don't have gflags installed.
## Supported platforms
* **Linux - Ubuntu**
* Upgrade your gcc to version at least 4.7 to get C++11 support.
* Upgrade your gcc to version at least 4.8 to get C++11 support.
* Install gflags. First, try: `sudo apt-get install libgflags-dev`
If this doesn't work and you're using Ubuntu, here's a nice tutorial:
(http://askubuntu.com/questions/312173/installing-gflags-12-04)
@@ -39,46 +54,91 @@ depend on gflags. You will need to have gflags installed to run `make all`.
`sudo apt-get install libsnappy-dev`.
* Install zlib. Try: `sudo apt-get install zlib1g-dev`.
* Install bzip2: `sudo apt-get install libbz2-dev`.
* **Linux - CentOS**
* Upgrade your gcc to version at least 4.7 to get C++11 support:
`yum install gcc47-c++`
* Install lz4: `sudo apt-get install liblz4-dev`.
* Install zstandard: `sudo apt-get install libzstd-dev`.
* **Linux - CentOS / RHEL**
* Upgrade your gcc to version at least 4.8 to get C++11 support:
`yum install gcc48-c++`
* Install gflags:
wget https://gflags.googlecode.com/files/gflags-2.0-no-svn-files.tar.gz
tar -xzvf gflags-2.0-no-svn-files.tar.gz
cd gflags-2.0
git clone https://github.com/gflags/gflags.git
cd gflags
git checkout v2.0
./configure && make && sudo make install
**Notice**: Once installed, please add the include path for gflags to your `CPATH` environment variable and the
lib path to `LIBRARY_PATH`. If installed with default settings, the include path will be `/usr/local/include`
and the lib path will be `/usr/local/lib`.
* Install snappy:
wget https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz
tar -xzvf snappy-1.1.1.tar.gz
cd snappy-1.1.1
./configure && make && sudo make install
sudo yum install snappy snappy-devel
* Install zlib:
sudo yum install zlib
sudo yum install zlib-devel
sudo yum install zlib zlib-devel
* Install bzip2:
sudo yum install bzip2
sudo yum install bzip2-devel
sudo yum install bzip2 bzip2-devel
* Install lz4:
sudo yum install lz4-devel
* Install ASAN (optional for debugging):
sudo yum install libasan
* Install zstandard:
wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
tar zxvf zstd-1.1.3.tar.gz
cd zstd-1.1.3
make && sudo make install
* **OS X**:
* Install latest C++ compiler that supports C++ 11:
* Update XCode: run `xcode-select --install` (or install it from XCode App's settting).
* Install via [homebrew](http://brew.sh/).
* If you're first time developer in MacOS, you still need to run: `xcode-select --install` in your command line.
* run `brew tap homebrew/dupes; brew install gcc47 --use-llvm` to install gcc 4.7 (or higher).
* Install zlib, bzip2 and snappy libraries for compression.
* Install gflags. We have included a script
`build_tools/mac-install-gflags.sh`, which should automatically install it (execute this file instead of runing using "source" command).
If you installed gflags by other means (for example, `brew install gflags`),
please set `LIBRARY_PATH` and `CPATH` accordingly.
* Please note that some of the optimizations/features are disabled in OSX.
We did not run any production workloads on it.
* run `brew tap homebrew/versions; brew install gcc48 --use-llvm` to install gcc 4.8 (or higher).
* run `brew install rocksdb`
* **iOS**:
* Run: `TARGET_OS=IOS make static_lib`
* Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`.
* **Windows**:
* For building with MS Visual Studio 13 you will need Update 4 installed.
* Read and follow the instructions at CMakeLists.txt
* Or install via [vcpkg](https://github.com/microsoft/vcpkg)
* run `vcpkg install rocksdb`
* **AIX 6.1**
* Install AIX Toolbox rpms with gcc
* Use these environment variables:
export PORTABLE=1
export CC=gcc
export AR="ar -X64"
export EXTRA_ARFLAGS=-X64
export EXTRA_CFLAGS=-maix64
export EXTRA_CXXFLAGS=-maix64
export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
export LIBPATH=/opt/freeware/lib
export JAVA_HOME=/usr/java8_64
export PATH=/opt/freeware/bin:$PATH
* **Solaris Sparc**
* Install GCC 4.8.2 and higher.
* Use these environment variables:
export CC=gcc
export EXTRA_CFLAGS=-m64
export EXTRA_CXXFLAGS=-m64
export EXTRA_LDFLAGS=-m64
export PORTABLE=1
export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"

16
LANGUAGE-BINDINGS.md Normal file
View File

@@ -0,0 +1,16 @@
This is the list of all known third-party language bindings for RocksDB. If something is missing, please open a pull request to add it.
* Java - https://github.com/facebook/rocksdb/tree/master/java
* Python - http://pyrocksdb.readthedocs.org/en/latest/
* Perl - https://metacpan.org/pod/RocksDB
* Node.js - https://npmjs.org/package/rocksdb
* Go - https://github.com/tecbot/gorocksdb
* Ruby - http://rubygems.org/gems/rocksdb-ruby
* Haskell - https://hackage.haskell.org/package/rocksdb-haskell
* PHP - https://github.com/Photonios/rocksdb-php
* C# - https://github.com/warrenfalk/rocksdb-sharp
* Rust
* https://github.com/spacejam/rust-rocksdb
* https://github.com/bh1xuw/rust-rocks
* D programming language - https://github.com/b1naryth1ef/rocksdb
* Erlang - https://gitlab.com/barrel-db/erlang-rocksdb

202
LICENSE.Apache Normal file
View File

@@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

BIN
LICENSE → LICENSE.leveldb
View File

Binary file not shown.

1655
Makefile
View File

File diff suppressed because it is too large Load Diff

23
PATENTS
View File

@@ -1,23 +0,0 @@
Additional Grant of Patent Rights
“Software” means the rocksdb software distributed by Facebook, Inc.
Facebook hereby grants you a perpetual, worldwide, royalty-free,
non-exclusive, irrevocable (subject to the termination provision below)
license under any rights in any patent claims owned by Facebook, to make,
have made, use, sell, offer to sell, import, and otherwise transfer the
Software. For avoidance of doubt, no license is granted under Facebook’s
rights in any patent claims that are infringed by (i) modifications to the
Software made by you or a third party, or (ii) the Software in combination
with any software or other technology provided by you or a third party.
The license granted hereunder will terminate, automatically and without
notice, for anyone that makes any claim (including by filing any lawsuit,
assertion or other action) alleging (a) direct, indirect, or contributory
infringement or inducement to infringe any patent: (i) by Facebook or any
of its subsidiaries or affiliates, whether or not such claim is related
to the Software, (ii) by any party if such claim arises in whole or in
part from any software, product or service of Facebook or any of its
subsidiaries or affiliates, whether or not such claim is related to the
Software, or (iii) by any party relating to the Software; or (b) that
any right in any patent claim of Facebook is invalid or unenforceable.

4
README.md
View File

@@ -1,9 +1,11 @@
## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage
[![Build Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
[![Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
RocksDB is developed and maintained by Facebook Database Engineering Team.
It is built on on earlier work on LevelDB by Sanjay Ghemawat (sanjay@google.com)
It is built on earlier work on LevelDB by Sanjay Ghemawat (sanjay@google.com)
and Jeff Dean (jeff@google.com)
This code is a library that forms the core building block for a fast

1
ROCKSDB_LITE.md
View File

@@ -8,6 +8,7 @@ Some examples of the features disabled by ROCKSDB_LITE:
* No support for replication (which we provide in form of TrasactionalIterator)
* No advanced monitoring tools
* No special-purpose memtables that are highly optimized for specific use cases
* No Transactions
When adding a new big feature to RocksDB, please add ROCKSDB_LITE compile guard if:
* Nobody from mobile really needs your feature,

534
TARGETS Normal file
View File

@@ -0,0 +1,534 @@
import os
TARGETS_PATH = os.path.dirname(__file__)
REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
BUCK_BINS = "buck-out/gen/" + REPO_PATH
TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
rocksdb_compiler_flags = [
"-fno-builtin-memcmp",
"-DROCKSDB_PLATFORM_POSIX",
"-DROCKSDB_LIB_IO_POSIX",
"-DROCKSDB_FALLOCATE_PRESENT",
"-DROCKSDB_MALLOC_USABLE_SIZE",
"-DROCKSDB_RANGESYNC_PRESENT",
"-DROCKSDB_SCHED_GETCPU_PRESENT",
"-DROCKSDB_SUPPORT_THREAD_LOCAL",
"-DOS_LINUX",
"-DROCKSDB_UBSAN_RUN",
# Flags to enable libs we include
"-DSNAPPY",
"-DZLIB",
"-DBZIP2",
"-DLZ4",
"-DZSTD",
"-DGFLAGS=gflags",
"-DNUMA",
"-DTBB",
# Needed to compile in fbcode
"-Wno-expansion-to-defined",
]
rocksdb_external_deps = [
('bzip2', None, 'bz2'),
('snappy', None, "snappy"),
('zlib', None, 'z'),
('gflags', None, 'gflags'),
('lz4', None, 'lz4'),
('zstd', None),
('tbb', None),
("numa", None, "numa"),
("googletest", None, "gtest"),
]
rocksdb_preprocessor_flags = [
# Directories with files for #include
"-I" + REPO_PATH + "include/",
"-I" + REPO_PATH,
]
rocksdb_arch_preprocessor_flags = {
"x86_64": ["-DHAVE_SSE42"],
}
cpp_library(
name = "rocksdb_lib",
headers = AutoHeaders.RECURSIVE_GLOB,
srcs = [
"cache/clock_cache.cc",
"cache/lru_cache.cc",
"cache/sharded_cache.cc",
"db/builder.cc",
"db/c.cc",
"db/column_family.cc",
"db/compacted_db_impl.cc",
"db/compaction.cc",
"db/compaction_iterator.cc",
"db/compaction_job.cc",
"db/compaction_picker.cc",
"db/compaction_picker_universal.cc",
"db/convenience.cc",
"db/db_filesnapshot.cc",
"db/db_impl.cc",
"db/db_impl_write.cc",
"db/db_impl_compaction_flush.cc",
"db/db_impl_files.cc",
"db/db_impl_open.cc",
"db/db_impl_debug.cc",
"db/db_impl_experimental.cc",
"db/db_impl_readonly.cc",
"db/db_info_dumper.cc",
"db/db_iter.cc",
"db/dbformat.cc",
"db/event_helpers.cc",
"db/experimental.cc",
"db/external_sst_file_ingestion_job.cc",
"db/file_indexer.cc",
"db/flush_job.cc",
"db/flush_scheduler.cc",
"db/forward_iterator.cc",
"db/internal_stats.cc",
"db/log_reader.cc",
"db/log_writer.cc",
"db/malloc_stats.cc",
"db/managed_iterator.cc",
"db/memtable.cc",
"db/memtable_list.cc",
"db/merge_helper.cc",
"db/merge_operator.cc",
"db/range_del_aggregator.cc",
"db/repair.cc",
"db/snapshot_impl.cc",
"db/table_cache.cc",
"db/table_properties_collector.cc",
"db/transaction_log_impl.cc",
"db/version_builder.cc",
"db/version_edit.cc",
"db/version_set.cc",
"db/wal_manager.cc",
"db/write_batch.cc",
"db/write_batch_base.cc",
"db/write_controller.cc",
"db/write_thread.cc",
"env/env.cc",
"env/env_chroot.cc",
"env/env_encryption.cc",
"env/env_hdfs.cc",
"env/env_posix.cc",
"env/io_posix.cc",
"env/mock_env.cc",
"memtable/alloc_tracker.cc",
"memtable/hash_cuckoo_rep.cc",
"memtable/hash_linklist_rep.cc",
"memtable/hash_skiplist_rep.cc",
"memtable/skiplistrep.cc",
"memtable/vectorrep.cc",
"memtable/write_buffer_manager.cc",
"monitoring/histogram.cc",
"monitoring/histogram_windowing.cc",
"monitoring/instrumented_mutex.cc",
"monitoring/iostats_context.cc",
"monitoring/perf_context.cc",
"monitoring/perf_level.cc",
"monitoring/statistics.cc",
"monitoring/thread_status_impl.cc",
"monitoring/thread_status_updater.cc",
"monitoring/thread_status_updater_debug.cc",
"monitoring/thread_status_util.cc",
"monitoring/thread_status_util_debug.cc",
"options/cf_options.cc",
"options/db_options.cc",
"options/options.cc",
"options/options_helper.cc",
"options/options_parser.cc",
"options/options_sanity_check.cc",
"port/port_posix.cc",
"port/stack_trace.cc",
"table/adaptive_table_factory.cc",
"table/block.cc",
"table/block_based_filter_block.cc",
"table/block_based_table_builder.cc",
"table/block_based_table_factory.cc",
"table/block_based_table_reader.cc",
"table/block_builder.cc",
"table/block_prefix_index.cc",
"table/bloom_block.cc",
"table/cuckoo_table_builder.cc",
"table/cuckoo_table_factory.cc",
"table/cuckoo_table_reader.cc",
"table/flush_block_policy.cc",
"table/format.cc",
"table/full_filter_block.cc",
"table/get_context.cc",
"table/index_builder.cc",
"table/iterator.cc",
"table/merging_iterator.cc",
"table/meta_blocks.cc",
"table/partitioned_filter_block.cc",
"table/persistent_cache_helper.cc",
"table/plain_table_builder.cc",
"table/plain_table_factory.cc",
"table/plain_table_index.cc",
"table/plain_table_key_coding.cc",
"table/plain_table_reader.cc",
"table/sst_file_writer.cc",
"table/table_properties.cc",
"table/two_level_iterator.cc",
"tools/dump/db_dump_tool.cc",
"util/arena.cc",
"util/auto_roll_logger.cc",
"util/bloom.cc",
"util/build_version.cc",
"util/coding.cc",
"util/compaction_job_stats_impl.cc",
"util/comparator.cc",
"util/concurrent_arena.cc",
"util/crc32c.cc",
"util/delete_scheduler.cc",
"util/dynamic_bloom.cc",
"util/event_logger.cc",
"util/file_reader_writer.cc",
"util/file_util.cc",
"util/filename.cc",
"util/filter_policy.cc",
"util/hash.cc",
"util/log_buffer.cc",
"util/murmurhash.cc",
"util/random.cc",
"util/rate_limiter.cc",
"util/slice.cc",
"util/sst_file_manager_impl.cc",
"util/status.cc",
"util/status_message.cc",
"util/string_util.cc",
"util/sync_point.cc",
"util/thread_local.cc",
"util/threadpool_imp.cc",
"util/transaction_test_util.cc",
"util/xxhash.cc",
"utilities/backupable/backupable_db.cc",
"utilities/blob_db/blob_db.cc",
"utilities/blob_db/blob_db_impl.cc",
"utilities/blob_db/blob_file.cc",
"utilities/blob_db/blob_log_reader.cc",
"utilities/blob_db/blob_log_writer.cc",
"utilities/blob_db/blob_log_format.cc",
"utilities/blob_db/ttl_extractor.cc",
"utilities/cassandra/cassandra_compaction_filter.cc",
"utilities/cassandra/format.cc",
"utilities/cassandra/merge_operator.cc",
"utilities/checkpoint/checkpoint_impl.cc",
"utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
"utilities/convenience/info_log_finder.cc",
"utilities/date_tiered/date_tiered_db_impl.cc",
"utilities/debug.cc",
"utilities/document/document_db.cc",
"utilities/document/json_document.cc",
"utilities/document/json_document_builder.cc",
"utilities/env_mirror.cc",
"utilities/env_timed.cc",
"utilities/geodb/geodb_impl.cc",
"utilities/leveldb_options/leveldb_options.cc",
"utilities/lua/rocks_lua_compaction_filter.cc",
"utilities/memory/memory_util.cc",
"utilities/merge_operators/max.cc",
"utilities/merge_operators/put.cc",
"utilities/merge_operators/string_append/stringappend.cc",
"utilities/merge_operators/string_append/stringappend2.cc",
"utilities/merge_operators/uint64add.cc",
"utilities/option_change_migration/option_change_migration.cc",
"utilities/options/options_util.cc",
"utilities/persistent_cache/block_cache_tier.cc",
"utilities/persistent_cache/block_cache_tier_file.cc",
"utilities/persistent_cache/block_cache_tier_metadata.cc",
"utilities/persistent_cache/persistent_cache_tier.cc",
"utilities/persistent_cache/volatile_tier_impl.cc",
"utilities/redis/redis_lists.cc",
"utilities/simulator_cache/sim_cache.cc",
"utilities/spatialdb/spatial_db.cc",
"utilities/table_properties_collectors/compact_on_deletion_collector.cc",
"utilities/transactions/optimistic_transaction_db_impl.cc",
"utilities/transactions/optimistic_transaction.cc",
"utilities/transactions/transaction_base.cc",
"utilities/transactions/pessimistic_transaction_db.cc",
"utilities/transactions/transaction_db_mutex_impl.cc",
"utilities/transactions/pessimistic_transaction.cc",
"utilities/transactions/transaction_lock_mgr.cc",
"utilities/transactions/transaction_util.cc",
"utilities/transactions/write_prepared_txn.cc",
"utilities/ttl/db_ttl_impl.cc",
"utilities/write_batch_with_index/write_batch_with_index.cc",
"utilities/write_batch_with_index/write_batch_with_index_internal.cc",
"tools/ldb_cmd.cc",
"tools/ldb_tool.cc",
"tools/sst_dump_tool.cc",
"utilities/blob_db/blob_dump_tool.cc",
],
deps = [],
preprocessor_flags = rocksdb_preprocessor_flags,
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
compiler_flags = rocksdb_compiler_flags,
external_deps = rocksdb_external_deps,
)
cpp_library(
name = "rocksdb_test_lib",
headers = AutoHeaders.RECURSIVE_GLOB,
srcs = [
"table/mock_table.cc",
"util/fault_injection_test_env.cc",
"util/testharness.cc",
"util/testutil.cc",
"db/db_test_util.cc",
"utilities/cassandra/test_utils.cc",
"utilities/col_buf_encoder.cc",
"utilities/col_buf_decoder.cc",
"utilities/column_aware_encoding_util.cc",
],
deps = [":rocksdb_lib"],
preprocessor_flags = rocksdb_preprocessor_flags,
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
compiler_flags = rocksdb_compiler_flags,
external_deps = rocksdb_external_deps,
)
cpp_library(
name = "rocksdb_tools_lib",
headers = AutoHeaders.RECURSIVE_GLOB,
srcs = [
"tools/db_bench_tool.cc",
"util/testutil.cc",
],
deps = [":rocksdb_lib"],
preprocessor_flags = rocksdb_preprocessor_flags,
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
compiler_flags = rocksdb_compiler_flags,
external_deps = rocksdb_external_deps,
)
cpp_library(
name = "env_basic_test_lib",
headers = AutoHeaders.RECURSIVE_GLOB,
srcs = ["env/env_basic_test.cc"],
deps = [":rocksdb_test_lib"],
preprocessor_flags = rocksdb_preprocessor_flags,
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
compiler_flags = rocksdb_compiler_flags,
external_deps = rocksdb_external_deps,
)
# [test_name, test_src, test_type]
ROCKS_TESTS = [['arena_test', 'util/arena_test.cc', 'serial'],
['auto_roll_logger_test', 'util/auto_roll_logger_test.cc', 'serial'],
['autovector_test', 'util/autovector_test.cc', 'serial'],
['backupable_db_test',
'utilities/backupable/backupable_db_test.cc',
'parallel'],
['blob_db_test', 'utilities/blob_db/blob_db_test.cc', 'serial'],
['block_based_filter_block_test',
'table/block_based_filter_block_test.cc',
'serial'],
['block_test', 'table/block_test.cc', 'serial'],
['bloom_test', 'util/bloom_test.cc', 'serial'],
['c_test', 'db/c_test.c', 'serial'],
['cache_test', 'cache/cache_test.cc', 'serial'],
['cassandra_format_test',
'utilities/cassandra/cassandra_format_test.cc',
'serial'],
['cassandra_functional_test',
'utilities/cassandra/cassandra_functional_test.cc',
'serial'],
['cassandra_row_merge_test',
'utilities/cassandra/cassandra_row_merge_test.cc',
'serial'],
['cassandra_serialize_test',
'utilities/cassandra/cassandra_serialize_test.cc',
'serial'],
['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'],
['cleanable_test', 'table/cleanable_test.cc', 'serial'],
['coding_test', 'util/coding_test.cc', 'serial'],
['column_aware_encoding_test',
'utilities/column_aware_encoding_test.cc',
'serial'],
['column_family_test', 'db/column_family_test.cc', 'serial'],
['compact_files_test', 'db/compact_files_test.cc', 'serial'],
['compact_on_deletion_collector_test',
'utilities/table_properties_collectors/compact_on_deletion_collector_test.cc',
'serial'],
['compaction_iterator_test', 'db/compaction_iterator_test.cc', 'serial'],
['compaction_job_stats_test', 'db/compaction_job_stats_test.cc', 'serial'],
['compaction_job_test', 'db/compaction_job_test.cc', 'serial'],
['compaction_picker_test', 'db/compaction_picker_test.cc', 'serial'],
['comparator_db_test', 'db/comparator_db_test.cc', 'serial'],
['corruption_test', 'db/corruption_test.cc', 'serial'],
['crc32c_test', 'util/crc32c_test.cc', 'serial'],
['cuckoo_table_builder_test', 'table/cuckoo_table_builder_test.cc', 'serial'],
['cuckoo_table_db_test', 'db/cuckoo_table_db_test.cc', 'serial'],
['cuckoo_table_reader_test', 'table/cuckoo_table_reader_test.cc', 'serial'],
['date_tiered_test', 'utilities/date_tiered/date_tiered_test.cc', 'serial'],
['db_basic_test', 'db/db_basic_test.cc', 'serial'],
['db_blob_index_test', 'db/db_blob_index_test.cc', 'serial'],
['db_block_cache_test', 'db/db_block_cache_test.cc', 'serial'],
['db_bloom_filter_test', 'db/db_bloom_filter_test.cc', 'serial'],
['db_compaction_filter_test', 'db/db_compaction_filter_test.cc', 'parallel'],
['db_compaction_test', 'db/db_compaction_test.cc', 'parallel'],
['db_dynamic_level_test', 'db/db_dynamic_level_test.cc', 'serial'],
['db_encryption_test', 'db/db_encryption_test.cc', 'serial'],
['db_flush_test', 'db/db_flush_test.cc', 'serial'],
['db_inplace_update_test', 'db/db_inplace_update_test.cc', 'serial'],
['db_io_failure_test', 'db/db_io_failure_test.cc', 'serial'],
['db_iter_test', 'db/db_iter_test.cc', 'serial'],
['db_iterator_test', 'db/db_iterator_test.cc', 'serial'],
['db_log_iter_test', 'db/db_log_iter_test.cc', 'serial'],
['db_memtable_test', 'db/db_memtable_test.cc', 'serial'],
['db_merge_operator_test', 'db/db_merge_operator_test.cc', 'serial'],
['db_options_test', 'db/db_options_test.cc', 'serial'],
['db_properties_test', 'db/db_properties_test.cc', 'serial'],
['db_range_del_test', 'db/db_range_del_test.cc', 'serial'],
['db_sst_test', 'db/db_sst_test.cc', 'parallel'],
['db_statistics_test', 'db/db_statistics_test.cc', 'serial'],
['db_table_properties_test', 'db/db_table_properties_test.cc', 'serial'],
['db_tailing_iter_test', 'db/db_tailing_iter_test.cc', 'serial'],
['db_test', 'db/db_test.cc', 'parallel'],
['db_test2', 'db/db_test2.cc', 'serial'],
['db_universal_compaction_test',
'db/db_universal_compaction_test.cc',
'parallel'],
['db_wal_test', 'db/db_wal_test.cc', 'parallel'],
['db_write_test', 'db/db_write_test.cc', 'serial'],
['dbformat_test', 'db/dbformat_test.cc', 'serial'],
['delete_scheduler_test', 'util/delete_scheduler_test.cc', 'serial'],
['deletefile_test', 'db/deletefile_test.cc', 'serial'],
['document_db_test', 'utilities/document/document_db_test.cc', 'serial'],
['dynamic_bloom_test', 'util/dynamic_bloom_test.cc', 'serial'],
['env_basic_test', 'env/env_basic_test.cc', 'serial'],
['env_test', 'env/env_test.cc', 'serial'],
['env_timed_test', 'utilities/env_timed_test.cc', 'serial'],
['event_logger_test', 'util/event_logger_test.cc', 'serial'],
['external_sst_file_basic_test',
'db/external_sst_file_basic_test.cc',
'serial'],
['external_sst_file_test', 'db/external_sst_file_test.cc', 'parallel'],
['fault_injection_test', 'db/fault_injection_test.cc', 'parallel'],
['file_indexer_test', 'db/file_indexer_test.cc', 'serial'],
['file_reader_writer_test', 'util/file_reader_writer_test.cc', 'serial'],
['filelock_test', 'util/filelock_test.cc', 'serial'],
['filename_test', 'db/filename_test.cc', 'serial'],
['flush_job_test', 'db/flush_job_test.cc', 'serial'],
['full_filter_block_test', 'table/full_filter_block_test.cc', 'serial'],
['geodb_test', 'utilities/geodb/geodb_test.cc', 'serial'],
['hash_table_test',
'utilities/persistent_cache/hash_table_test.cc',
'serial'],
['hash_test', 'util/hash_test.cc', 'serial'],
['heap_test', 'util/heap_test.cc', 'serial'],
['histogram_test', 'monitoring/histogram_test.cc', 'serial'],
['inlineskiplist_test', 'memtable/inlineskiplist_test.cc', 'parallel'],
['iostats_context_test', 'monitoring/iostats_context_test.cc', 'serial'],
['json_document_test', 'utilities/document/json_document_test.cc', 'serial'],
['ldb_cmd_test', 'tools/ldb_cmd_test.cc', 'serial'],
['listener_test', 'db/listener_test.cc', 'serial'],
['log_test', 'db/log_test.cc', 'serial'],
['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'],
['manual_compaction_test', 'db/manual_compaction_test.cc', 'parallel'],
['memory_test', 'utilities/memory/memory_test.cc', 'serial'],
['memtable_list_test', 'db/memtable_list_test.cc', 'serial'],
['merge_helper_test', 'db/merge_helper_test.cc', 'serial'],
['merge_test', 'db/merge_test.cc', 'serial'],
['merger_test', 'table/merger_test.cc', 'serial'],
['mock_env_test', 'env/mock_env_test.cc', 'serial'],
['object_registry_test', 'utilities/object_registry_test.cc', 'serial'],
['optimistic_transaction_test',
'utilities/transactions/optimistic_transaction_test.cc',
'serial'],
['option_change_migration_test',
'utilities/option_change_migration/option_change_migration_test.cc',
'serial'],
['options_file_test', 'db/options_file_test.cc', 'serial'],
['options_settable_test', 'options/options_settable_test.cc', 'serial'],
['options_test', 'options/options_test.cc', 'serial'],
['options_util_test', 'utilities/options/options_util_test.cc', 'serial'],
['partitioned_filter_block_test',
'table/partitioned_filter_block_test.cc',
'serial'],
['perf_context_test', 'db/perf_context_test.cc', 'serial'],
['persistent_cache_test',
'utilities/persistent_cache/persistent_cache_test.cc',
'parallel'],
['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial'],
['prefix_test', 'db/prefix_test.cc', 'serial'],
['range_del_aggregator_test', 'db/range_del_aggregator_test.cc', 'serial'],
['rate_limiter_test', 'util/rate_limiter_test.cc', 'serial'],
['reduce_levels_test', 'tools/reduce_levels_test.cc', 'serial'],
['repair_test', 'db/repair_test.cc', 'serial'],
['sim_cache_test', 'utilities/simulator_cache/sim_cache_test.cc', 'serial'],
['skiplist_test', 'memtable/skiplist_test.cc', 'serial'],
['slice_transform_test', 'util/slice_transform_test.cc', 'serial'],
['spatial_db_test', 'utilities/spatialdb/spatial_db_test.cc', 'serial'],
['sst_dump_test', 'tools/sst_dump_test.cc', 'serial'],
['statistics_test', 'monitoring/statistics_test.cc', 'serial'],
['stringappend_test',
'utilities/merge_operators/string_append/stringappend_test.cc',
'serial'],
['table_properties_collector_test',
'db/table_properties_collector_test.cc',
'serial'],
['table_test', 'table/table_test.cc', 'parallel'],
['thread_list_test', 'util/thread_list_test.cc', 'serial'],
['thread_local_test', 'util/thread_local_test.cc', 'serial'],
['timer_queue_test', 'util/timer_queue_test.cc', 'serial'],
['transaction_test', 'utilities/transactions/transaction_test.cc', 'serial'],
['ttl_test', 'utilities/ttl/ttl_test.cc', 'serial'],
['util_merge_operators_test',
'utilities/util_merge_operators_test.cc',
'serial'],
['version_builder_test', 'db/version_builder_test.cc', 'serial'],
['version_edit_test', 'db/version_edit_test.cc', 'serial'],
['version_set_test', 'db/version_set_test.cc', 'serial'],
['wal_manager_test', 'db/wal_manager_test.cc', 'serial'],
['write_batch_test', 'db/write_batch_test.cc', 'serial'],
['write_batch_with_index_test',
'utilities/write_batch_with_index/write_batch_with_index_test.cc',
'serial'],
['write_buffer_manager_test',
'memtable/write_buffer_manager_test.cc',
'serial'],
['write_callback_test', 'db/write_callback_test.cc', 'serial'],
['write_controller_test', 'db/write_controller_test.cc', 'serial']]
# Generate a test rule for each entry in ROCKS_TESTS
for test_cfg in ROCKS_TESTS:
test_name = test_cfg[0]
test_cc = test_cfg[1]
ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
test_bin = test_name + "_bin"
cpp_binary (
name = test_bin,
srcs = [test_cc],
deps = [":rocksdb_test_lib"],
preprocessor_flags = rocksdb_preprocessor_flags,
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
compiler_flags = rocksdb_compiler_flags,
external_deps = rocksdb_external_deps,
)
custom_unittest(
name = test_name,
type = ttype,
deps = [":" + test_bin],
command = [TEST_RUNNER, BUCK_BINS + test_bin]
)
custom_unittest(
name = "make_rocksdbjavastatic",
type = "simple",
command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
)
custom_unittest(
name = "make_rocksdb_lite_release",
type = "simple",
command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
)

85
USERS.md Normal file
View File

@@ -0,0 +1,85 @@
This document lists users of RocksDB and their use cases. If you are using RocksDB, please open a pull request and add yourself to the list.
## Facebook
At Facebook, we use RocksDB as storage engines in multiple data management services and a backend for many different stateful services, including:
1. MyRocks -- https://github.com/MySQLOnRocksDB/mysql-5.6
2. MongoRocks -- https://github.com/mongodb-partners/mongo-rocks
3. ZippyDB -- Facebook's distributed key-value store with Paxos-style replication, built on top of RocksDB.[*] https://www.youtube.com/watch?v=DfiN7pG0D0khtt
4. Laser -- Laser is a high query throughput, low (millisecond) latency, key-value storage service built on top of RocksDB.[*]
4. Dragon -- a distributed graph query engine. https://code.facebook.com/posts/1737605303120405/dragon-a-distributed-graph-query-engine/
5. Stylus -- a low-level stream processing framework writtenin C++.[*]
[*] https://research.facebook.com/publications/realtime-data-processing-at-facebook/
## LinkedIn
Two different use cases at Linkedin are using RocksDB as a storage engine:
1. LinkedIn's follow feed for storing user's activities. Check out the blog post: https://engineering.linkedin.com/blog/2016/03/followfeed--linkedin-s-feed-made-faster-and-smarter
2. Apache Samza, open source framework for stream processing
Learn more about those use cases in a Tech Talk by Ankit Gupta and Naveen Somasundaram: http://www.youtube.com/watch?v=plqVp_OnSzg
## Yahoo
Yahoo is using RocksDB as a storage engine for their biggest distributed data store Sherpa. Learn more about it here: http://yahooeng.tumblr.com/post/120730204806/sherpa-scales-new-heights
## CockroachDB
CockroachDB is an open-source geo-replicated transactional database (still in development). They are using RocksDB as their storage engine. Check out their github: https://github.com/cockroachdb/cockroach
## DNANexus
DNANexus is using RocksDB to speed up processing of genomics data.
You can learn more from this great blog post by Mike Lin: http://devblog.dnanexus.com/faster-bam-sorting-with-samtools-and-rocksdb/
## Iron.io
Iron.io is using RocksDB as a storage engine for their distributed queueing system.
Learn more from Tech Talk by Reed Allman: http://www.youtube.com/watch?v=HTjt6oj-RL4
## Tango Me
Tango is using RocksDB as a graph storage to store all users' connection data and other social activity data.
## Turn
Turn is using RocksDB as a storage layer for their key/value store, serving at peak 2.4MM QPS out of different datacenters.
Check out our RocksDB Protobuf merge operator at: https://github.com/vladb38/rocksdb_protobuf
## Santanader UK/Cloudera Profession Services
Check out their blog post: http://blog.cloudera.com/blog/2015/08/inside-santanders-near-real-time-data-ingest-architecture/
## Airbnb
Airbnb is using RocksDB as a storage engine for their personalized search service. You can learn more about it here: https://www.youtube.com/watch?v=ASQ6XMtogMs
## Pinterest
Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtube.com/watch?v=MtFEVEs_2Vo
## Smyte
[Smyte](https://www.smyte.com/) uses RocksDB as the storage layer for their core key-value storage, high-performance counters and time-windowed HyperLogLog services.
## Rakuten Marketing
[Rakuten Marketing](https://marketing.rakuten.com/) uses RocksDB as the disk cache layer for the real-time bidding service in their Performance DSP.
## VWO, Wingify
[VWO's](https://vwo.com/) Smart Code checker and URL helper uses RocksDB to store all the URLs where VWO's Smart Code is installed.
## quasardb
[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark.
quasardb uses a heavily tuned RocksDB as its persistence layer.
## Netflix
[Netflix](http://techblog.netflix.com/2016/05/application-data-caching-using-ssds.html) Netflix uses RocksDB on AWS EC2 instances with local SSD drives to cache application data.
## TiKV
[TiKV](https://github.com/pingcap/tikv) is a GEO-replicated, high-performance, distributed, transactional key-value database. TiKV is powered by Rust and Raft. TiKV uses RocksDB as its persistence layer.
## Apache Flink
[Apache Flink](https://flink.apache.org/news/2016/03/08/release-1.0.0.html) uses RocksDB to store state locally on a machine.
## Dgraph
[Dgraph](https://github.com/dgraph-io/dgraph) is an open-source, scalable, distributed, low latency, high throughput Graph database .They use RocksDB to store state locally on a machine.
## Uber
[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and scalable task queue.
## 360 Pika
[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company
## LzLabs
LzLabs is using RocksDB as a storage engine in their multi-database distributed framework to store application configuration and user data.

34
Vagrantfile vendored Normal file
View File

@@ -0,0 +1,34 @@
# Vagrant file
Vagrant.configure("2") do |config|
config.vm.provider "virtualbox" do |v|
v.memory = 4096
v.cpus = 2
end
config.vm.define "ubuntu14" do |box|
box.vm.box = "ubuntu/trusty64"
end
config.vm.define "centos65" do |box|
box.vm.box = "chef/centos-6.5"
end
config.vm.define "FreeBSD10" do |box|
box.vm.guest = :freebsd
box.vm.box = "robin/freebsd-10"
# FreeBSD does not support 'mount_virtualbox_shared_folder', use NFS
box.vm.synced_folder ".", "/vagrant", :nfs => true, id: "vagrant-root"
box.vm.network "private_network", ip: "10.0.1.10"
# build everything after creating VM, skip using --no-provision
box.vm.provision "shell", inline: <<-SCRIPT
pkg install -y gmake clang35
export CXX=/usr/local/bin/clang++35
cd /vagrant
gmake clean
gmake all OPT=-g
SCRIPT
end
end

228
WINDOWS_PORT.md Normal file
View File

@@ -0,0 +1,228 @@
# Microsoft Contribution Notes
## Contributors
* Alexander Zinoviev https://github.com/zinoale
* Dmitri Smirnov https://github.com/yuslepukhin
* Praveen Rao https://github.com/PraveenSinghRao
* Sherlock Huang https://github.com/SherlockNoMad
## Introduction
RocksDB is a well proven open source key-value persistent store, optimized for fast storage. It provides scalability with number of CPUs and storage IOPS, to support IO-bound, in-memory and write-once workloads, most importantly, to be flexible to allow for innovation.
As Microsoft Bing team we have been continuously pushing hard to improve the scalability, efficiency of platform and eventually benefit Bing end-user satisfaction. We would like to explore the opportunity to embrace open source, RocksDB here, to use, enhance and customize for our usage, and also contribute back to the RocksDB community. Herein, we are pleased to offer this RocksDB port for Windows platform.
These notes describe some decisions and changes we had to make with regards to porting RocksDB on Windows. We hope this will help both reviewers and users of the Windows port.
We are open for comments and improvements.
## OS specifics
All of the porting, testing and benchmarking was done on Windows Server 2012 R2 Datacenter 64-bit but to the best of our knowledge there is not a specific API we used during porting that is unsupported on other Windows OS after Vista.
## Porting goals
We strive to achieve the following goals:
* make use of the existing porting interface of RocksDB
* make minimum [WY2]modifications within platform independent code.
* make all unit test pass both in debug and release builds.
* Note: latest introduction of SyncPoint seems to disable running db_test in Release.
* make performance on par with published benchmarks accounting for HW differences
* we would like to keep the port code inline with the master branch with no forking
## Build system
We have chosen CMake as a widely accepted build system to build the Windows port. It is very fast and convenient.
At the same time it generates Visual Studio projects that are both usable from a command line and IDE.
The top-level CMakeLists.txt file contains description of all targets and build rules. It also provides brief instructions on how to build the software for Windows. One more build related file is thirdparty.inc that also resides on the top level. This file must be edited to point to actual third party libraries location.
We think that it would be beneficial to merge the existing make-based build system and the new cmake-based build system into a single one to use on all platforms.
All building and testing was done for 64-bit. We have not conducted any testing for 32-bit and early reports indicate that it will not run on 32-bit.
## C++ and STL notes
We had to make some minimum changes within the portable files that either account for OS differences or the shortcomings of C++11 support in the current version of the MS compiler. Most or all of them are expected to be fixed in the upcoming compiler releases.
We plan to use this port for our business purposes here at Bing and this provided business justification for this port. This also means, we do not have at present to choose the compiler version at will.
* Certain headers that are not present and not necessary on Windows were simply `#ifndef OS_WIN` in a few places (`unistd.h`)
* All posix specific headers were replaced to port/port.h which worked well
* Replaced `dirent.h` for `port/dirent.h` (very few places) with the implementation of the relevant interfaces within `rocksdb::port` namespace
* Replaced `sys/time.h` to `port/sys_time.h` (few places) implemented equivalents within `rocksdb::port`
* `printf %z` specification is not supported on Windows. To imitate existing standards we came up with a string macro `ROCKSDB_PRIszt` which expands to `%z` on posix systems and to Iu on windows.
* in class member initialization were moved to a __ctors in some cases
* `constexpr` is not supported. We had to replace `std::numeric_limits<>::max/min()` to its C macros for constants. Sometimes we had to make class members `static const` and place a definition within a .cc file.
* `constexpr` for functions was replaced to a template specialization (1 place)
* Union members that have non-trivial constructors were replaced to `char[]` in one place along with bug fixes (spatial experimental feature)
* Zero-sized arrays are deemed a non-standard extension which we converted to 1 size array and that should work well for the purposes of these classes.
* `std::chrono` lacks nanoseconds support (fixed in the upcoming release of the STL) and we had to use `QueryPerfCounter()` within env_win.cc
* Function local statics initialization is still not safe. Used `std::once` to mitigate within WinEnv.
## Windows Environments notes
We endeavored to make it functionally on par with posix_env. This means we replicated the functionality of the thread pool and other things as precise as possible, including:
* Replicate posix logic using std:thread primitives.
* Implement all posix_env disk access functionality.
* Set `use_os_buffer=false` to disable OS disk buffering for WinWritableFile and WinRandomAccessFile.
* Replace `pread/pwrite` with `WriteFile/ReadFile` with `OVERLAPPED` structure.
* Use `SetFileInformationByHandle` to compensate absence of `fallocate`.
### In detail
Even though Windows provides its own efficient thread-pool implementation we chose to replicate posix logic using `std::thread` primitives. This allows anyone to quickly detect any changes within the posix source code and replicate them within windows env. This has proven to work very well. At the same time for anyone who wishes to replace the built-in thread-pool can do so using RocksDB stackable environments.
For disk access we implemented all of the functionality present within the posix_env which includes memory mapped files, random access, rate-limiter support etc.
The `use_os_buffer` flag on Posix platforms currently denotes disabling read-ahead log via `fadvise` mechanism. Windows does not have `fadvise` system call. What is more, it implements disk cache in a way that differs from Linux greatly. It<49>s not an uncommon practice on Windows to perform un-buffered disk access to gain control of the memory consumption. We think that in our use case this may also be a good configuration option at the expense of disk throughput. To compensate one may increase the configured in-memory cache size instead. Thus we have chosen `use_os_buffer=false` to disable OS disk buffering for `WinWritableFile` and `WinRandomAccessFile`. The OS imposes restrictions on the alignment of the disk offsets, buffers used and the amount of data that is read/written when accessing files in un-buffered mode. When the option is true, the classes behave in a standard way. This allows to perform writes and reads in cases when un-buffered access does not make sense such as WAL and MANIFEST.
We have replaced `pread/pwrite` with `WriteFile/ReadFile` with `OVERLAPPED` structure so we can atomically seek to the position of the disk operation but still perform the operation synchronously. Thus we able to emulate that functionality of `pread/pwrite` reasonably well. The only difference is that the file pointer is not returned to its original position but that hardly matters given the random nature of access.
We used `SetFileInformationByHandle` both to truncate files after writing a full final page to disk and to pre-allocate disk space for faster I/O thus compensating for the absence of `fallocate` although some differences remain. For example, the pre-allocated space is not filled with zeros like on Linux, however, on a positive note, the end of file position is also not modified after pre-allocation.
RocksDB renames, copies and deletes files at will even though they may be opened with another handle at the same time. We had to relax and allow nearly all the concurrent access permissions possible.
## Thread-Local Storage
Thread-Local storage plays a significant role for RocksDB performance. Rather than creating a separate implementation we chose to create inline wrappers that forward `pthread_specific` calls to Windows `Tls` interfaces within `rocksdb::port` namespace. This leaves the existing meat of the logic in tact and unchanged and just as maintainable.
To mitigate the lack of thread local storage cleanup on thread-exit we added a limited amount of windows specific code within the same thread_local.cc file that injects a cleanup callback into a `"__tls"` structure within `".CRT$XLB"` data segment. This approach guarantees that the callback is invoked regardless of whether RocksDB used within an executable, standalone DLL or within another DLL.
## Jemalloc usage
When RocksDB is used with Jemalloc the latter needs to be initialized before any of the C++ globals or statics. To accomplish that we injected an initialization routine into `".CRT$XCT"` that is automatically invoked by the runtime before initializing static objects. je-uninit is queued to `atexit()`.
The jemalloc redirecting `new/delete` global operators are used by the linker providing certain conditions are met. See build section in these notes.
## Stack Trace and Unhandled Exception Handler
We decided not to implement these two features because the hosting program as a rule has these two things in it.
We experienced no inconveniences debugging issues in the debugger or analyzing process dumps if need be and thus we did not
see this as a priority.
## Performance results
### Setup
All of the benchmarks are run on the same set of machines. Here are the details of the test setup:
* 2 Intel(R) Xeon(R) E5 2450 0 @ 2.10 GHz (total 16 cores)
* 2 XK0480GDQPH SSD Device, total 894GB free disk
* Machine has 128 GB of RAM
* Operating System: Windows Server 2012 R2 Datacenter
* 100 Million keys; each key is of size 10 bytes, each value is of size 800 bytes
* total database size is ~76GB
* The performance result is based on RocksDB 3.11.
* The parameters used, unless specified, were exactly the same as published in the GitHub Wiki page.
### RocksDB on flash storage
#### Test 1. Bulk Load of keys in Random Order
Version 3.11
* Total Run Time: 17.6 min
* Fillrandom: 5.480 micros/op 182465 ops/sec; 142.0 MB/s
* Compact: 486056544.000 micros/op 0 ops/sec
Version 3.10
* Total Run Time: 16.2 min
* Fillrandom: 5.018 micros/op 199269 ops/sec; 155.1 MB/s
* Compact: 441313173.000 micros/op 0 ops/sec;
#### Test 2. Bulk Load of keys in Sequential Order
Version 3.11
* Fillseq: 4.944 micros/op 202k ops/sec; 157.4 MB/s
Version 3.10
* Fillseq: 4.105 micros/op 243.6k ops/sec; 189.6 MB/s
#### Test 3. Random Write
Version 3.11
* Unbuffered I/O enabled
* Overwrite: 52.661 micros/op 18.9k ops/sec; 14.8 MB/s
Version 3.10
* Unbuffered I/O enabled
* Overwrite: 52.661 micros/op 18.9k ops/sec;
#### Test 4. Random Read
Version 3.11
* Unbuffered I/O enabled
* Readrandom: 15.716 micros/op 63.6k ops/sec; 49.5 MB/s
Version 3.10
* Unbuffered I/O enabled
* Readrandom: 15.548 micros/op 64.3k ops/sec;
#### Test 5. Multi-threaded read and single-threaded write
Version 3.11
* Unbuffered I/O enabled
* Readwhilewriting: 25.128 micros/op 39.7k ops/sec;
Version 3.10
* Unbuffered I/O enabled
* Readwhilewriting: 24.854 micros/op 40.2k ops/sec;
### RocksDB In Memory
#### Test 1. Point Lookup
Version 3.11
80K writes/sec
* Write Rate Achieved: 40.5k write/sec;
* Readwhilewriting: 0.314 micros/op 3187455 ops/sec; 364.8 MB/s (715454999 of 715454999 found)
Version 3.10
* Write Rate Achieved: 50.6k write/sec
* Readwhilewriting: 0.316 micros/op 3162028 ops/sec; (719576999 of 719576999 found)
*10K writes/sec*
Version 3.11
* Write Rate Achieved: 5.8k/s write/sec
* Readwhilewriting: 0.246 micros/op 4062669 ops/sec; 464.9 MB/s (915481999 of 915481999 found)
Version 3.10
* Write Rate Achieved: 5.8k/s write/sec
* Readwhilewriting: 0.244 micros/op 4106253 ops/sec; (927986999 of 927986999 found)
#### Test 2. Prefix Range Query
Version 3.11
80K writes/sec
* Write Rate Achieved: 46.3k/s write/sec
* Readwhilewriting: 0.362 micros/op 2765052 ops/sec; 316.4 MB/s (611549999 of 611549999 found)
Version 3.10
* Write Rate Achieved: 45.8k/s write/sec
* Readwhilewriting: 0.317 micros/op 3154941 ops/sec; (708158999 of 708158999 found)
Version 3.11
10K writes/sec
* Write Rate Achieved: 5.78k write/sec
* Readwhilewriting: 0.269 micros/op 3716692 ops/sec; 425.3 MB/s (837401999 of 837401999 found)
Version 3.10
* Write Rate Achieved: 5.7k write/sec
* Readwhilewriting: 0.261 micros/op 3830152 ops/sec; (863482999 of 863482999 found)
We think that there is still big room to improve the performance, which will be an ongoing effort for us.

15
appveyor.yml Normal file
View File

@@ -0,0 +1,15 @@
version: 1.0.{build}
image: Visual Studio 2015
before_build:
- md %APPVEYOR_BUILD_FOLDER%\build
- cd %APPVEYOR_BUILD_FOLDER%\build
- cmake -G "Visual Studio 14 2015 Win64" -DOPTDBG=1 -DXPRESS=1 -DPORTABLE=1 ..
- cd ..
build:
project: build\rocksdb.sln
parallel: true
verbosity: minimal
test:
test_script:
- ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test -Concurrency 8

172
buckifier/buckify_rocksdb.py Normal file
View File

@@ -0,0 +1,172 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from targets_builder import TARGETSBuilder
from optparse import OptionParser
import os
import fnmatch
import sys
import tempfile
from util import ColorString
import util
# tests to export as libraries for inclusion in other projects
_EXPORTED_TEST_LIBS = ["env_basic_test"]
# Parse src.mk files as a Dictionary of
# VAR_NAME => list of files
def parse_src_mk(repo_path):
src_mk = repo_path + "/src.mk"
src_files = {}
for line in open(src_mk):
line = line.strip()
if len(line) == 0 or line[0] == '#':
continue
if '=' in line:
current_src = line.split('=')[0].strip()
src_files[current_src] = []
elif '.cc' in line:
src_path = line.split('.cc')[0].strip() + '.cc'
src_files[current_src].append(src_path)
return src_files
# get all .cc / .c files
def get_cc_files(repo_path):
cc_files = []
for root, dirnames, filenames in os.walk(repo_path):
root = root[(len(repo_path) + 1):]
if "java" in root:
# Skip java
continue
for filename in fnmatch.filter(filenames, '*.cc'):
cc_files.append(os.path.join(root, filename))
for filename in fnmatch.filter(filenames, '*.c'):
cc_files.append(os.path.join(root, filename))
return cc_files
# Get tests from Makefile
def get_tests(repo_path):
Makefile = repo_path + "/Makefile"
# Dictionary TEST_NAME => IS_PARALLEL
tests = {}
found_tests = False
for line in open(Makefile):
line = line.strip()
if line.startswith("TESTS ="):
found_tests = True
elif found_tests:
if line.endswith("\\"):
# remove the trailing \
line = line[:-1]
line = line.strip()
tests[line] = False
else:
# we consumed all the tests
break
found_parallel_tests = False
for line in open(Makefile):
line = line.strip()
if line.startswith("PARALLEL_TEST ="):
found_parallel_tests = True
elif found_parallel_tests:
if line.endswith("\\"):
# remove the trailing \
line = line[:-1]
line = line.strip()
tests[line] = True
else:
# we consumed all the parallel tests
break
return tests
# Prepare TARGETS file for buck
def generate_targets(repo_path):
print(ColorString.info("Generating TARGETS"))
# parsed src.mk file
src_mk = parse_src_mk(repo_path)
# get all .cc files
cc_files = get_cc_files(repo_path)
# get tests from Makefile
tests = get_tests(repo_path)
if src_mk is None or cc_files is None or tests is None:
return False
TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
# rocksdb_lib
TARGETS.add_library(
"rocksdb_lib",
src_mk["LIB_SOURCES"] +
src_mk["TOOL_LIB_SOURCES"])
# rocksdb_test_lib
TARGETS.add_library(
"rocksdb_test_lib",
src_mk.get("MOCK_LIB_SOURCES", []) +
src_mk.get("TEST_LIB_SOURCES", []) +
src_mk.get("EXP_LIB_SOURCES", []),
[":rocksdb_lib"])
# rocksdb_tools_lib
TARGETS.add_library(
"rocksdb_tools_lib",
src_mk.get("BENCH_LIB_SOURCES", []) +
["util/testutil.cc"],
[":rocksdb_lib"])
# test for every test we found in the Makefile
for test in sorted(tests):
match_src = [src for src in cc_files if ("/%s.c" % test) in src]
if len(match_src) == 0:
print(ColorString.warning("Cannot find .cc file for %s" % test))
continue
elif len(match_src) > 1:
print(ColorString.warning("Found more than one .cc for %s" % test))
print(match_src)
continue
assert(len(match_src) == 1)
is_parallel = tests[test]
TARGETS.register_test(test, match_src[0], is_parallel)
if test in _EXPORTED_TEST_LIBS:
test_library = "%s_lib" % test
TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
TARGETS.flush_tests()
print(ColorString.info("Generated TARGETS Summary:"))
print(ColorString.info("- %d libs" % TARGETS.total_lib))
print(ColorString.info("- %d binarys" % TARGETS.total_bin))
print(ColorString.info("- %d tests" % TARGETS.total_test))
return True
def get_rocksdb_path():
# rocksdb = {script_dir}/..
script_dir = os.path.dirname(sys.argv[0])
script_dir = os.path.abspath(script_dir)
rocksdb_path = os.path.abspath(
os.path.join(script_dir, "../"))
return rocksdb_path
def exit_with_error(msg):
print(ColorString.error(msg))
sys.exit(1)
def main():
# Generate TARGETS file for buck
ok = generate_targets(get_rocksdb_path())
if not ok:
exit_with_error("Failed to generate TARGETS files")
if __name__ == "__main__":
main()

4
buckifier/rocks_test_runner.sh Executable file
View File

@@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Create a tmp directory for the test to use
TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"

65
buckifier/targets_builder.py Normal file
View File

@@ -0,0 +1,65 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import targets_cfg
import pprint
# TODO(tec): replace this with PrettyPrinter
def pretty_list(lst, indent=6):
if lst is None or len(lst) == 0:
return ""
if len(lst) == 1:
return "\"%s\"" % lst[0]
separator = "\",\n%s\"" % (" " * indent)
res = separator.join(lst)
res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 2))
return res
class TARGETSBuilder:
def __init__(self, path):
self.path = path
self.targets_file = open(path, 'w')
self.targets_file.write(targets_cfg.rocksdb_target_header)
self.total_lib = 0
self.total_bin = 0
self.total_test = 0
self.tests_cfg = []
def __del__(self):
self.targets_file.close()
def add_library(self, name, srcs, deps=None, headers=None):
if headers is None:
headers = "AutoHeaders.RECURSIVE_GLOB"
self.targets_file.write(targets_cfg.library_template % (
name,
headers,
pretty_list(srcs),
pretty_list(deps)))
self.total_lib = self.total_lib + 1
def add_binary(self, name, srcs, deps=None):
self.targets_file.write(targets_cfg.binary_template % (
name,
pretty_list(srcs),
pretty_list(deps)))
self.total_bin = self.total_bin + 1
def register_test(self, test_name, src, is_parallel):
exec_mode = "serial"
if is_parallel:
exec_mode = "parallel"
self.tests_cfg.append([test_name, str(src), str(exec_mode)])
self.total_test = self.total_test + 1
def flush_tests(self):
self.targets_file.write(targets_cfg.unittests_template % (
pprint.PrettyPrinter().pformat(self.tests_cfg)
))
self.tests_cfg = []

124
buckifier/targets_cfg.py Normal file
View File

@@ -0,0 +1,124 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
rocksdb_target_header = """
import os
TARGETS_PATH = os.path.dirname(__file__)
REPO_PATH = "rocksdb/src/"
BUCK_BINS = "buck-out/gen/" + REPO_PATH
TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
rocksdb_compiler_flags = [
"-fno-builtin-memcmp",
"-DROCKSDB_PLATFORM_POSIX",
"-DROCKSDB_LIB_IO_POSIX",
"-DROCKSDB_FALLOCATE_PRESENT",
"-DROCKSDB_MALLOC_USABLE_SIZE",
"-DROCKSDB_RANGESYNC_PRESENT",
"-DROCKSDB_SCHED_GETCPU_PRESENT",
"-DROCKSDB_SUPPORT_THREAD_LOCAL",
"-DOS_LINUX",
# Flags to enable libs we include
"-DSNAPPY",
"-DZLIB",
"-DBZIP2",
"-DLZ4",
"-DZSTD",
"-DGFLAGS=gflags",
"-DNUMA",
"-DTBB",
# Needed to compile in fbcode
"-Wno-expansion-to-defined",
]
rocksdb_external_deps = [
('bzip2', None, 'bz2'),
('snappy', None, "snappy"),
('zlib', None, 'z'),
('gflags', None, 'gflags'),
('lz4', None, 'lz4'),
('zstd', None),
('tbb', None),
("numa", None, "numa"),
("googletest", None, "gtest"),
]
rocksdb_preprocessor_flags = [
# Directories with files for #include
"-I" + REPO_PATH + "include/",
"-I" + REPO_PATH,
]
rocksdb_arch_preprocessor_flags = {
"x86_64": ["-DHAVE_SSE42"],
}
"""
library_template = """
cpp_library(
name = "%s",
headers = %s,
srcs = [%s],
deps = [%s],
preprocessor_flags = rocksdb_preprocessor_flags,
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
compiler_flags = rocksdb_compiler_flags,
external_deps = rocksdb_external_deps,
)
"""
binary_template = """
cpp_binary(
name = "%s",
srcs = [%s],
deps = [%s],
preprocessor_flags = rocksdb_preprocessor_flags,
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
compiler_flags = rocksdb_compiler_flags,
external_deps = rocksdb_external_deps,
)
"""
unittests_template = """
# [test_name, test_src, test_type]
ROCKS_TESTS = %s
# Generate a test rule for each entry in ROCKS_TESTS
for test_cfg in ROCKS_TESTS:
test_name = test_cfg[0]
test_cc = test_cfg[1]
ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
test_bin = test_name + "_bin"
cpp_binary (
name = test_bin,
srcs = [test_cc],
deps = [":rocksdb_test_lib"],
preprocessor_flags = rocksdb_preprocessor_flags,
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
compiler_flags = rocksdb_compiler_flags,
external_deps = rocksdb_external_deps,
)
custom_unittest(
name = test_name,
type = ttype,
deps = [":" + test_bin],
command = [TEST_RUNNER, BUCK_BINS + test_bin]
)
custom_unittest(
name = "make_rocksdbjavastatic",
type = "simple",
command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
)
custom_unittest(
name = "make_rocksdb_lite_release",
type = "simple",
command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
)
"""

107
buckifier/util.py Normal file
View File

@@ -0,0 +1,107 @@
"""
This module keeps commonly used components.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import subprocess
import os
import time
class ColorString:
""" Generate colorful strings on terminal """
HEADER = '\033[95m'
BLUE = '\033[94m'
GREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
@staticmethod
def _make_color_str(text, color):
return "".join([color, text.encode('utf-8'), ColorString.ENDC])
@staticmethod
def ok(text):
if ColorString.is_disabled:
return text
return ColorString._make_color_str(text, ColorString.GREEN)
@staticmethod
def info(text):
if ColorString.is_disabled:
return text
return ColorString._make_color_str(text, ColorString.BLUE)
@staticmethod
def header(text):
if ColorString.is_disabled:
return text
return ColorString._make_color_str(text, ColorString.HEADER)
@staticmethod
def error(text):
if ColorString.is_disabled:
return text
return ColorString._make_color_str(text, ColorString.FAIL)
@staticmethod
def warning(text):
if ColorString.is_disabled:
return text
return ColorString._make_color_str(text, ColorString.WARNING)
is_disabled = False
def run_shell_command(shell_cmd, cmd_dir=None):
""" Run a single shell command.
@returns a tuple of shell command return code, stdout, stderr """
if cmd_dir is not None and not os.path.exists(cmd_dir):
run_shell_command("mkdir -p %s" % cmd_dir)
start = time.time()
print("\t>>> Running: " + shell_cmd)
p = subprocess.Popen(shell_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cmd_dir)
stdout, stderr = p.communicate()
end = time.time()
# Report time if we spent more than 5 minutes executing a command
execution_time = end - start
if execution_time > (60 * 5):
mins = (execution_time / 60)
secs = (execution_time % 60)
print("\t>time spent: %d minutes %d seconds" % (mins, secs))
return p.returncode, stdout, stderr
def run_shell_commands(shell_cmds, cmd_dir=None, verbose=False):
""" Execute a sequence of shell commands, which is equivalent to
running `cmd1 && cmd2 && cmd3`
@returns boolean indication if all commands succeeds.
"""
if cmd_dir:
print("\t=== Set current working directory => %s" % cmd_dir)
for shell_cmd in shell_cmds:
ret_code, stdout, stderr = run_shell_command(shell_cmd, cmd_dir)
if stdout:
if verbose or ret_code != 0:
print(ColorString.info("stdout: \n"), stdout)
if stderr:
# contents in stderr is not necessarily to be error messages.
if verbose or ret_code != 0:
print(ColorString.error("stderr: \n"), stderr)
if ret_code != 0:
return False
return True

377
build_tools/RocksDBCommonHelper.php Normal file
View File

@@ -0,0 +1,377 @@
<?php
// Copyright 2004-present Facebook. All Rights Reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
// Name of the environment variables which need to be set by the entity which
// triggers continuous runs so that code at the end of the file gets executed
// and Sandcastle run starts.
define("ENV_POST_RECEIVE_HOOK", "POST_RECEIVE_HOOK");
define("ENV_HTTPS_APP_VALUE", "HTTPS_APP_VALUE");
define("ENV_HTTPS_TOKEN_VALUE", "HTTPS_TOKEN_VALUE");
define("PRIMARY_TOKEN_FILE", '/home/krad/.sandcastle');
define("CONT_RUN_ALIAS", "leveldb");
//////////////////////////////////////////////////////////////////////
/* Run tests in sandcastle */
function postURL($diffID, $url) {
assert(strlen($diffID) > 0);
assert(is_numeric($diffID));
assert(strlen($url) > 0);
$cmd_args = array(
'diff_id' => (int)$diffID,
'name' => sprintf(
'click here for sandcastle tests for D%d',
(int)$diffID
),
'link' => $url
);
$cmd = 'echo ' . escapeshellarg(json_encode($cmd_args))
. ' | arc call-conduit differential.updateunitresults';
shell_exec($cmd);
}
function buildUpdateTestStatusCmd($diffID, $test, $status) {
assert(strlen($diffID) > 0);
assert(is_numeric($diffID));
assert(strlen($test) > 0);
assert(strlen($status) > 0);
$cmd_args = array(
'diff_id' => (int)$diffID,
'name' => $test,
'result' => $status
);
$cmd = 'echo ' . escapeshellarg(json_encode($cmd_args))
. ' | arc call-conduit differential.updateunitresults';
return $cmd;
}
function updateTestStatus($diffID, $test) {
assert(strlen($diffID) > 0);
assert(is_numeric($diffID));
assert(strlen($test) > 0);
shell_exec(buildUpdateTestStatusCmd($diffID, $test, "waiting"));
}
function getSteps($applyDiff, $diffID, $username, $test) {
assert(strlen($username) > 0);
assert(strlen($test) > 0);
if ($applyDiff) {
assert(strlen($diffID) > 0);
assert(is_numeric($diffID));
$arcrc_content = (PHP_OS == "Darwin" ?
exec("cat ~/.arcrc | gzip -f | base64") :
exec("cat ~/.arcrc | gzip -f | base64 -w0"));
assert(strlen($arcrc_content) > 0);
// Sandcastle machines don't have arc setup. We copy the user certificate
// and authenticate using that in Sandcastle.
$setup = array(
"name" => "Setup arcrc",
"shell" => "echo " . escapeshellarg($arcrc_content) . " | base64 --decode"
. " | gzip -d > ~/.arcrc",
"user" => "root"
);
// arc demands certain permission on its config.
// also fix the sticky bit issue in sandcastle
$fix_permission = array(
"name" => "Fix environment",
"shell" => "chmod 600 ~/.arcrc && chmod +t /dev/shm",
"user" => "root"
);
// Construct the steps in the order of execution.
$steps[] = $setup;
$steps[] = $fix_permission;
}
// fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise
// Git thinks it is an uncommited change.
$fix_git_ignore = array(
"name" => "Fix git ignore",
"shell" => "echo fbcode >> .git/info/exclude",
"user" => "root"
);
// This fixes "FATAL: ThreadSanitizer can not mmap the shadow memory"
// Source:
// https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual#FAQ
$fix_kernel_issue = array(
"name" => "Fix kernel issue with tsan",
"shell" => "echo 2 >/proc/sys/kernel/randomize_va_space",
"user" => "root"
);
$steps[] = $fix_git_ignore;
$steps[] = $fix_kernel_issue;
// This will be the command used to execute particular type of tests.
$cmd = "";
if ($applyDiff) {
// Patch the code (keep your fingures crossed).
$patch = array(
"name" => "Patch " . $diffID,
"shell" => "arc --arcrc-file ~/.arcrc "
. "patch --nocommit --diff " . escapeshellarg($diffID),
"user" => "root"
);
$steps[] = $patch;
updateTestStatus($diffID, $test);
$cmd = buildUpdateTestStatusCmd($diffID, $test, "running") . "; ";
}
// Run the actual command.
$cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " .
escapeshellarg($test) . "; exit_code=$?; ";
if ($applyDiff) {
$cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&"
. buildUpdateTestStatusCmd($diffID, $test, "pass") . ")"
. "||" . buildUpdateTestStatusCmd($diffID, $test, "fail")
. "; ";
}
// shell command to sort the tests based on exit code and print
// the output of the log files.
$cat_sorted_logs = "
while read code log_file;
do echo \"################ cat \$log_file [exit_code : \$code] ################\";
cat \$log_file;
done < <(tail -n +2 LOG | sort -k7,7n -k4,4gr | awk '{print \$7,\$NF}')";
// Shell command to cat all log files
$cat_all_logs = "for f in `ls t/!(run-*)`; do echo \$f;cat \$f; done";
// If LOG file exist use it to cat log files sorted by exit code, otherwise
// cat everything
$logs_cmd = "if [ -f LOG ]; then {$cat_sorted_logs}; else {$cat_all_logs}; fi";
$cmd = $cmd . " cat /tmp/precommit-check.log"
. "; shopt -s extglob; {$logs_cmd}"
. "; shopt -u extglob; [[ \$exit_code -eq 0 ]]";
assert(strlen($cmd) > 0);
$run_test = array(
"name" => "Run " . $test,
"shell" => $cmd,
"user" => "root",
"parser" => "python build_tools/error_filter.py " . escapeshellarg($test),
);
$steps[] = $run_test;
if ($applyDiff) {
// Clean up the user arc config we are using.
$cleanup = array(
"name" => "Arc cleanup",
"shell" => "rm -f ~/.arcrc",
"user" => "root"
);
$steps[] = $cleanup;
}
assert(count($steps) > 0);
return $steps;
}
function getSandcastleConfig() {
$sandcastle_config = array();
$cwd = getcwd();
$cwd_token_file = "{$cwd}/.sandcastle";
// This is a case when we're executed from a continuous run. Fetch the values
// from the environment.
if (getenv(ENV_POST_RECEIVE_HOOK)) {
$sandcastle_config[0] = getenv(ENV_HTTPS_APP_VALUE);
$sandcastle_config[1] = getenv(ENV_HTTPS_TOKEN_VALUE);
} else {
// This is a typical `[p]arc diff` case. Fetch the values from the specific
// configuration files.
for ($i = 0; $i < 50; $i++) {
if (file_exists(PRIMARY_TOKEN_FILE) ||
file_exists($cwd_token_file)) {
break;
}
// If we failed to fetch the tokens, sleep for 0.2 second and try again
usleep(200000);
}
assert(file_exists(PRIMARY_TOKEN_FILE) ||
file_exists($cwd_token_file));
// Try the primary location first, followed by a secondary.
if (file_exists(PRIMARY_TOKEN_FILE)) {
$cmd = 'cat ' . PRIMARY_TOKEN_FILE;
} else {
$cmd = 'cat ' . escapeshellarg($cwd_token_file);
}
assert(strlen($cmd) > 0);
$sandcastle_config = explode(':', rtrim(shell_exec($cmd)));
}
// In this case be very explicit about the implications.
if (count($sandcastle_config) != 2) {
echo "Sandcastle configuration files don't contain valid information " .
"or the necessary environment variables aren't defined. Unable " .
"to validate the code changes.";
exit(1);
}
assert(strlen($sandcastle_config[0]) > 0);
assert(strlen($sandcastle_config[1]) > 0);
assert(count($sandcastle_config) > 0);
return $sandcastle_config;
}
// This function can be called either from `[p]arc diff` command or during
// the Git post-receive hook.
function startTestsInSandcastle($applyDiff, $workflow, $diffID) {
// Default options don't terminate on failure, but that's what we want. In
// the current case we use assertions intentionally as "terminate on failure
// invariants".
assert_options(ASSERT_BAIL, true);
// In case of a diff we'll send notificatios to the author. Else it'll go to
// the entire team because failures indicate that build quality has regressed.
$username = $applyDiff ? exec("whoami") : CONT_RUN_ALIAS;
assert(strlen($username) > 0);
if ($applyDiff) {
assert($workflow);
assert(strlen($diffID) > 0);
assert(is_numeric($diffID));
}
// List of tests we want to run in Sandcastle.
$tests = array("unit", "unit_non_shm", "unit_481", "clang_unit", "tsan",
"asan", "lite_test", "valgrind", "release", "release_481",
"clang_release", "clang_analyze", "code_cov",
"java_build", "no_compression", "unity", "ubsan");
$send_email_template = array(
'type' => 'email',
'triggers' => array('fail'),
'emails' => array($username . '@fb.com'),
);
// Construct a job definition for each test and add it to the master plan.
foreach ($tests as $test) {
$stepName = "RocksDB diff " . $diffID . " test " . $test;
if (!$applyDiff) {
$stepName = "RocksDB continuous integration test " . $test;
}
$arg[] = array(
"name" => $stepName,
"report" => array($send_email_template),
"steps" => getSteps($applyDiff, $diffID, $username, $test)
);
}
// We cannot submit the parallel execution master plan to Sandcastle and
// need supply the job plan as a determinator. So we construct a small job
// that will spit out the master job plan which Sandcastle will parse and
// execute. Why compress the job definitions? Otherwise we run over the max
// string size.
$cmd = "echo " . base64_encode(json_encode($arg))
. (PHP_OS == "Darwin" ?
" | gzip -f | base64" :
" | gzip -f | base64 -w0");
assert(strlen($cmd) > 0);
$arg_encoded = shell_exec($cmd);
assert(strlen($arg_encoded) > 0);
$runName = "Run diff " . $diffID . "for user " . $username;
if (!$applyDiff) {
$runName = "RocksDB continuous integration build and test run";
}
$command = array(
"name" => $runName,
"steps" => array()
);
$command["steps"][] = array(
"name" => "Generate determinator",
"shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d"
. " | base64 --decode",
"determinator" => true,
"user" => "root"
);
// Submit to Sandcastle.
$url = 'https://interngraph.intern.facebook.com/sandcastle/create';
$job = array(
'command' => 'SandcastleUniversalCommand',
'args' => $command,
'capabilities' => array(
'vcs' => 'rocksdb-int-git',
'type' => 'lego',
),
'hash' => 'origin/master',
'user' => $username,
'alias' => 'rocksdb-precommit',
'tags' => array('rocksdb'),
'description' => 'Rocksdb precommit job',
);
// Fetch the configuration necessary to submit a successful HTTPS request.
$sandcastle_config = getSandcastleConfig();
$app = $sandcastle_config[0];
$token = $sandcastle_config[1];
$cmd = 'curl -s -k '
. ' -F app=' . escapeshellarg($app)
. ' -F token=' . escapeshellarg($token)
. ' -F job=' . escapeshellarg(json_encode($job))
.' ' . escapeshellarg($url);
$output = shell_exec($cmd);
assert(strlen($output) > 0);
// Extract Sandcastle URL from the response.
preg_match('/url": "(.+)"/', $output, $sandcastle_url);
assert(count($sandcastle_url) > 0, "Unable to submit Sandcastle request.");
assert(strlen($sandcastle_url[1]) > 0, "Unable to extract Sandcastle URL.");
if ($applyDiff) {
echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n";
// Ask Phabricator to display it on the diff UI.
postURL($diffID, $sandcastle_url[1]);
} else {
echo "Continuous integration started Sandcastle tests. You can look at ";
echo "the progress at:\n" . $sandcastle_url[1] . "\n";
}
}
// Continuous run cript will set the environment variable and based on that
// we'll trigger the execution of tests in Sandcastle. In that case we don't
// need to apply any diffs and there's no associated workflow either.
if (getenv(ENV_POST_RECEIVE_HOOK)) {
startTestsInSandcastle(
false /* $applyDiff */,
NULL /* $workflow */,
NULL /* $diffID */);
}

110
build_tools/amalgamate.py Executable file
View File

@@ -0,0 +1,110 @@
#!/usr/bin/python
# amalgamate.py creates an amalgamation from a unity build.
# It can be run with either Python 2 or 3.
# An amalgamation consists of a header that includes the contents of all public
# headers and a source file that includes the contents of all source files and
# private headers.
#
# This script works by starting with the unity build file and recursively expanding
# #include directives. If the #include is found in a public include directory,
# that header is expanded into the amalgamation header.
#
# A particular header is only expanded once, so this script will
# break if there are multiple inclusions of the same header that are expected to
# expand differently. Similarly, this type of code causes issues:
#
# #ifdef FOO
# #include "bar.h"
# // code here
# #else
# #include "bar.h" // oops, doesn't get expanded
# // different code here
# #endif
#
# The solution is to move the include out of the #ifdef.
from __future__ import print_function
import argparse
from os import path
import re
import sys
include_re = re.compile('^[ \t]*#include[ \t]+"(.*)"[ \t]*$')
included = set()
excluded = set()
def find_header(name, abs_path, include_paths):
samedir = path.join(path.dirname(abs_path), name)
if path.exists(samedir):
return samedir
for include_path in include_paths:
include_path = path.join(include_path, name)
if path.exists(include_path):
return include_path
return None
def expand_include(include_path, f, abs_path, source_out, header_out, include_paths, public_include_paths):
if include_path in included:
return False
included.add(include_path)
with open(include_path) as f:
print('#line 1 "{}"'.format(include_path), file=source_out)
process_file(f, include_path, source_out, header_out, include_paths, public_include_paths)
return True
def process_file(f, abs_path, source_out, header_out, include_paths, public_include_paths):
for (line, text) in enumerate(f):
m = include_re.match(text)
if m:
filename = m.groups()[0]
# first check private headers
include_path = find_header(filename, abs_path, include_paths)
if include_path:
if include_path in excluded:
source_out.write(text)
expanded = False
else:
expanded = expand_include(include_path, f, abs_path, source_out, header_out, include_paths, public_include_paths)
else:
# now try public headers
include_path = find_header(filename, abs_path, public_include_paths)
if include_path:
# found public header
expanded = False
if include_path in excluded:
source_out.write(text)
else:
expand_include(include_path, f, abs_path, header_out, None, public_include_paths, [])
else:
sys.exit("unable to find {}, included in {} on line {}".format(filename, abs_path, line))
if expanded:
print('#line {} "{}"'.format(line+1, abs_path), file=source_out)
elif text != "#pragma once\n":
source_out.write(text)
def main():
parser = argparse.ArgumentParser(description="Transform a unity build into an amalgamation")
parser.add_argument("source", help="source file")
parser.add_argument("-I", action="append", dest="include_paths", help="include paths for private headers")
parser.add_argument("-i", action="append", dest="public_include_paths", help="include paths for public headers")
parser.add_argument("-x", action="append", dest="excluded", help="excluded header files")
parser.add_argument("-o", dest="source_out", help="output C++ file", required=True)
parser.add_argument("-H", dest="header_out", help="output C++ header file", required=True)
args = parser.parse_args()
include_paths = list(map(path.abspath, args.include_paths or []))
public_include_paths = list(map(path.abspath, args.public_include_paths or []))
excluded.update(map(path.abspath, args.excluded or []))
filename = args.source
abs_path = path.abspath(filename)
with open(filename) as f, open(args.source_out, 'w') as source_out, open(args.header_out, 'w') as header_out:
print('#line 1 "{}"'.format(filename), file=source_out)
print('#include "{}"'.format(header_out.name), file=source_out)
process_file(f, abs_path, source_out, header_out, include_paths, public_include_paths)
if __name__ == "__main__":
main()

361
build_tools/build_detect_platform
View File

@@ -8,6 +8,7 @@
# CXX C++ Compiler path
# PLATFORM_LDFLAGS Linker flags
# JAVA_LDFLAGS Linker flags for RocksDBJava
# JAVA_STATIC_LDFLAGS Linker flags for RocksDBJava static build
# PLATFORM_SHARED_EXT Extension for shared libraries
# PLATFORM_SHARED_LDFLAGS Flags for building shared library
# PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library
@@ -18,21 +19,21 @@
#
# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
#
# -DLEVELDB_PLATFORM_POSIX if cstdatomic is present
# -DLEVELDB_PLATFORM_NOATOMIC if it is not
# -DROCKSDB_PLATFORM_POSIX if posix-platform based
# -DSNAPPY if the Snappy library is present
# -DLZ4 if the LZ4 library is present
# -DZSTD if the ZSTD library is present
# -DNUMA if the NUMA library is present
# -DTBB if the TBB library is present
#
# Using gflags in rocksdb:
# Our project depends on gflags, which requires users to take some extra steps
# before they can compile the whole repository:
# 1. Install gflags. You may download it from here:
# https://code.google.com/p/gflags/
# 2. Once install, add the include path/lib path for gflags to CPATH and
# LIBRARY_PATH respectively. If installed with default mode, the
# lib and include path will be /usr/local/lib and /usr/local/include
# Mac user can do this by running build_tools/mac-install-gflags.sh
# https://gflags.github.io/gflags/ (Mac users can `brew install gflags`)
# 2. Once installed, add the include path for gflags to your CPATH env var and
# the lib path to LIBRARY_PATH. If installed with default settings, the lib
# will be /usr/local/lib and the include path will be /usr/local/include
OUTPUT=$1
if test -z "$OUTPUT"; then
@@ -43,21 +44,18 @@ fi
# we depend on C++11
PLATFORM_CXXFLAGS="-std=c++11"
# we currently depend on POSIX platform
COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX"
COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX"
# Default to fbcode gcc on internal fb machines
if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
FBCODE_BUILD="true"
if [ -z "$USE_CLANG" ]; then
CENTOS_VERSION=`rpm -q --qf "%{VERSION}" \
$(rpm -q --whatprovides redhat-release)`
if [ "$CENTOS_VERSION" = "6" ]; then
source "$PWD/build_tools/fbcode.gcc481.sh"
else
source "$PWD/build_tools/fbcode.gcc471.sh"
fi
# If we're compiling with TSAN we need pic build
PIC_BUILD=$COMPILE_WITH_TSAN
if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
source "$PWD/build_tools/fbcode_config.sh"
else
source "$PWD/build_tools/fbcode.clang31.sh"
# we need this to build with MySQL. Don't use for other purposes.
source "$PWD/build_tools/fbcode_config4.8.1.sh"
fi
fi
@@ -78,15 +76,26 @@ if test -z "$TARGET_OS"; then
TARGET_OS=`uname -s`
fi
if test -z "$TARGET_ARCHITECTURE"; then
TARGET_ARCHITECTURE=`uname -m`
fi
if test -z "$CLANG_SCAN_BUILD"; then
CLANG_SCAN_BUILD=scan-build
fi
if test -z "$CLANG_ANALYZER"; then
CLANG_ANALYZER=$(which clang++ 2> /dev/null)
fi
COMMON_FLAGS="$COMMON_FLAGS ${CFLAGS}"
CROSS_COMPILE=
PLATFORM_CCFLAGS=
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS"
PLATFORM_SHARED_EXT="so"
PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl,"
PLATFORM_SHARED_LDFLAGS="-Wl,--no-as-needed -shared -Wl,-soname -Wl,"
PLATFORM_SHARED_CFLAGS="-fPIC"
PLATFORM_SHARED_VERSIONED=false
PLATFORM_SHARED_VERSIONED=true
# generic port files (working on all platform by #ifdef) go directly in /port
GENERIC_PORT_FILES=`cd "$ROCKSDB_ROOT"; find port -name '*.cc' | tr "\n" " "`
@@ -106,6 +115,7 @@ case "$TARGET_OS" in
PLATFORM_SHARED_EXT=dylib
PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
CROSS_COMPILE=true
PLATFORM_SHARED_VERSIONED=
;;
Linux)
PLATFORM=OS_LINUX
@@ -118,10 +128,17 @@ case "$TARGET_OS" in
;;
SunOS)
PLATFORM=OS_SOLARIS
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -static-libstdc++ -static-libgcc -m64"
# PORT_FILES=port/sunos/sunos_specific.cc
;;
AIX)
PLATFORM=OS_AIX
CC=gcc
COMMON_FLAGS="$COMMON_FLAGS -maix64 -pthread -fno-builtin-memcmp -D_REENTRANT -DOS_AIX -D__STDC_FORMAT_MACROS"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread -lpthread -lrt -maix64 -static-libstdc++ -static-libgcc"
# PORT_FILES=port/aix/aix_specific.cc
;;
FreeBSD)
PLATFORM=OS_FREEBSD
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
@@ -146,9 +163,20 @@ case "$TARGET_OS" in
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
# PORT_FILES=port/dragonfly/dragonfly_specific.cc
;;
Cygwin)
PLATFORM=CYGWIN
PLATFORM_SHARED_CFLAGS=""
PLATFORM_CXXFLAGS="-std=gnu++11"
COMMON_FLAGS="$COMMON_FLAGS -DCYGWIN"
if [ -z "$USE_CLANG" ]; then
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
fi
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
# PORT_FILES=port/linux/linux_specific.cc
;;
OS_ANDROID_CROSSCOMPILE)
PLATFORM=OS_ANDROID
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DROCKSDB_PLATFORM_POSIX"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS " # All pthread features are in the Android C library
# PORT_FILES=port/android/android.cc
CROSS_COMPILE=true
@@ -158,55 +186,28 @@ case "$TARGET_OS" in
exit 1
esac
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}"
JAVA_LDFLAGS="$PLATFORM_LDFLAGS"
if test -z "$DO_NOT_RUN_BUILD_DETECT_VERSION"; then
"$PWD/build_tools/build_detect_version"
fi
# We want to make a list of all cc files within util, db, table, and helpers
# except for the test and benchmark files. By default, find will output a list
# of all files matching either rule, so we need to append -print to make the
# prune take effect.
DIRS="util db table utilities"
set -f # temporarily disable globbing so that our patterns arent expanded
PRUNE_TEST="-name *test*.cc -prune"
PRUNE_BENCH="-name *bench*.cc -prune"
PORTABLE_FILES=`cd "$ROCKSDB_ROOT"; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
PORTABLE_CPP=`cd "$ROCKSDB_ROOT"; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
set +f # re-enable globbing
# The sources consist of the portable files, plus the platform-specific port
# file.
echo "SOURCES=$PORTABLE_FILES $GENERIC_PORT_FILES $PORT_FILES" >> "$OUTPUT"
echo "SOURCESCPP=$PORTABLE_CPP" >> "$OUTPUT"
echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> "$OUTPUT"
JAVA_STATIC_LDFLAGS="$PLATFORM_LDFLAGS"
if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
# Cross-compiling; do not try any compilation tests.
# Also don't need any compilation tests if compiling on fbcode
true
else
# If -std=c++0x works, use <atomic>. Otherwise use port_posix.h.
$CXX $CFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <atomic>
int main() {}
if ! test $ROCKSDB_DISABLE_FALLOCATE; then
# Test whether fallocate is available
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <fcntl.h>
#include <linux/falloc.h>
int main() {
int fd = open("/dev/null", 0);
fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_ATOMIC_PRESENT"
fi
# Test whether fallocate is available
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <fcntl.h>
int main() {
int fd = open("/dev/null", 0);
fallocate(fd, 0, 0, 1024);
}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT"
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT"
fi
fi
# Test whether Snappy library is installed
@@ -221,11 +222,10 @@ EOF
JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy"
fi
# Test whether gflags library is installed
# http://code.google.com/p/gflags/
# http://gflags.github.io/gflags/
# check if the namespace is gflags
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
#include <gflags/gflags.h>
using namespace gflags;
int main() {}
@@ -233,17 +233,17 @@ EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=gflags"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
fi
# check if namespace is google
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <gflags/gflags.h>
using namespace google;
int main() {}
else
# check if namespace is google
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
#include <gflags/gflags.h>
using namespace google;
int main() {}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
fi
fi
# Test whether zlib library is installed
@@ -280,10 +280,21 @@ EOF
JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
fi
# Test whether zstd library is installed
$CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <zstd.h>
int main() {}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
fi
# Test whether numa is available
$CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null <<EOF
#include <numa.h>
#inlcude <numaif.h>
#include <numaif.h>
int main() {}
EOF
if [ "$?" = 0 ]; then
@@ -292,14 +303,129 @@ EOF
JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma"
fi
# Test whether tcmalloc is available
$CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null <<EOF
# Test whether tbb is available
$CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null <<EOF
#include <tbb/tbb.h>
int main() {}
EOF
if [ "$?" = 0 ]; then
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc"
COMMON_FLAGS="$COMMON_FLAGS -DTBB"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb"
JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb"
fi
# Test whether jemalloc is available
if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
2>/dev/null; then
# This will enable some preprocessor identifiers in the Makefile
JEMALLOC=1
# JEMALLOC can be enabled either using the flag (like here) or by
# providing direct link to the jemalloc library
WITH_JEMALLOC_FLAG=1
else
# jemalloc is not available. Let's try tcmalloc
if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
-ltcmalloc 2>/dev/null; then
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc"
fi
fi
# Test whether malloc_usable_size is available
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <malloc.h>
int main() {
size_t res = malloc_usable_size(0);
return 0;
}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE"
fi
# Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <pthread.h>
int main() {
int x = PTHREAD_MUTEX_ADAPTIVE_NP;
return 0;
}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX"
fi
# Test whether backtrace is available
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <execinfo.h>>
int main() {
void* frames[1];
backtrace_symbols(frames, backtrace(frames, 1));
return 0;
}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
else
# Test whether execinfo library is installed
$CXX $CFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <execinfo.h>
int main() {
void* frames[1];
backtrace_symbols(frames, backtrace(frames, 1));
}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo"
JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo"
fi
fi
# Test if -pg is supported
$CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null <<EOF
int main() {
return 0;
}
EOF
if [ "$?" = 0 ]; then
PROFILING_FLAGS=-pg
fi
# Test whether sync_file_range is supported for compatibility with an old glibc
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <fcntl.h>
int main() {
int fd = open("/dev/null", 0);
sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT"
fi
# Test whether sched_getcpu is supported
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <sched.h>
int main() {
int cpuid = sched_getcpu();
}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT"
fi
fi
# TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning.
# -Wshorten-64-to-32 breaks compilation on FreeBSD i386
if ! [ "$TARGET_OS" = FreeBSD -a "$TARGET_ARCHITECTURE" = i386 ]; then
# Test whether -Wshorten-64-to-32 is available
$CXX $CFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null <<EOF
int main() {}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -Wshorten-64-to-32"
fi
fi
# shall we use HDFS?
@@ -310,7 +436,7 @@ if test "$USE_HDFS"; then
exit 1
fi
HDFS_CCFLAGS="$HDFS_CCFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/linux -DUSE_HDFS"
HDFS_LDFLAGS="$HDFS_LDFLAGS -Wl,--no-whole-archive -lhdfs -L$JAVA_HOME/jre/lib/amd64"
HDFS_LDFLAGS="$HDFS_LDFLAGS -lhdfs -L$JAVA_HOME/jre/lib/amd64"
HDFS_LDFLAGS="$HDFS_LDFLAGS -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib"
HDFS_LDFLAGS="$HDFS_LDFLAGS -ldl -lverify -ljava -ljvm"
COMMON_FLAGS="$COMMON_FLAGS $HDFS_CCFLAGS"
@@ -318,19 +444,65 @@ if test "$USE_HDFS"; then
JAVA_LDFLAGS="$JAVA_LDFLAGS $HDFS_LDFLAGS"
fi
# if Intel SSE instruction set is supported, set USE_SSE=" -msse -msse4.2 "
COMMON_FLAGS="$COMMON_FLAGS $USE_SSE"
if test "$USE_SSE"; then
COMMON_FLAGS="$COMMON_FLAGS -msse4.2"
elif test -z "$PORTABLE"; then
if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
# Tune for this POWER processor, treating '+' models as base models
POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+`
COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then
COMMON_FLAGS="$COMMON_FLAGS -march=native "
fi
fi
$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <cstdint>
#include <nmmintrin.h>
int main() {
volatile uint32_t x = _mm_crc32_u32(0, 0);
}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DHAVE_SSE42"
elif test "$USE_SSE"; then
echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling"
fi
# iOS doesn't support thread-local storage, but this check would erroneously
# succeed because the cross-compiler flags are added by the Makefile, not this
# script.
if [ "$PLATFORM" != IOS ]; then
$CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#if defined(_MSC_VER) && !defined(__thread)
#define __thread __declspec(thread)
#endif
int main() {
static __thread int tls;
}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SUPPORT_THREAD_LOCAL"
fi
fi
PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
VALGRIND_VER="$VALGRIND_VER"
ROCKSDB_MAJOR=`build_tools/version.sh major`
ROCKSDB_MINOR=`build_tools/version.sh minor`
ROCKSDB_PATCH=`build_tools/version.sh patch`
echo "CC=$CC" >> "$OUTPUT"
echo "CXX=$CXX" >> "$OUTPUT"
echo "PLATFORM=$PLATFORM" >> "$OUTPUT"
echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT"
echo "JAVA_LDFLAGS=$JAVA_LDFLAGS" >> "$OUTPUT"
echo "JAVA_STATIC_LDFLAGS=$JAVA_STATIC_LDFLAGS" >> "$OUTPUT"
echo "VALGRIND_VER=$VALGRIND_VER" >> "$OUTPUT"
echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT"
echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> "$OUTPUT"
@@ -341,3 +513,20 @@ echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> "$OUTPUT"
echo "EXEC_LDFLAGS=$EXEC_LDFLAGS" >> "$OUTPUT"
echo "JEMALLOC_INCLUDE=$JEMALLOC_INCLUDE" >> "$OUTPUT"
echo "JEMALLOC_LIB=$JEMALLOC_LIB" >> "$OUTPUT"
echo "ROCKSDB_MAJOR=$ROCKSDB_MAJOR" >> "$OUTPUT"
echo "ROCKSDB_MINOR=$ROCKSDB_MINOR" >> "$OUTPUT"
echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
# This will enable some related identifiers for the preprocessor
if test -n "$JEMALLOC"; then
echo "JEMALLOC=1" >> "$OUTPUT"
fi
# Indicates that jemalloc should be enabled using -ljemalloc flag
# The alternative is to porvide a direct link to the library via JEMALLOC_LIB
# and JEMALLOC_INCLUDE
if test -n "$WITH_JEMALLOC_FLAG"; then
echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT"
fi
echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"

22
build_tools/build_detect_version
View File

@@ -1,22 +0,0 @@
#!/bin/sh
#
# Record the version of the source that we are compiling.
# We keep a record of the git revision in util/version.cc. This source file
# is then built as a regular source file as part of the compilation process.
# One can run "strings executable_filename | grep _build_" to find the version of
# the source that we used to build the executable file.
OUTFILE="$PWD/util/build_version.cc"
GIT_SHA=""
if command -v git >/dev/null 2>&1; then
GIT_SHA=$(git rev-parse HEAD 2>/dev/null)
fi
cat > "${OUTFILE}" <<EOF
#include "build_version.h"
const char* rocksdb_build_git_sha = "rocksdb_build_git_sha:${GIT_SHA}";
const char* rocksdb_build_git_datetime = "rocksdb_build_git_datetime:$(date)";
const char* rocksdb_build_compile_date = __DATE__;
const char* rocksdb_build_compile_time = __TIME__;
EOF

135
build_tools/cont_integration.sh Executable file
View File

@@ -0,0 +1,135 @@
#!/usr/bin/env bash
#
# Copyright (c) 2016, Facebook. All rights reserved.
#
# Overall wrapper script for RocksDB continuous builds. The implementation is a
# trivial pulling scheme. We loop infinitely, check if any new changes have been
# committed, if yes then trigger a Sandcastle run, and finally go to sleep again
# for a certain interval.
#
SRC_GIT_REPO=/data/git/rocksdb-public
error=0
function log {
DATE=`date +%Y-%m-%d:%H:%M:%S`
echo $DATE $@
}
function log_err {
log "ERROR: $@ Error code: $error."
}
function update_repo_status {
# Update the parent first.
pushd $SRC_GIT_REPO
# This is a fatal error. Something in the environment isn't right and we will
# terminate the execution.
error=$?
if [ ! $error -eq 0 ]; then
log_err "Where is $SRC_GIT_REPO?"
exit $error
fi
HTTPS_PROXY=fwdproxy:8080 git fetch -f
error=$?
if [ ! $error -eq 0 ]; then
log_err "git fetch -f failed."
popd
return $error
fi
git update-ref refs/heads/master refs/remotes/origin/master
error=$?
if [ ! $error -eq 0 ]; then
log_err "git update-ref failed."
popd
return $error
fi
popd
# We're back in an instance-specific directory. Get the latest changes.
git pull --rebase
error=$?
if [ ! $error -eq 0 ]; then
log_err "git pull --rebase failed."
return $error
fi
}
#
# Execution starts here.
#
# Path to the determinator from the root of the RocksDB repo.
CONTRUN_DETERMINATOR=./build_tools/RocksDBCommonHelper.php
# Value of the previous commit.
PREV_COMMIT=
log "Starting to monitor for new RocksDB changes ..."
log "Running under `pwd` as `whoami`."
# Paranoia. Make sure that we're using the right branch.
git checkout master
error=$?
if [ ! $error -eq 0 ]; then
log_err "This is not good. Can't checkout master. Bye-bye!"
exit 1
fi
# We'll run forever and let the execution environment terminate us if we'll
# exceed whatever timeout is set for the job.
while true;
do
# Get the latest changes committed.
update_repo_status
error=$?
if [ $error -eq 0 ]; then
LAST_COMMIT=`git log -1 | head -1 | grep commit | awk '{ print $2; }'`
log "Last commit is '$LAST_COMMIT', previous commit is '$PREV_COMMIT'."
if [ "$PREV_COMMIT" == "$LAST_COMMIT" ]; then
log "There were no changes since the last time I checked. Going to sleep."
else
if [ ! -z "$LAST_COMMIT" ]; then
log "New code has been committed or previous commit not known. " \
"Will trigger the tests."
PREV_COMMIT=$LAST_COMMIT
log "Updated previous commit to '$PREV_COMMIT'."
#
# This is where we'll trigger the Sandcastle run. The values for
# HTTPS_APP_VALUE and HTTPS_APP_VALUE will be set in the container we're
# running in.
#
POST_RECEIVE_HOOK=1 php $CONTRUN_DETERMINATOR
error=$?
if [ $error -eq 0 ]; then
log "Sandcastle run successfully triggered."
else
log_err "Failed to trigger Sandcastle run."
fi
else
log_err "Previous commit not updated. Don't know what the last one is."
fi
fi
else
log_err "Getting latest changes failed. Will skip running tests for now."
fi
# Always sleep, even if errors happens while trying to determine the latest
# commit. This will prevent us terminating in case of transient errors.
log "Will go to sleep for 5 minutes."
sleep 5m
done

18
build_tools/dependencies.sh Normal file
View File

@@ -0,0 +1,18 @@
GCC_BASE=/mnt/gvfs/third-party2/gcc/2928bb3ed95bf64f5b388ee88c30dc74710c3b35/5.x/centos6-native/f4950a1
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/a5fea028cb7ba43498976e1f8054b0b2e790c295/stable/centos6-native/6aaf4de
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/7a9099f6587ee4378c0b1fa32bb8934019d30ca4/5.x/gcc-5-glibc-2.23/339d858
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/3b7c6469854dfc7832a1c3cc5b86919a84e5f865/2.23/gcc-5-glibc-2.23/ca1d1c0
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-5-glibc-2.23/9bc6787
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/d7861abe6f0e27ab98c9303b95a662f0e4cdedb5/1.2.8/gcc-5-glibc-2.23/9bc6787
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-5-glibc-2.23/9bc6787
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0815d59804160c96caac5f27ca004f51af893dc6/r131/gcc-5-glibc-2.23/9bc6787
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/c15a4f5f619a2930478d01e2e34dc1e0652b0873/1.1.4/gcc-5-glibc-2.23/03859b5
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f905a5e1032fb30c05db3d3752319857388c0c49/2.2.0/gcc-5-glibc-2.23/9bc6787
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/8d60633d822a2a55849c73db24e74a25e52b71db/master/gcc-5-glibc-2.23/1c32b4b
NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-5-glibc-2.23/9bc6787
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/8db74270cd6d0212ac92d69e7fc7beefe617d772/trunk/gcc-5-glibc-2.23/b1847cb
TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-5-glibc-2.23/9bc6787
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/90c9734afc5579c9d1db529fa788d09f97763b85/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/9e829389ef61b92c62de8748c80169aaf25ce1f0/2.26.1/centos6-native/da39a3e
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.11.0/gcc-5-glibc-2.23/9bc6787
LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/gcc-5-glibc-2.23/65372bd

18
build_tools/dependencies_4.8.1.sh Normal file
View File

@@ -0,0 +1,18 @@
GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/d282e6e8f3d20f4e40a516834847bdc038e07973/2.17/gcc-4.8.1-glibc-2.17/99df8fc
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-4.8.1-glibc-2.17/c3f970a
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-4.8.1-glibc-2.17/c3f970a
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-4.8.1-glibc-2.17/8d31e51
NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/ad576de2a1ea560c4d3434304f0fc4e079bede42/trunk/gcc-4.8.1-glibc-2.17/675d945
TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-4.8.1-glibc-2.17/c3f970a
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-4.8.1-glibc-2.17/da39a3e
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a
LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/centos6-native/730f94e

2
build_tools/dockerbuild.sh Executable file
View File

@@ -0,0 +1,2 @@
#!/usr/bin/env bash
docker run -v $PWD:/rocks -w /rocks buildpack-deps make

167
build_tools/error_filter.py Normal file
View File

@@ -0,0 +1,167 @@
# Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
# This source code is licensed under both the GPLv2 (found in the
# COPYING file in the root directory) and Apache 2.0 License
# (found in the LICENSE.Apache file in the root directory).
'''Filter for error messages in test output:
- Receives merged stdout/stderr from test on stdin
- Finds patterns of known error messages for test name (first argument)
- Prints those error messages to stdout
'''
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import re
import sys
class ErrorParserBase(object):
def parse_error(self, line):
'''Parses a line of test output. If it contains an error, returns a
formatted message describing the error; otherwise, returns None.
Subclasses must override this method.
'''
raise NotImplementedError
class GTestErrorParser(ErrorParserBase):
'''A parser that remembers the last test that began running so it can print
that test's name upon detecting failure.
'''
_GTEST_NAME_PATTERN = re.compile(r'\[ RUN \] (\S+)$')
# format: '<filename or "unknown file">:<line #>: Failure'
_GTEST_FAIL_PATTERN = re.compile(r'(unknown file|\S+:\d+): Failure$')
def __init__(self):
self._last_gtest_name = 'Unknown test'
def parse_error(self, line):
gtest_name_match = self._GTEST_NAME_PATTERN.match(line)
if gtest_name_match:
self._last_gtest_name = gtest_name_match.group(1)
return None
gtest_fail_match = self._GTEST_FAIL_PATTERN.match(line)
if gtest_fail_match:
return '%s failed: %s' % (
self._last_gtest_name, gtest_fail_match.group(1))
return None
class MatchErrorParser(ErrorParserBase):
'''A simple parser that returns the whole line if it matches the pattern.
'''
def __init__(self, pattern):
self._pattern = re.compile(pattern)
def parse_error(self, line):
if self._pattern.match(line):
return line
return None
class CompilerErrorParser(MatchErrorParser):
def __init__(self):
# format: '<filename>:<line #>:<column #>: error: <error msg>'
super(CompilerErrorParser, self).__init__(r'\S+:\d+:\d+: error:')
class ScanBuildErrorParser(MatchErrorParser):
def __init__(self):
super(ScanBuildErrorParser, self).__init__(
r'scan-build: \d+ bugs found.$')
class DbCrashErrorParser(MatchErrorParser):
def __init__(self):
super(DbCrashErrorParser, self).__init__(r'\*\*\*.*\^$|TEST FAILED.')
class WriteStressErrorParser(MatchErrorParser):
def __init__(self):
super(WriteStressErrorParser, self).__init__(
r'ERROR: write_stress died with exitcode=\d+')
class AsanErrorParser(MatchErrorParser):
def __init__(self):
super(AsanErrorParser, self).__init__(
r'==\d+==ERROR: AddressSanitizer:')
class UbsanErrorParser(MatchErrorParser):
def __init__(self):
# format: '<filename>:<line #>:<column #>: runtime error: <error msg>'
super(UbsanErrorParser, self).__init__(r'\S+:\d+:\d+: runtime error:')
class ValgrindErrorParser(MatchErrorParser):
def __init__(self):
# just grab the summary, valgrind doesn't clearly distinguish errors
# from other log messages.
super(ValgrindErrorParser, self).__init__(r'==\d+== ERROR SUMMARY:')
class CompatErrorParser(MatchErrorParser):
def __init__(self):
super(CompatErrorParser, self).__init__(r'==== .*[Ee]rror.* ====$')
class TsanErrorParser(MatchErrorParser):
def __init__(self):
super(TsanErrorParser, self).__init__(r'WARNING: ThreadSanitizer:')
_TEST_NAME_TO_PARSERS = {
'punit': [CompilerErrorParser, GTestErrorParser],
'unit': [CompilerErrorParser, GTestErrorParser],
'release': [CompilerErrorParser, GTestErrorParser],
'unit_481': [CompilerErrorParser, GTestErrorParser],
'release_481': [CompilerErrorParser, GTestErrorParser],
'clang_unit': [CompilerErrorParser, GTestErrorParser],
'clang_release': [CompilerErrorParser, GTestErrorParser],
'clang_analyze': [CompilerErrorParser, ScanBuildErrorParser],
'code_cov': [CompilerErrorParser, GTestErrorParser],
'unity': [CompilerErrorParser, GTestErrorParser],
'lite': [CompilerErrorParser],
'lite_test': [CompilerErrorParser, GTestErrorParser],
'stress_crash': [CompilerErrorParser, DbCrashErrorParser],
'write_stress': [CompilerErrorParser, WriteStressErrorParser],
'asan': [CompilerErrorParser, GTestErrorParser, AsanErrorParser],
'asan_crash': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
'ubsan': [CompilerErrorParser, GTestErrorParser, UbsanErrorParser],
'ubsan_crash': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
'valgrind': [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser],
'tsan': [CompilerErrorParser, GTestErrorParser, TsanErrorParser],
'format_compatible': [CompilerErrorParser, CompatErrorParser],
'run_format_compatible': [CompilerErrorParser, CompatErrorParser],
'no_compression': [CompilerErrorParser, GTestErrorParser],
'run_no_compression': [CompilerErrorParser, GTestErrorParser],
'regression': [CompilerErrorParser],
'run_regression': [CompilerErrorParser],
}
def main():
if len(sys.argv) != 2:
return 'Usage: %s <test name>' % sys.argv[0]
test_name = sys.argv[1]
if test_name not in _TEST_NAME_TO_PARSERS:
return 'Unknown test name: %s' % test_name
error_parsers = []
for parser_cls in _TEST_NAME_TO_PARSERS[test_name]:
error_parsers.append(parser_cls())
for line in sys.stdin:
line = line.strip()
for error_parser in error_parsers:
error_msg = error_parser.parse_error(line)
if error_msg is not None:
print(error_msg)
if __name__ == '__main__':
sys.exit(main())

55
build_tools/fb_compile_mongo.sh Executable file
View File

@@ -0,0 +1,55 @@
#!/bin/sh
# fail early
set -e
if test -z $ROCKSDB_PATH; then
ROCKSDB_PATH=~/rocksdb
fi
source $ROCKSDB_PATH/build_tools/fbcode_config4.8.1.sh
EXTRA_LDFLAGS=""
if test -z $ALLOC; then
# default
ALLOC=tcmalloc
elif [[ $ALLOC == "jemalloc" ]]; then
ALLOC=system
EXTRA_LDFLAGS+=" -Wl,--whole-archive $JEMALLOC_LIB -Wl,--no-whole-archive"
fi
# we need to force mongo to use static library, not shared
STATIC_LIB_DEP_DIR='build/static_library_dependencies'
test -d $STATIC_LIB_DEP_DIR || mkdir $STATIC_LIB_DEP_DIR
test -h $STATIC_LIB_DEP_DIR/`basename $SNAPPY_LIBS` || ln -s $SNAPPY_LIBS $STATIC_LIB_DEP_DIR
test -h $STATIC_LIB_DEP_DIR/`basename $LZ4_LIBS` || ln -s $LZ4_LIBS $STATIC_LIB_DEP_DIR
EXTRA_LDFLAGS+=" -L $STATIC_LIB_DEP_DIR"
set -x
EXTRA_CMD=""
if ! test -e version.json; then
# this is Mongo 3.0
EXTRA_CMD="--rocksdb \
--variant-dir=linux2/norm
--cxx=${CXX} \
--cc=${CC} \
--use-system-zlib" # add this line back to normal code path
# when https://jira.mongodb.org/browse/SERVER-19123 is resolved
fi
scons \
LINKFLAGS="$EXTRA_LDFLAGS $EXEC_LDFLAGS $PLATFORM_LDFLAGS" \
CCFLAGS="$CXXFLAGS -L $STATIC_LIB_DEP_DIR" \
LIBS="lz4 gcc stdc++" \
LIBPATH="$ROCKSDB_PATH" \
CPPPATH="$ROCKSDB_PATH/include" \
-j32 \
--allocator=$ALLOC \
--nostrip \
--opt=on \
--disable-minimum-compiler-version-enforcement \
--use-system-snappy \
--disable-warnings-as-errors \
$EXTRA_CMD $*

74
build_tools/fbcode.clang31.sh
View File

@@ -1,74 +0,0 @@
#!/bin/sh
#
# Set environment variables so that we can compile leveldb using
# fbcode settings. It uses the latest g++ compiler and also
# uses jemalloc
TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3
GLIBC_RUNTIME_PATH=/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1
# location of libgcc
LIBGCC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include"
LIBGCC_LIBS=" -L $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/libs"
# location of glibc
GLIBC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/include"
GLIBC_LIBS=" -L $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib"
# location of snappy headers and libraries
SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include"
SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
# location of zlib headers and libraries
ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/include"
ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/lib/libz.a"
# location of gflags headers and libraries
GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include"
GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a"
# location of bzip headers and libraries
BZIP_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/91ddd43/include"
BZIP_LIBS=" $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/91ddd43/lib/libbz2.a"
# location of gflags headers and libraries
GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include"
GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a"
# use Intel SSE support for checksum calculations
export USE_SSE=" -msse -msse4.2 "
CC="$TOOLCHAIN_EXECUTABLES/clang/clang-3.2/0b7c69d/bin/clang $CLANG_INCLUDES"
CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.2/0b7c69d/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $GFLAGS_INCLUDE"
AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin -nostdlib "
CFLAGS+=" -nostdinc -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1 "
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1/x86_64-facebook-linux "
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1/backward "
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/include "
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include "
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/clang/clang-3.2/0b7c69d/lib/clang/3.2/include "
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/kernel-headers/kernel-headers-3.2.18_70_fbk11_00129_gc8882d0/da39a3e/include/linux "
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/kernel-headers/kernel-headers-3.2.18_70_fbk11_00129_gc8882d0/da39a3e/include "
CFLAGS+=" -Wall -Wno-sign-compare -Wno-unused-variable -Winvalid-pch -Wno-deprecated -Woverloaded-virtual"
CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE"
CXXFLAGS="$CFLAGS -nostdinc++"
CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a"
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $GFLAGS_LIBS"
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,$GLIBC_RUNTIME_PATH/lib/ld-linux-x86-64.so.2"
EXEC_LDFLAGS+=" -B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin"
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS "
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $GFLAGS_LIBS"
export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED

70
build_tools/fbcode.gcc471.sh
View File

@@ -1,70 +0,0 @@
#!/bin/sh
#
# Set environment variables so that we can compile leveldb using
# fbcode settings. It uses the latest g++ compiler and also
# uses jemalloc
TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3
# location of libhdfs libraries
if test "$USE_HDFS"; then
JAVA_HOME="/usr/local/jdk-6u22-64"
JINCLUDE="-I$JAVA_HOME/include -I$JAVA_HOME/include/linux"
GLIBC_RUNTIME_PATH="/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1"
HDFSLIB=" -Wl,--no-whole-archive hdfs/libhdfs.a -L$JAVA_HOME/jre/lib/amd64 "
HDFSLIB+=" -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib "
HDFSLIB+=" -ldl -lverify -ljava -ljvm "
fi
# location of libgcc
LIBGCC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include"
LIBGCC_LIBS=" -L $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/libs"
# location of glibc
GLIBC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/include"
GLIBC_LIBS=" -L $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib"
# location of snappy headers and libraries
SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include"
SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
# location of zlib headers and libraries
ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/include"
ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/lib/libz.a"
# location of bzip headers and libraries
BZIP_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/91ddd43/include"
BZIP_LIBS=" $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/91ddd43/lib/libbz2.a"
# location of gflags headers and libraries
GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include"
GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a"
# use Intel SSE support for checksum calculations
export USE_SSE=" -msse -msse4.2 "
CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/gcc"
CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $GFLAGS_INCLUDE"
AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic"
CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE"
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT"
CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2"
EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a"
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $GFLAGS_LIBS"
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS "
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $GFLAGS_LIBS"
VALGRIND_VER="$TOOLCHAIN_LIB_BASE/valgrind/valgrind-3.8.1/91ddd43/bin/"
export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER

86
build_tools/fbcode.gcc481.sh
View File

@@ -1,86 +0,0 @@
#!/bin/sh
#
# Set environment variables so that we can compile rocksdb using
# fbcode settings. It uses the latest g++ compiler and also
# uses jemalloc
TOOLCHAIN_REV=53dc1fe83f84e9145b9ffb81b81aa7f6a49c87cc
CENTOS_VERSION=`rpm -q --qf "%{VERSION}" $(rpm -q --whatprovides redhat-release)`
if [ "$CENTOS_VERSION" = "6" ]; then
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos6-native"
else
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
fi
TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.8.1-glibc-2.17"
# location of libhdfs libraries
if test "$USE_HDFS"; then
JAVA_HOME="/usr/local/jdk-6u22-64"
JINCLUDE="-I$JAVA_HOME/include -I$JAVA_HOME/include/linux"
GLIBC_RUNTIME_PATH="/usr/local/fbcode/gcc-4.8.1-glibc-2.17"
HDFSLIB=" -Wl,--no-whole-archive hdfs/libhdfs.a -L$JAVA_HOME/jre/lib/amd64 "
HDFSLIB+=" -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib "
HDFSLIB+=" -ldl -lverify -ljava -ljvm "
fi
# location of libgcc
LIBGCC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.8.1/8aac7fc/include"
LIBGCC_LIBS=" -L $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.8.1/8aac7fc/libs"
# location of glibc
GLIBC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/glibc/glibc-2.17/99df8fc/include"
GLIBC_LIBS=" -L $TOOLCHAIN_LIB_BASE/glibc/glibc-2.17/99df8fc/lib"
# location of snappy headers and libraries
SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/43d84e2/include"
SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/43d84e2/lib/libsnappy.a"
# location of zlib headers and libraries
ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/c3f970a/include"
ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/c3f970a/lib/libz.a"
# location of bzip headers and libraries
BZIP_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/c3f970a/include"
BZIP_LIBS=" $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/c3f970a/lib/libbz2.a"
LZ4_REV=065ec7e38fe83329031f6668c43bef83eff5808b
LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/include"
LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/lib/liblz4.a"
# location of gflags headers and libraries
GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/c3f970a/include"
GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/c3f970a/lib/libgflags.a"
# location of jemalloc
JEMALLOC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/jemalloc/jemalloc-3.4.1/4d53c6f/include/"
JEMALLOC_LIB=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/jemalloc-3.4.1/4d53c6f/lib/libjemalloc.a"
# location of numa
NUMA_REV=829d10dac0230f99cd7e1778869d2adf3da24b65
NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/include/"
NUMA_LIB=" /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/lib/libnuma.a"
# use Intel SSE support for checksum calculations
export USE_SSE=" -msse -msse4.2 "
CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.8.1/cc6c9dc/bin/gcc"
CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.8.1/cc6c9dc/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic"
CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE"
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT"
CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DNUMA"
EXEC_LDFLAGS="-Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/675d945/lib/libunwind.a"
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS $NUMA_LIB"
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS "
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS"
VALGRIND_VER="$TOOLCHAIN_LIB_BASE/valgrind/valgrind-3.8.1/c3f970a/bin/"
export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE

156
build_tools/fbcode_config.sh Normal file
View File

@@ -0,0 +1,156 @@
#!/bin/sh
#
# Set environment variables so that we can compile rocksdb using
# fbcode settings. It uses the latest g++ and clang compilers and also
# uses jemalloc
# Environment variables that change the behavior of this script:
# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
BASEDIR=`dirname $BASH_SOURCE`
source "$BASEDIR/dependencies.sh"
CFLAGS=""
# libgcc
LIBGCC_INCLUDE="$LIBGCC_BASE/include"
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
# glibc
GLIBC_INCLUDE="$GLIBC_BASE/include"
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
# snappy
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
if test -z $PIC_BUILD; then
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
else
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
fi
CFLAGS+=" -DSNAPPY"
if test -z $PIC_BUILD; then
# location of zlib headers and libraries
ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
CFLAGS+=" -DZLIB"
# location of bzip headers and libraries
BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
CFLAGS+=" -DBZIP2"
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
CFLAGS+=" -DLZ4"
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
CFLAGS+=" -DZSTD"
fi
# location of gflags headers and libraries
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
if test -z $PIC_BUILD; then
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
else
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
fi
CFLAGS+=" -DGFLAGS=gflags"
# location of jemalloc
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
if test -z $PIC_BUILD; then
# location of numa
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
CFLAGS+=" -DNUMA"
# location of libunwind
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
fi
# location of TBB
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
if test -z $PIC_BUILD; then
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
else
TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
fi
CFLAGS+=" -DTBB"
# use Intel SSE support for checksum calculations
export USE_SSE=1
BINUTILS="$BINUTILS_BASE/bin"
AR="$BINUTILS/ar"
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
STDLIBS="-L $GCC_BASE/lib64"
CLANG_BIN="$CLANG_BASE/bin"
CLANG_LIB="$CLANG_BASE/lib"
CLANG_SRC="$CLANG_BASE/../../src"
CLANG_ANALYZER="$CLANG_BIN/clang++"
CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
if [ -z "$USE_CLANG" ]; then
# gcc
CC="$GCC_BASE/bin/gcc"
CXX="$GCC_BASE/bin/g++"
CFLAGS+=" -B$BINUTILS/gold"
CFLAGS+=" -isystem $GLIBC_INCLUDE"
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
JEMALLOC=1
else
# clang
CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
CC="$CLANG_BIN/clang"
CXX="$CLANG_BIN/clang++"
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x "
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x/x86_64-facebook-linux "
CFLAGS+=" -isystem $GLIBC_INCLUDE"
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
CFLAGS+=" -isystem $CLANG_INCLUDE"
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
CFLAGS+=" -Wno-expansion-to-defined "
CXXFLAGS="-nostdinc++"
fi
CFLAGS+=" $DEPS_INCLUDE"
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
CXXFLAGS+=" $CFLAGS"
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
EXEC_LDFLAGS+=" -B$BINUTILS/gold"
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-5-glibc-2.23/lib/ld.so"
EXEC_LDFLAGS+=" $LIBUNWIND"
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-5-glibc-2.23/lib"
# required by libtbb
EXEC_LDFLAGS+=" -ldl"
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
VALGRIND_VER="$VALGRIND_BASE/bin/"
LUA_PATH="$LUA_BASE"
if test -z $PIC_BUILD; then
LUA_LIB=" $LUA_PATH/lib/liblua.a"
else
LUA_LIB=" $LUA_PATH/lib/liblua_pic.a"
fi
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB

115
build_tools/fbcode_config4.8.1.sh Normal file
View File

@@ -0,0 +1,115 @@
#!/bin/sh
#
# Set environment variables so that we can compile rocksdb using
# fbcode settings. It uses the latest g++ compiler and also
# uses jemalloc
BASEDIR=`dirname $BASH_SOURCE`
source "$BASEDIR/dependencies_4.8.1.sh"
# location of libgcc
LIBGCC_INCLUDE="$LIBGCC_BASE/include"
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
# location of glibc
GLIBC_INCLUDE="$GLIBC_BASE/include"
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
# location of snappy headers and libraries
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include"
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
# location of zlib headers and libraries
ZLIB_INCLUDE=" -I $ZLIB_BASE/include"
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
# location of bzip headers and libraries
BZIP2_INCLUDE=" -I $BZIP2_BASE/include/"
BZIP2_LIBS=" $BZIP2_BASE/lib/libbz2.a"
LZ4_INCLUDE=" -I $LZ4_BASE/include"
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
ZSTD_INCLUDE=" -I $ZSTD_BASE/include"
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
# location of gflags headers and libraries
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
# location of jemalloc
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include"
JEMALLOC_LIB="$JEMALLOC_BASE/lib/libjemalloc.a"
# location of numa
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
# location of libunwind
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
# location of tbb
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
# use Intel SSE support for checksum calculations
export USE_SSE=1
BINUTILS="$BINUTILS_BASE/bin"
AR="$BINUTILS/ar"
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP2_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
STDLIBS="-L $GCC_BASE/lib64"
if [ -z "$USE_CLANG" ]; then
# gcc
CC="$GCC_BASE/bin/gcc"
CXX="$GCC_BASE/bin/g++"
CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic"
CFLAGS+=" -isystem $GLIBC_INCLUDE"
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
JEMALLOC=1
else
# clang
CLANG_BIN="$CLANG_BASE/bin"
CLANG_LIB="$CLANG_BASE/lib"
CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
CC="$CLANG_BIN/clang"
CXX="$CLANG_BIN/clang++"
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include/"
CFLAGS="-B$BINUTILS/gold -nostdinc -nostdlib"
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1 "
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1/x86_64-facebook-linux "
CFLAGS+=" -isystem $GLIBC_INCLUDE"
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
CFLAGS+=" -isystem $CLANG_INCLUDE"
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
CXXFLAGS="-nostdinc++"
fi
CFLAGS+=" $DEPS_INCLUDE"
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB"
CXXFLAGS+=" $CFLAGS"
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
EXEC_LDFLAGS+=" $LIBUNWIND"
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib"
# required by libtbb
EXEC_LDFLAGS+=" -ldl"
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
VALGRIND_VER="$VALGRIND_BASE/bin/"
LUA_PATH="$LUA_BASE"
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE LUA_PATH

31
build_tools/format-diff.sh
View File

@@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# If clang_format_diff.py command is not specfied, we assume we are able to
# access directly without any path.
if [ -z $CLANG_FORMAT_DIFF ]
@@ -9,9 +9,12 @@ fi
# Check clang-format-diff.py
if ! which $CLANG_FORMAT_DIFF &> /dev/null
then
echo "You didn't have clang-format-diff.py available in your computer!"
echo "You can download it by running: "
echo " curl http://goo.gl/iUW1u2"
echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
echo "You can download clang-format-diff.py by running: "
echo " curl --location http://goo.gl/iUW1u2 -o ${CLANG_FORMAT_DIFF}"
echo "You can download clang-format by running: "
echo " brew install clang-format"
echo "Then, move both files (i.e. ${CLANG_FORMAT_DIFF} and clang-format) to some directory within PATH=${PATH}"
exit 128
fi
@@ -50,14 +53,15 @@ fi
set -e
uncommitted_code=`git diff HEAD`
LAST_MASTER=`git merge-base master HEAD`
# If there's no uncommitted changes, we assume user are doing post-commit
# format check, in which case we'll check the modified lines from latest commit.
# Otherwise, we'll check format of the uncommitted code only.
# format check, in which case we'll check the modified lines since last commit
# from master. Otherwise, we'll check format of the uncommitted code only.
if [ -z "$uncommitted_code" ]
then
# Check the format of last commit
diffs=$(git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -p 1)
diffs=$(git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -p 1)
else
# Check the format of uncommitted lines,
diffs=$(git diff -U0 HEAD | $CLANG_FORMAT_DIFF -p 1)
@@ -79,6 +83,12 @@ echo -e "Detect lines that doesn't follow the format rules:\r"
echo "$diffs" |
sed -e "s/\(^-.*$\)/`echo -e \"$COLOR_RED\1$COLOR_END\"`/" |
sed -e "s/\(^+.*$\)/`echo -e \"$COLOR_GREEN\1$COLOR_END\"`/"
if [[ "$OPT" == *"-DTRAVIS"* ]]
then
exit 1
fi
echo -e "Would you like to fix the format automatically (y/n): \c"
# Make sure under any mode, we can read user input.
@@ -91,7 +101,12 @@ then
fi
# Do in-place format adjustment.
git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1
if [ -z "$uncommitted_code" ]
then
git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -i -p 1
else
git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1
fi
echo "Files reformatted!"
# Amend to last commit if user do the post-commit format check

7936
build_tools/gnu_parallel Executable file
View File

File diff suppressed because it is too large Load Diff

25
build_tools/mac-install-gflags.sh
View File

@@ -1,25 +0,0 @@
#!/bin/sh
# Install gflags for mac developers.
set -e
DIR=`mktemp -d /tmp/rocksdb_gflags_XXXX`
cd $DIR
wget https://gflags.googlecode.com/files/gflags-2.0.tar.gz
tar xvfz gflags-2.0.tar.gz
cd gflags-2.0
./configure
make
make install
# Add include/lib path for g++
echo 'export LIBRARY_PATH+=":/usr/local/lib"' >> ~/.bash_profile
echo 'export CPATH+=":/usr/local/include"' >> ~/.bash_profile
echo ""
echo "-----------------------------------------------------------------------------"
echo "| Installation Completed |"
echo "-----------------------------------------------------------------------------"
echo "Please run \`. ~/.bash_profile\` to be able to compile with gflags"

46
build_tools/make_new_version.sh
View File

@@ -1,46 +0,0 @@
#!/bin/bash
# Copyright (c) 2013, Facebook, Inc. All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
set -e
if [ -z "$GIT" ]
then
GIT="git"
fi
# Print out the colored progress info so that it can be brainlessly
# distinguished by users.
function title() {
echo -e "\033[1;32m$*\033[0m"
}
usage="Create new RocksDB version and prepare it for the release process\n"
usage+="USAGE: ./make_new_version.sh <version>"
# -- Pre-check
if [[ $# < 1 ]]; then
echo -e $usage
exit 1
fi
ROCKSDB_VERSION=$1
GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
echo $GIT_BRANCH
if [ $GIT_BRANCH != "master" ]; then
echo "Error: Current branch is '$GIT_BRANCH', Please switch to master branch."
exit 1
fi
title "Adding new tag for this release ..."
BRANCH="$ROCKSDB_VERSION.fb"
$GIT co -b $BRANCH
# Setting up the proxy for remote repo access
title "Pushing new branch to remote repo ..."
git push origin --set-upstream $BRANCH
title "Branch $BRANCH is pushed to github;"

128
build_tools/make_package.sh Executable file
View File

@@ -0,0 +1,128 @@
#/usr/bin/env bash
set -e
function log() {
echo "[+] $1"
}
function fatal() {
echo "[!] $1"
exit 1
}
function platform() {
local __resultvar=$1
if [[ -f "/etc/yum.conf" ]]; then
eval $__resultvar="centos"
elif [[ -f "/etc/dpkg/dpkg.cfg" ]]; then
eval $__resultvar="ubuntu"
else
fatal "Unknwon operating system"
fi
}
platform OS
function package() {
if [[ $OS = "ubuntu" ]]; then
if dpkg --get-selections | grep --quiet $1; then
log "$1 is already installed. skipping."
else
apt-get install $@ -y
fi
elif [[ $OS = "centos" ]]; then
if rpm -qa | grep --quiet $1; then
log "$1 is already installed. skipping."
else
yum install $@ -y
fi
fi
}
function detect_fpm_output() {
if [[ $OS = "ubuntu" ]]; then
export FPM_OUTPUT=deb
elif [[ $OS = "centos" ]]; then
export FPM_OUTPUT=rpm
fi
}
detect_fpm_output
function gem_install() {
if gem list | grep --quiet $1; then
log "$1 is already installed. skipping."
else
gem install $@
fi
}
function main() {
if [[ $# -ne 1 ]]; then
fatal "Usage: $0 <rocksdb_version>"
else
log "using rocksdb version: $1"
fi
if [[ -d /vagrant ]]; then
if [[ $OS = "ubuntu" ]]; then
package g++-4.8
export CXX=g++-4.8
# the deb would depend on libgflags2, but the static lib is the only thing
# installed by make install
package libgflags-dev
package ruby-all-dev
elif [[ $OS = "centos" ]]; then
pushd /etc/yum.repos.d
if [[ ! -f /etc/yum.repos.d/devtools-1.1.repo ]]; then
wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo
fi
package devtoolset-1.1-gcc --enablerepo=testing-1.1-devtools-6
package devtoolset-1.1-gcc-c++ --enablerepo=testing-1.1-devtools-6
export CC=/opt/centos/devtoolset-1.1/root/usr/bin/gcc
export CPP=/opt/centos/devtoolset-1.1/root/usr/bin/cpp
export CXX=/opt/centos/devtoolset-1.1/root/usr/bin/c++
export PATH=$PATH:/opt/centos/devtoolset-1.1/root/usr/bin
popd
if ! rpm -qa | grep --quiet gflags; then
rpm -i https://github.com/schuhschuh/gflags/releases/download/v2.1.0/gflags-devel-2.1.0-1.amd64.rpm
fi
package ruby
package ruby-devel
package rubygems
package rpm-build
fi
fi
gem_install fpm
make static_lib
make install INSTALL_PATH=package
cd package
LIB_DIR=lib
if [[ -z "$ARCH" ]]; then
ARCH=$(getconf LONG_BIT)
fi
if [[ ("$FPM_OUTPUT" = "rpm") && ($ARCH -eq 64) ]]; then
mv lib lib64
LIB_DIR=lib64
fi
fpm \
-s dir \
-t $FPM_OUTPUT \
-n rocksdb \
-v $1 \
--prefix /usr \
--url http://rocksdb.org/ \
-m rocksdb@fb.com \
--license BSD \
--vendor Facebook \
--description "RocksDB is an embeddable persistent key-value store for fast storage." \
include $LIB_DIR
}
main $@

208
build_tools/precommit_checker.py Executable file
View File

@@ -0,0 +1,208 @@
#!/usr/local/fbcode/gcc-4.9-glibc-2.20-fb/bin/python2.7
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import argparse
import commands
import subprocess
import sys
import re
import os
import time
#
# Simple logger
#
class Log:
def __init__(self, filename):
self.filename = filename
self.f = open(self.filename, 'w+', 0)
def caption(self, str):
line = "\n##### %s #####\n" % str
if self.f:
self.f.write("%s \n" % line)
else:
print(line)
def error(self, str):
data = "\n\n##### ERROR ##### %s" % str
if self.f:
self.f.write("%s \n" % data)
else:
print(data)
def log(self, str):
if self.f:
self.f.write("%s \n" % str)
else:
print(str)
#
# Shell Environment
#
class Env(object):
def __init__(self, logfile, tests):
self.tests = tests
self.log = Log(logfile)
def shell(self, cmd, path=os.getcwd()):
if path:
os.chdir(path)
self.log.log("==== shell session ===========================")
self.log.log("%s> %s" % (path, cmd))
status = subprocess.call("cd %s; %s" % (path, cmd), shell=True,
stdout=self.log.f, stderr=self.log.f)
self.log.log("status = %s" % status)
self.log.log("============================================== \n\n")
return status
def GetOutput(self, cmd, path=os.getcwd()):
if path:
os.chdir(path)
self.log.log("==== shell session ===========================")
self.log.log("%s> %s" % (path, cmd))
status, out = commands.getstatusoutput(cmd)
self.log.log("status = %s" % status)
self.log.log("out = %s" % out)
self.log.log("============================================== \n\n")
return status, out
#
# Pre-commit checker
#
class PreCommitChecker(Env):
def __init__(self, args):
Env.__init__(self, args.logfile, args.tests)
self.ignore_failure = args.ignore_failure
#
# Get commands for a given job from the determinator file
#
def get_commands(self, test):
status, out = self.GetOutput(
"RATIO=1 build_tools/rocksdb-lego-determinator %s" % test, ".")
return status, out
#
# Run a specific CI job
#
def run_test(self, test):
self.log.caption("Running test %s locally" % test)
# get commands for the CI job determinator
status, cmds = self.get_commands(test)
if status != 0:
self.log.error("Error getting commands for test %s" % test)
return False
# Parse the JSON to extract the commands to run
cmds = re.findall("'shell':'([^\']*)'", cmds)
if len(cmds) == 0:
self.log.log("No commands found")
return False
# Run commands
for cmd in cmds:
# Replace J=<..> with the local environment variable
if "J" in os.environ:
cmd = cmd.replace("J=1", "J=%s" % os.environ["J"])
cmd = cmd.replace("make ", "make -j%s " % os.environ["J"])
# Run the command
status = self.shell(cmd, ".")
if status != 0:
self.log.error("Error running command %s for test %s"
% (cmd, test))
return False
return True
#
# Run specified CI jobs
#
def run_tests(self):
if not self.tests:
self.log.error("Invalid args. Please provide tests")
return False
self.print_separator()
self.print_row("TEST", "RESULT")
self.print_separator()
result = True
for test in self.tests:
start_time = time.time()
self.print_test(test)
result = self.run_test(test)
elapsed_min = (time.time() - start_time) / 60
if not result:
self.log.error("Error running test %s" % test)
self.print_result("FAIL (%dm)" % elapsed_min)
if not self.ignore_failure:
return False
result = False
else:
self.print_result("PASS (%dm)" % elapsed_min)
self.print_separator()
return result
#
# Print a line
#
def print_separator(self):
print("".ljust(60, "-"))
#
# Print two colums
#
def print_row(self, c0, c1):
print("%s%s" % (c0.ljust(40), c1.ljust(20)))
def print_test(self, test):
print(test.ljust(40), end="")
sys.stdout.flush()
def print_result(self, result):
print(result.ljust(20))
#
# Main
#
parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.')
# --log <logfile>
parser.add_argument('--logfile', default='/tmp/precommit-check.log',
help='Log file. Default is /tmp/precommit-check.log')
# --ignore_failure
parser.add_argument('--ignore_failure', action='store_true', default=False,
help='Stop when an error occurs')
# <test ....>
parser.add_argument('tests', nargs='+',
help='CI test(s) to run. e.g: unit punit asan tsan ubsan')
args = parser.parse_args()
checker = PreCommitChecker(args)
print("Please follow log %s" % checker.log.filename)
if not checker.run_tests():
print("Error running tests. Please check log file %s"
% checker.log.filename)
sys.exit(1)
sys.exit(0)

53
build_tools/regression_build_test.sh
View File

@@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -e
@@ -55,7 +55,6 @@ make release
--open_files=55000 \
--statistics=1 \
--histogram=1 \
--disable_data_sync=1 \
--disable_wal=1 \
--sync=0 > ${STAT_FILE}.fillseq
@@ -73,7 +72,6 @@ make release
--open_files=55000 \
--statistics=1 \
--histogram=1 \
--disable_data_sync=1 \
--disable_wal=1 \
--sync=0 \
--threads=8 > ${STAT_FILE}.overwrite
@@ -92,7 +90,6 @@ make release
--open_files=55000 \
--statistics=1 \
--histogram=1 \
--disable_data_sync=1 \
--disable_wal=1 \
--sync=0 \
--threads=1 > /dev/null
@@ -111,7 +108,6 @@ make release
--open_files=55000 \
--statistics=1 \
--histogram=1 \
--disable_data_sync=1 \
--disable_wal=1 \
--sync=0 \
--threads=16 > ${STAT_FILE}.readrandom
@@ -131,7 +127,6 @@ make release
--use_tailing_iterator=1 \
--statistics=1 \
--histogram=1 \
--disable_data_sync=1 \
--disable_wal=1 \
--sync=0 \
--threads=16 > ${STAT_FILE}.readrandomtailing
@@ -150,7 +145,6 @@ make release
--open_files=55000 \
--statistics=1 \
--histogram=1 \
--disable_data_sync=1 \
--disable_wal=1 \
--sync=0 \
--threads=16 > ${STAT_FILE}.readrandomsmallblockcache
@@ -171,7 +165,6 @@ make release
--open_files=55000 \
--statistics=1 \
--histogram=1 \
--disable_data_sync=1 \
--disable_wal=1 \
--sync=0 \
--threads=16 > ${STAT_FILE}.readrandom_mem_sst
@@ -191,7 +184,6 @@ make release
--open_files=55000 \
--statistics=1 \
--histogram=1 \
--disable_data_sync=1 \
--disable_wal=1 \
--sync=0 \
--threads=1 > /dev/null
@@ -210,7 +202,6 @@ make release
--open_files=55000 \
--statistics=1 \
--histogram=1 \
--disable_data_sync=1 \
--disable_wal=1 \
--sync=0 \
--threads=16 > /dev/null
@@ -230,7 +221,6 @@ make release
--disable_auto_compactions=1 \
--statistics=1 \
--histogram=1 \
--disable_data_sync=1 \
--disable_wal=1 \
--sync=0 \
--threads=16 > ${STAT_FILE}.readrandom_filluniquerandom
@@ -243,7 +233,7 @@ make release
--bloom_bits=10 \
--num=$((NUM / 4)) \
--reads=$((NUM / 4)) \
--writes_per_second=1000 \
--benchmark_write_rate_limit=$(( 110 * 1024 )) \
--write_buffer_size=100000000 \
--cache_size=6442450944 \
--cache_numshardbits=6 \
@@ -251,7 +241,6 @@ make release
--open_files=55000 \
--statistics=1 \
--histogram=1 \
--disable_data_sync=1 \
--disable_wal=1 \
--sync=0 \
--threads=16 > ${STAT_FILE}.readwhilewriting
@@ -270,7 +259,6 @@ make release
--open_files=55000 \
--statistics=1 \
--histogram=1 \
--disable_data_sync=1 \
--disable_wal=1 \
--sync=0 \
--value_size=10 \
@@ -295,7 +283,6 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
--disable_wal=0 \
--wal_dir=/dev/shm/rocksdb \
--sync=0 \
--disable_data_sync=1 \
--verify_checksum=1 \
--delete_obsolete_files_period_micros=314572800 \
--max_grandparent_overlap_factor=10 \
@@ -329,7 +316,7 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
--use_existing_db=1 \
--duration=600 \
--threads=32 \
--writes_per_second=81920 > ${STAT_FILE}.readwhilewriting_in_ram
--benchmark_write_rate_limit=9502720 > ${STAT_FILE}.readwhilewriting_in_ram
# Seekrandomwhilewriting
./db_bench \
@@ -342,8 +329,38 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
--use_tailing_iterator=1 \
--duration=600 \
--threads=32 \
--writes_per_second=81920 > ${STAT_FILE}.seekwhilewriting_in_ram
--benchmark_write_rate_limit=9502720 > ${STAT_FILE}.seekwhilewriting_in_ram
# measure fillseq with bunch of column families
./db_bench \
--benchmarks=fillseq \
--num_column_families=500 \
--write_buffer_size=1048576 \
--db=$DATA_DIR \
--use_existing_db=0 \
--num=$NUM \
--writes=$NUM \
--open_files=55000 \
--statistics=1 \
--histogram=1 \
--disable_wal=1 \
--sync=0 > ${STAT_FILE}.fillseq_lots_column_families
# measure overwrite performance with bunch of column families
./db_bench \
--benchmarks=overwrite \
--num_column_families=500 \
--write_buffer_size=1048576 \
--db=$DATA_DIR \
--use_existing_db=1 \
--num=$NUM \
--writes=$((NUM / 10)) \
--open_files=55000 \
--statistics=1 \
--histogram=1 \
--disable_wal=1 \
--sync=0 \
--threads=8 > ${STAT_FILE}.overwrite_lots_column_families
# send data to ods
function send_to_ods {
@@ -392,3 +409,5 @@ send_benchmark_to_ods readrandom memtablereadrandom $STAT_FILE.memtablefillreadr
send_benchmark_to_ods readwhilewriting readwhilewriting $STAT_FILE.readwhilewriting
send_benchmark_to_ods readwhilewriting readwhilewriting_in_ram ${STAT_FILE}.readwhilewriting_in_ram
send_benchmark_to_ods seekrandomwhilewriting seekwhilewriting_in_ram ${STAT_FILE}.seekwhilewriting_in_ram
send_benchmark_to_ods fillseq fillseq_lots_column_families ${STAT_FILE}.fillseq_lots_column_families
send_benchmark_to_ods overwrite overwrite_lots_column_families ${STAT_FILE}.overwrite_lots_column_families

782
build_tools/rocksdb-lego-determinator Executable file
View File

@@ -0,0 +1,782 @@
#!/usr/bin/env bash
# This script is executed by Sandcastle
# to determine next steps to run
# Usage:
# EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> rocks_ci.py <test-name>
#
# Input Value
# -------------------------------------------------------------------------
# EMAIL Email address to report on trigger conditions
# ONCALL Email address to raise a task on failure
# TRIGGER Trigger conditions for email. Valid values are fail, warn, all
# SUBSCRIBER Email addresss to add as subscriber for task
#
#
# Report configuration
#
REPORT_EMAIL=
if [ ! -z $EMAIL ]; then
if [ -z $TRIGGER ]; then
TRIGGER="fail"
fi
REPORT_EMAIL="
{
'type':'email',
'triggers': [ '$TRIGGER' ],
'emails':['$EMAIL']
},"
fi
CREATE_TASK=
if [ ! -z $ONCALL ]; then
CREATE_TASK="
{
'type':'task',
'triggers':[ 'fail' ],
'priority':0,
'subscribers':[ '$SUBSCRIBER' ],
'tags':[ 'rocksdb', 'ci' ],
},"
fi
# For now, create the tasks using only the dedicated task creation tool.
CREATE_TASK=
REPORT=
if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then
REPORT="'report': [
$REPORT_EMAIL
$CREATE_TASK
]"
fi
#
# Helper variables
#
CLEANUP_ENV="
{
'name':'Cleanup environment',
'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true) && make clean',
'user':'root'
}"
# We will eventually set the RATIO to 1, but we want do this
# in steps. RATIO=$(nproc) will make it work as J=1
if [ -z $RATIO ]; then
RATIO=$(nproc)
fi
if [ -z $PARALLEL_J ]; then
PARALLEL_J="J=$(expr $(nproc) / ${RATIO})"
fi
if [ -z $PARALLEL_j ]; then
PARALLEL_j="-j$(expr $(nproc) / ${RATIO})"
fi
PARALLELISM="$PARALLEL_J $PARALLEL_j"
DEBUG="OPT=-g"
SHM="TEST_TMPDIR=/dev/shm/rocksdb"
NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
ASAN="COMPILE_WITH_ASAN=1"
CLANG="USE_CLANG=1"
LITE="OPT=\"-DROCKSDB_LITE -g\""
TSAN="COMPILE_WITH_TSAN=1"
UBSAN="COMPILE_WITH_UBSAN=1"
DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH"
PARSER="'parser':'python build_tools/error_filter.py $1'"
CONTRUN_NAME="ROCKSDB_CONTRUN_NAME"
# This code is getting called under various scenarios. What we care about is to
# understand when it's called from nightly contruns because in that case we'll
# create tasks for any failures. To follow the existing pattern, we'll check
# the value of $ONCALL. If it's a diff then just call `false` to make sure
# that errors will be properly propagated to the caller.
if [ ! -z $ONCALL ]; then
TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb --oncall $ONCALL"
else
TASK_CREATION_TOOL="false"
fi
ARTIFACTS=" 'artifacts': [
{
'name':'database',
'paths':[ '/dev/shm/rocksdb' ],
}
]"
#
# A mechanism to disable tests temporarily
#
DISABLE_COMMANDS="[
{
'name':'Disable test',
'oncall':'$ONCALL',
'steps': [
{
'name':'Job disabled. Please contact test owner',
'shell':'exit 1',
'user':'root'
},
],
}
]"
#
# RocksDB unit test
#
UNIT_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB unit test not under /dev/shm
#
UNIT_TEST_NON_SHM_COMMANDS="[
{
'name':'Rocksdb Unit Test',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'timeout': 86400,
'shell':'$NON_SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB release build and unit tests
#
RELEASE_BUILD_COMMANDS="[
{
'name':'Rocksdb Release Build',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build RocksDB release',
'shell':'make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB unit test on gcc-4.8.1
#
UNIT_TEST_COMMANDS_481="[
{
'name':'Rocksdb Unit Test on GCC 4.8.1',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB release build and unit tests
#
RELEASE_BUILD_COMMANDS_481="[
{
'name':'Rocksdb Release on GCC 4.8.1',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build RocksDB release on GCC 4.8.1',
'shell':'$GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB unit test with CLANG
#
CLANG_UNIT_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug',
'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB release build with CLANG
#
CLANG_RELEASE_BUILD_COMMANDS="[
{
'name':'Rocksdb CLANG Release Build',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build RocksDB release',
'shell':'$CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB analyze
#
CLANG_ANALYZE_COMMANDS="[
{
'name':'Rocksdb analyze',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'RocksDB build and analyze',
'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB code coverage
#
CODE_COV_COMMANDS="[
{
'name':'Rocksdb Unit Test Code Coverage',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build, test and collect code coverage info',
'shell':'$SHM $DEBUG make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB unity
#
UNITY_COMMANDS="[
{
'name':'Rocksdb Unity',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build, test unity test',
'shell':'$SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# Build RocksDB lite
#
LITE_BUILD_COMMANDS="[
{
'name':'Rocksdb Lite build',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build RocksDB debug version',
'shell':'$LITE make J=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB stress/crash test
#
STRESS_CRASH_TEST_COMMANDS="[
{
'name':'Rocksdb Stress/Crash Test',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug stress tests',
'shell':'$SHM $DEBUG make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
{
'name':'Build and run RocksDB debug crash tests',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
}
],
$ARTIFACTS,
$REPORT
}
]"
# RocksDB write stress test.
# We run on disk device on purpose (i.e. no $SHM)
# because we want to add some randomness to fsync commands
WRITE_STRESS_COMMANDS="[
{
'name':'Rocksdb Write Stress Test',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB write stress tests',
'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL',
'user':'root',
$PARSER
}
],
'artifacts': [{'name': 'database', 'paths': ['/tmp/rocksdb_write_stress']}],
$REPORT
}
]"
#
# RocksDB test under address sanitizer
#
ASAN_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test under ASAN',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Test RocksDB debug under ASAN',
'shell':'set -o pipefail && ($SHM $ASAN $DEBUG make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
'user':'root',
$PARSER
}
],
$REPORT
}
]"
#
# RocksDB crash testing under address sanitizer
#
ASAN_CRASH_TEST_COMMANDS="[
{
'name':'Rocksdb crash test under ASAN',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug asan_crash_test',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB test under undefined behavior sanitizer
#
UBSAN_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test under UBSAN',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Test RocksDB debug under UBSAN',
'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
}
],
$REPORT
}
]"
#
# RocksDB crash testing under udnefined behavior sanitizer
#
UBSAN_CRASH_TEST_COMMANDS="[
{
'name':'Rocksdb crash test under UBSAN',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug ubsan_crash_test',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB unit test under valgrind
#
VALGRIND_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test under valgrind',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit tests',
'timeout': 86400,
'shell':'$SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB test under TSAN
#
TSAN_UNIT_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test under TSAN',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'timeout': 86400,
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB crash test under TSAN
#
TSAN_CRASH_TEST_COMMANDS="[
{
'name':'Rocksdb Crash Test under TSAN',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Compile and run',
'timeout': 86400,
'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 CRASH_TEST_EXT_ARGS=--log2_keys_per_lock=22 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB format compatible
#
run_format_compatible()
{
export TEST_TMPDIR=/dev/shm/rocksdb
rm -rf /dev/shm/rocksdb
mkdir /dev/shm/rocksdb
tools/check_format_compatible.sh
}
FORMAT_COMPATIBLE_COMMANDS="[
{
'name':'Rocksdb Format Compatible tests',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'shell':'build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB no compression
#
run_no_compression()
{
export TEST_TMPDIR=/dev/shm/rocksdb
rm -rf /dev/shm/rocksdb
mkdir /dev/shm/rocksdb
make clean
cat build_tools/fbcode_config.sh | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh
mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh
cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile
mv .tmp.Makefile Makefile
make $DEBUG J=1 check
}
NO_COMPRESSION_COMMANDS="[
{
'name':'Rocksdb No Compression tests',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'shell':'build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB regression
#
run_regression()
{
time -v bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKSPACE/leveldb.XXXX) $(mktemp leveldb_test_stats.XXXX)
# ======= report size to ODS ========
# parameters: $1 -- key, $2 -- value
function send_size_to_ods {
curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \
--connect-timeout 60
}
# === normal build ===
make clean
make -j$(nproc) static_lib
send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a)
strip librocksdb.a
send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a)
make -j$(nproc) shared_lib
send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`)
strip `readlink -f librocksdb.so`
send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
# === lite build ===
make clean
OPT=-DROCKSDB_LITE make -j$(nproc) static_lib
send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a)
strip librocksdb.a
send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a)
OPT=-DROCKSDB_LITE make -j$(nproc) shared_lib
send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`)
strip `readlink -f librocksdb.so`
send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
}
REGRESSION_COMMANDS="[
{
'name':'Rocksdb regression commands',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Make and run script',
'shell':'build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB Java build
#
JAVA_BUILD_TEST_COMMANDS="[
{
'name':'Rocksdb Java Build',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build RocksDB for Java',
'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
case $1 in
unit)
echo $UNIT_TEST_COMMANDS
;;
unit_non_shm)
echo $UNIT_TEST_NON_SHM_COMMANDS
;;
release)
echo $RELEASE_BUILD_COMMANDS
;;
unit_481)
echo $UNIT_TEST_COMMANDS_481
;;
release_481)
echo $RELEASE_BUILD_COMMANDS_481
;;
clang_unit)
echo $CLANG_UNIT_TEST_COMMANDS
;;
clang_release)
echo $CLANG_RELEASE_BUILD_COMMANDS
;;
clang_analyze)
echo $CLANG_ANALYZE_COMMANDS
;;
code_cov)
echo $CODE_COV_COMMANDS
;;
unity)
echo $UNITY_COMMANDS
;;
lite)
echo $LITE_BUILD_COMMANDS
;;
stress_crash)
echo $STRESS_CRASH_TEST_COMMANDS
;;
write_stress)
echo $WRITE_STRESS_COMMANDS
;;
asan)
echo $ASAN_TEST_COMMANDS
;;
asan_crash)
echo $ASAN_CRASH_TEST_COMMANDS
;;
ubsan)
echo $UBSAN_TEST_COMMANDS
;;
ubsan_crash)
echo $UBSAN_CRASH_TEST_COMMANDS
;;
valgrind)
echo $VALGRIND_TEST_COMMANDS
;;
tsan)
echo $TSAN_UNIT_TEST_COMMANDS
;;
tsan_crash)
echo $TSAN_CRASH_TEST_COMMANDS
;;
format_compatible)
echo $FORMAT_COMPATIBLE_COMMANDS
;;
run_format_compatible)
run_format_compatible
;;
no_compression)
echo $NO_COMPRESSION_COMMANDS
;;
run_no_compression)
run_no_compression
;;
regression)
echo $REGRESSION_COMMANDS
;;
run_regression)
run_regression
;;
java_build)
echo $JAVA_BUILD_TEST_COMMANDS
;;
*)
echo "Invalid determinator command"
;;
esac

456
build_tools/run_ci_db_test.ps1 Normal file
View File

@@ -0,0 +1,456 @@
# This script enables you running RocksDB tests by running
# All the tests concurrently and utilizing all the cores
Param(
[switch]$EnableJE = $false, # Look for and use _je executable, append _je to listed exclusions
[switch]$RunAll = $false, # Will attempt discover all *_test[_je].exe binaries and run all
# of them as Google suites. I.e. It will run test cases concurrently
# except those mentioned as $Run, those will run as individual test cases
# And any execlued with $ExcludeExes or $ExcludeCases
# It will also not run any individual test cases
# excluded but $ExcludeCasese
[string]$SuiteRun = "", # Split test suites in test cases and run in parallel, not compatible with $RunAll
[string]$Run = "", # Run specified executables in parallel but do not split to test cases
[string]$ExcludeCases = "", # Exclude test cases, expects a comma separated list, no spaces
# Takes effect when $RunAll or $SuiteRun is specified. Must have full
# Test cases name including a group and a parameter if any
[string]$ExcludeExes = "", # Exclude exes from consideration, expects a comma separated list,
# no spaces. Takes effect only when $RunAll is specified
[string]$WorkFolder = "", # Direct tests to use that folder. SSD or Ram drive are better options.
# Number of async tasks that would run concurrently. Recommend a number below 64.
# However, CPU utlization really depends on the storage media. Recommend ram based disk.
# a value of 1 will run everything serially
[int]$Concurrency = 8,
[int]$Limit = -1 # -1 means do not limit for test purposes
)
# Folders and commands must be fullpath to run assuming
# the current folder is at the root of the git enlistment
$StartDate = (Get-Date)
$StartDate
$DebugPreference = "Continue"
# These tests are not google test suites and we should guard
# Against running them as suites
$RunOnly = New-Object System.Collections.Generic.HashSet[string]
$RunOnly.Add("c_test") | Out-Null
$RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
$RunOnly.Add("merge_test") | Out-Null
$RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
$RunOnly.Add("backupable_db_test") | Out-Null # Disabled
if($RunAll -and $SuiteRun -ne "") {
Write-Error "$RunAll and $SuiteRun are not compatible"
exit 1
}
# If running under Appveyor assume that root
[string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
if($Appveyor -ne "") {
$RootFolder = $Appveyor
} else {
$RootFolder = $PSScriptRoot -replace '\\build_tools', ''
}
$LogFolder = -Join($RootFolder, "\db_logs\")
$BinariesFolder = -Join($RootFolder, "\build\Debug\")
if($WorkFolder -eq "") {
# If TEST_TMPDIR is set use it
[string]$var = $Env:TEST_TMPDIR
if($var -eq "") {
$WorkFolder = -Join($RootFolder, "\db_tests\")
$Env:TEST_TMPDIR = $WorkFolder
} else {
$WorkFolder = $var
}
} else {
# Override from a command line
$Env:TEST_TMPDIR = $WorkFolder
}
Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
Write-Output "BinariesFolder: $BinariesFolder, LogFolder: $LogFolder"
# Create test directories in the current folder
md -Path $WorkFolder -ErrorAction Ignore | Out-Null
md -Path $LogFolder -ErrorAction Ignore | Out-Null
$ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string]
if($ExcludeCases -ne "") {
Write-Host "ExcludeCases: $ExcludeCases"
$l = $ExcludeCases -split ' '
ForEach($t in $l) {
$ExcludeCasesSet.Add($t) | Out-Null
}
}
$ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string]
if($ExcludeExes -ne "") {
Write-Host "ExcludeExe: $ExcludeExes"
$l = $ExcludeExes -split ' '
ForEach($t in $l) {
$ExcludeExesSet.Add($t) | Out-Null
}
}
# Extract the names of its tests by running db_test with --gtest_list_tests.
# This filter removes the "#"-introduced comments, and expands to
# fully-qualified names by changing input like this:
#
# DBTest.
# Empty
# WriteEmptyBatch
# MultiThreaded/MultiThreadedDBTest.
# MultiThreaded/0 # GetParam() = 0
# MultiThreaded/1 # GetParam() = 1
#
# into this:
#
# DBTest.Empty
# DBTest.WriteEmptyBatch
# MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
# MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
#
# Output into the parameter in a form TestName -> Log File Name
function ExtractTestCases([string]$GTestExe, $HashTable) {
$Tests = @()
# Run db_test to get a list of tests and store it into $a array
&$GTestExe --gtest_list_tests | tee -Variable Tests | Out-Null
# Current group
$Group=""
ForEach( $l in $Tests) {
# Leading whitespace is fine
$l = $l -replace '^\s+',''
# but no whitespace any other place
if($l -match "\s+") {
continue
}
# Trailing dot is a test group but no whitespace
elseif ( $l -match "\.$" ) {
$Group = $l
} else {
# Otherwise it is a test name, remove leading space
$test = $l
# remove trailing comment if any and create a log name
$test = $test -replace '\s+\#.*',''
$test = "$Group$test"
if($ExcludeCasesSet.Contains($test)) {
Write-Warning "$test case is excluded"
continue
}
$test_log = $test -replace '[\./]','_'
$test_log += ".log"
$log_path = -join ($LogFolder, $test_log)
# Add to a hashtable
$HashTable.Add($test, $log_path);
}
}
}
# The function removes trailing .exe siffix if any,
# creates a name for the log file
# Then adds the test name if it was not excluded into
# a HashTable in a form of test_name -> log_path
function MakeAndAdd([string]$token, $HashTable) {
$test_name = $token -replace '.exe$', ''
$log_name = -join ($test_name, ".log")
$log_path = -join ($LogFolder, $log_name)
$HashTable.Add($test_name, $log_path)
}
# This function takes a list of Suites to run
# Lists all the test cases in each of the suite
# and populates HashOfHashes
# Ordered by suite(exe) @{ Exe = @{ TestCase = LogName }}
function ProcessSuites($ListOfSuites, $HashOfHashes) {
$suite_list = $ListOfSuites
# Problem: if you run --gtest_list_tests on
# a non Google Test executable then it will start executing
# and we will get nowhere
ForEach($suite in $suite_list) {
if($RunOnly.Contains($suite)) {
Write-Warning "$suite is excluded from running as Google test suite"
continue
}
if($EnableJE) {
$suite += "_je"
}
$Cases = [ordered]@{}
$Cases.Clear()
$suite_exe = -Join ($BinariesFolder, $suite)
ExtractTestCases -GTestExe $suite_exe -HashTable $Cases
if($Cases.Count -gt 0) {
$HashOfHashes.Add($suite, $Cases);
}
}
# Make logs and run
if($CasesToRun.Count -lt 1) {
Write-Error "Failed to extract tests from $SuiteRun"
exit 1
}
}
# This will contain all test executables to run
# Hash table that contains all non suite
# Test executable to run
$TestExes = [ordered]@{}
# Check for test exe that are not
# Google Test Suites
# Since this is explicitely mentioned it is not subject
# for exclusions
if($Run -ne "") {
$test_list = $Run -split ' '
ForEach($t in $test_list) {
if($EnableJE) {
$t += "_je"
}
MakeAndAdd -token $t -HashTable $TestExes
}
if($TestExes.Count -lt 1) {
Write-Error "Failed to extract tests from $Run"
exit 1
}
}
# Ordered by exe @{ Exe = @{ TestCase = LogName }}
$CasesToRun = [ordered]@{}
if($SuiteRun -ne "") {
$suite_list = $SuiteRun -split ' '
ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun
}
if($RunAll) {
# Discover all the test binaries
if($EnableJE) {
$pattern = "*_test_je.exe"
} else {
$pattern = "*_test.exe"
}
$search_path = -join ($BinariesFolder, $pattern)
Write-Host "Binaries Search Path: $search_path"
$ListOfExe = @()
dir -Path $search_path | ForEach-Object {
$ListOfExe += ($_.Name)
}
# Exclude those in RunOnly from running as suites
$ListOfSuites = @()
ForEach($e in $ListOfExe) {
$e = $e -replace '.exe$', ''
$bare_name = $e -replace '_je$', ''
if($ExcludeExesSet.Contains($bare_name)) {
Write-Warning "Test $e is excluded"
continue
}
if($RunOnly.Contains($bare_name)) {
MakeAndAdd -token $e -HashTable $TestExes
} else {
$ListOfSuites += $bare_name
}
}
ProcessSuites -ListOfSuites $ListOfSuites -HashOfHashes $CasesToRun
}
Write-Host "Attempting to start: $NumTestsToStart tests"
# Invoke a test with a filter and redirect all output
$InvokeTestCase = {
param($exe, $test, $log);
&$exe --gtest_filter=$test > $log 2>&1
}
# Invoke all tests and redirect output
$InvokeTestAsync = {
param($exe, $log)
&$exe > $log 2>&1
}
# Hash that contains tests to rerun if any failed
# Those tests will be rerun sequentially
# $Rerun = [ordered]@{}
# Test limiting factor here
[int]$count = 0
# Overall status
[bool]$success = $true;
function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
{
# Array to wait for any of the running jobs
$jobs = @()
# Hash JobToLog
$JobToLog = @{}
# Wait for all to finish and get the results
while(($JobToLog.Count -gt 0) -or
($TestCmds.Count -gt 0) -or
($Suites.Count -gt 0)) {
# Make sure we have maximum concurrent jobs running if anything
# and the $Limit either not set or allows to proceed
while(($JobToLog.Count -lt $ConcurrencyVal) -and
((($TestCmds.Count -gt 0) -or ($Suites.Count -gt 0)) -and
(($Limit -lt 0) -or ($count -lt $Limit)))) {
# We always favore suites to run if available
[string]$exe_name = ""
[string]$log_path = ""
$Cases = @{}
if($Suites.Count -gt 0) {
# Will the first one
ForEach($e in $Suites.Keys) {
$exe_name = $e
$Cases = $Suites[$e]
break
}
[string]$test_case = ""
[string]$log_path = ""
ForEach($c in $Cases.Keys) {
$test_case = $c
$log_path = $Cases[$c]
break
}
Write-Host "Starting $exe_name::$test_case"
[string]$Exe = -Join ($BinariesFolder, $exe_name)
$job = Start-Job -Name "$exe_name::$test_case" -ArgumentList @($Exe,$test_case,$log_path) -ScriptBlock $InvokeTestCase
$JobToLog.Add($job, $log_path)
$Cases.Remove($test_case)
if($Cases.Count -lt 1) {
$Suites.Remove($exe_name)
}
} elseif ($TestCmds.Count -gt 0) {
ForEach($e in $TestCmds.Keys) {
$exe_name = $e
$log_path = $TestCmds[$e]
break
}
[string]$Exe = -Join ($BinariesFolder, $exe_name)
$job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path)
$JobToLog.Add($job, $log_path)
$TestCmds.Remove($exe_name)
} else {
Write-Error "In the job loop but nothing to run"
exit 1
}
++$count
} # End of Job starting loop
if($JobToLog.Count -lt 1) {
break
}
$jobs = @()
foreach($k in $JobToLog.Keys) { $jobs += $k }
$completed = Wait-Job -Job $jobs -Any
$log = $JobToLog[$completed]
$JobToLog.Remove($completed)
$message = -join @($completed.Name, " State: ", ($completed.State))
$log_content = @(Get-Content $log)
if($completed.State -ne "Completed") {
$success = $false
Write-Warning $message
$log_content | Write-Warning
} else {
# Scan the log. If we find PASSED and no occurrence of FAILED
# then it is a success
[bool]$pass_found = $false
ForEach($l in $log_content) {
if(($l -match "^\[\s+FAILED") -or
($l -match "Assertion failed:")) {
$pass_found = $false
break
}
if(($l -match "^\[\s+PASSED") -or
($l -match " : PASSED$") -or
($l -match "^PASS$") -or # Special c_test case
($l -match "Passed all tests!") ) {
$pass_found = $true
}
}
if(!$pass_found) {
$success = $false;
Write-Warning $message
$log_content | Write-Warning
} else {
Write-Host $message
}
}
# Remove cached job info from the system
# Should be no output
Receive-Job -Job $completed | Out-Null
}
}
RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency
$EndDate = (Get-Date)
New-TimeSpan -Start $StartDate -End $EndDate |
ForEach-Object {
"Elapsed time: {0:g}" -f $_
}
if(!$success) {
# This does not succeed killing off jobs quick
# So we simply exit
# Remove-Job -Job $jobs -Force
# indicate failure using this exit code
exit 1
}
exit 0

78
build_tools/unity
View File

@@ -1,78 +0,0 @@
#!/bin/sh
#
# Create the unity file
#
OUTPUT=$1
if test -z "$OUTPUT"; then
echo "usage: $0 <output-filename>" >&2
exit 1
fi
# Delete existing file, if it exists
rm -f "$OUTPUT"
touch "$OUTPUT"
# Detect OS
if test -z "$TARGET_OS"; then
TARGET_OS=`uname -s`
fi
# generic port files (working on all platform by #ifdef) go directly in /port
GENERIC_PORT_FILES=`cd "$ROCKSDB_ROOT"; find port -name '*.cc' | tr "\n" " "`
# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
case "$TARGET_OS" in
Darwin)
# PORT_FILES=port/darwin/darwin_specific.cc
;;
IOS)
;;
Linux)
# PORT_FILES=port/linux/linux_specific.cc
;;
SunOS)
# PORT_FILES=port/sunos/sunos_specific.cc
;;
FreeBSD)
# PORT_FILES=port/freebsd/freebsd_specific.cc
;;
NetBSD)
# PORT_FILES=port/netbsd/netbsd_specific.cc
;;
OpenBSD)
# PORT_FILES=port/openbsd/openbsd_specific.cc
;;
DragonFly)
# PORT_FILES=port/dragonfly/dragonfly_specific.cc
;;
OS_ANDROID_CROSSCOMPILE)
# PORT_FILES=port/android/android.cc
;;
*)
echo "Unknown platform!" >&2
exit 1
esac
# We want to make a list of all cc files within util, db, table, and helpers
# except for the test and benchmark files. By default, find will output a list
# of all files matching either rule, so we need to append -print to make the
# prune take effect.
DIRS="util db table utilities"
set -f # temporarily disable globbing so that our patterns arent expanded
PRUNE_TEST="-name *test*.cc -prune"
PRUNE_BENCH="-name *bench*.cc -prune"
PORTABLE_FILES=`cd "$ROCKSDB_ROOT"; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort`
PORTABLE_CPP=`cd "$ROCKSDB_ROOT"; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort`
set +f # re-enable globbing
# The sources consist of the portable files, plus the platform-specific port
# file.
for SOURCE_FILE in $PORTABLE_FILES $GENERIC_PORT_FILES $PORT_FILES $PORTABLE_CPP
do
echo "#include <$SOURCE_FILE>" >> "$OUTPUT"
done
echo "int main(int argc, char** argv){ return 0; }" >> "$OUTPUT"

131
build_tools/update_dependencies.sh Executable file
View File

@@ -0,0 +1,131 @@
#!/bin/sh
#
# Update dependencies.sh file with the latest avaliable versions
BASEDIR=$(dirname $0)
OUTPUT=""
function log_variable()
{
echo "$1=${!1}" >> "$OUTPUT"
}
TP2_LATEST="/mnt/vol/engshare/fbcode/third-party2"
## $1 => lib name
## $2 => lib version (if not provided, will try to pick latest)
## $3 => platform (if not provided, will try to pick latest gcc)
##
## get_lib_base will set a variable named ${LIB_NAME}_BASE to the lib location
function get_lib_base()
{
local lib_name=$1
local lib_version=$2
local lib_platform=$3
local result="$TP2_LATEST/$lib_name/"
# Lib Version
if [ -z "$lib_version" ] || [ "$lib_version" = "LATEST" ]; then
# version is not provided, use latest
result=`ls -dr1v $result/*/ | head -n1`
else
result="$result/$lib_version/"
fi
# Lib Platform
if [ -z "$lib_platform" ]; then
# platform is not provided, use latest gcc
result=`ls -dr1v $result/gcc-*[^fb]/ | head -n1`
else
result="$result/$lib_platform/"
fi
result=`ls -1d $result/*/ | head -n1`
# lib_name => LIB_NAME_BASE
local __res_var=${lib_name^^}"_BASE"
__res_var=`echo $__res_var | tr - _`
# LIB_NAME_BASE=$result
eval $__res_var=`readlink -f $result`
log_variable $__res_var
}
###########################################################
# 5.x dependencies #
###########################################################
OUTPUT="$BASEDIR/dependencies.sh"
rm -f "$OUTPUT"
touch "$OUTPUT"
echo "Writing dependencies to $OUTPUT"
# Compilers locations
GCC_BASE=`readlink -f $TP2_LATEST/gcc/5.x/centos6-native/*/`
CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
log_variable GCC_BASE
log_variable CLANG_BASE
# Libraries locations
get_lib_base libgcc 5.x
get_lib_base glibc 2.23
get_lib_base snappy LATEST gcc-5-glibc-2.23
get_lib_base zlib LATEST
get_lib_base bzip2 LATEST
get_lib_base lz4 LATEST
get_lib_base zstd LATEST
get_lib_base gflags LATEST
get_lib_base jemalloc LATEST
get_lib_base numa LATEST
get_lib_base libunwind LATEST
get_lib_base tbb 4.0_update2 gcc-5-glibc-2.23
get_lib_base kernel-headers LATEST
get_lib_base binutils LATEST centos6-native
get_lib_base valgrind 3.10.0 gcc-5-glibc-2.23
get_lib_base lua 5.2.3 gcc-5-glibc-2.23
git diff $OUTPUT
###########################################################
# 4.8.1 dependencies #
###########################################################
OUTPUT="$BASEDIR/dependencies_4.8.1.sh"
rm -f "$OUTPUT"
touch "$OUTPUT"
echo "Writing 4.8.1 dependencies to $OUTPUT"
# Compilers locations
GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.8.1/centos6-native/*/`
CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
log_variable GCC_BASE
log_variable CLANG_BASE
# Libraries locations
get_lib_base libgcc 4.8.1 gcc-4.8.1-glibc-2.17
get_lib_base glibc 2.17 gcc-4.8.1-glibc-2.17
get_lib_base snappy LATEST gcc-4.8.1-glibc-2.17
get_lib_base zlib LATEST gcc-4.8.1-glibc-2.17
get_lib_base bzip2 LATEST gcc-4.8.1-glibc-2.17
get_lib_base lz4 LATEST gcc-4.8.1-glibc-2.17
get_lib_base zstd LATEST gcc-4.8.1-glibc-2.17
get_lib_base gflags LATEST gcc-4.8.1-glibc-2.17
get_lib_base jemalloc LATEST gcc-4.8.1-glibc-2.17
get_lib_base numa LATEST gcc-4.8.1-glibc-2.17
get_lib_base libunwind LATEST gcc-4.8.1-glibc-2.17
get_lib_base tbb 4.0_update2 gcc-4.8.1-glibc-2.17
get_lib_base kernel-headers LATEST gcc-4.8.1-glibc-2.17
get_lib_base binutils LATEST centos6-native
get_lib_base valgrind 3.8.1 gcc-4.8.1-glibc-2.17
get_lib_base lua 5.2.3 centos6-native
git diff $OUTPUT

15
build_tools/valgrind_test.sh
View File

@@ -1,15 +0,0 @@
#!/bin/bash
#A shell script for Jenknis to run valgrind on rocksdb tests
#Returns 0 on success when there are no failed tests
VALGRIND_DIR=build_tools/VALGRIND_LOGS
make clean
make -j$(nproc) valgrind_check
NUM_FAILED_TESTS=$((`wc -l $VALGRIND_DIR/valgrind_failed_tests | awk '{print $1}'` - 1))
if [ $NUM_FAILED_TESTS -lt 1 ]; then
echo No tests have valgrind errors
exit 0
else
cat $VALGRIND_DIR/valgrind_failed_tests
exit 1
fi

22
build_tools/version.sh Executable file
View File

@@ -0,0 +1,22 @@
#!/bin/sh
if [ "$#" = "0" ]; then
echo "Usage: $0 major|minor|patch|full"
exit 1
fi
if [ "$1" = "major" ]; then
cat include/rocksdb/version.h | grep MAJOR | head -n1 | awk '{print $3}'
fi
if [ "$1" = "minor" ]; then
cat include/rocksdb/version.h | grep MINOR | head -n1 | awk '{print $3}'
fi
if [ "$1" = "patch" ]; then
cat include/rocksdb/version.h | grep PATCH | head -n1 | awk '{print $3}'
fi
if [ "$1" = "full" ]; then
awk '/#define ROCKSDB/ { env[$2] = $3 }
END { printf "%s.%s.%s\n", env["ROCKSDB_MAJOR"],
env["ROCKSDB_MINOR"],
env["ROCKSDB_PATCH"] }' \
include/rocksdb/version.h
fi

284
cache/cache_bench.cc vendored Normal file
View File

@@ -0,0 +1,284 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#ifndef GFLAGS
#include <cstdio>
int main() {
fprintf(stderr, "Please install gflags to run rocksdb tools\n");
return 1;
}
#else
#include <inttypes.h>
#include <sys/types.h>
#include <stdio.h>
#include <gflags/gflags.h>
#include "rocksdb/db.h"
#include "rocksdb/cache.h"
#include "rocksdb/env.h"
#include "port/port.h"
#include "util/mutexlock.h"
#include "util/random.h"
using GFLAGS::ParseCommandLineFlags;
static const uint32_t KB = 1024;
DEFINE_int32(threads, 16, "Number of concurrent threads to run.");
DEFINE_int64(cache_size, 8 * KB * KB,
"Number of bytes to use as a cache of uncompressed data.");
DEFINE_int32(num_shard_bits, 4, "shard_bits.");
DEFINE_int64(max_key, 1 * KB * KB * KB, "Max number of key to place in cache");
DEFINE_uint64(ops_per_thread, 1200000, "Number of operations per thread.");
DEFINE_bool(populate_cache, false, "Populate cache before operations");
DEFINE_int32(insert_percent, 40,
"Ratio of insert to total workload (expressed as a percentage)");
DEFINE_int32(lookup_percent, 50,
"Ratio of lookup to total workload (expressed as a percentage)");
DEFINE_int32(erase_percent, 10,
"Ratio of erase to total workload (expressed as a percentage)");
DEFINE_bool(use_clock_cache, false, "");
namespace rocksdb {
class CacheBench;
namespace {
void deleter(const Slice& key, void* value) {
delete reinterpret_cast<char *>(value);
}
// State shared by all concurrent executions of the same benchmark.
class SharedState {
public:
explicit SharedState(CacheBench* cache_bench)
: cv_(&mu_),
num_threads_(FLAGS_threads),
num_initialized_(0),
start_(false),
num_done_(0),
cache_bench_(cache_bench) {
}
~SharedState() {}
port::Mutex* GetMutex() {
return &mu_;
}
port::CondVar* GetCondVar() {
return &cv_;
}
CacheBench* GetCacheBench() const {
return cache_bench_;
}
void IncInitialized() {
num_initialized_++;
}
void IncDone() {
num_done_++;
}
bool AllInitialized() const {
return num_initialized_ >= num_threads_;
}
bool AllDone() const {
return num_done_ >= num_threads_;
}
void SetStart() {
start_ = true;
}
bool Started() const {
return start_;
}
private:
port::Mutex mu_;
port::CondVar cv_;
const uint64_t num_threads_;
uint64_t num_initialized_;
bool start_;
uint64_t num_done_;
CacheBench* cache_bench_;
};
// Per-thread state for concurrent executions of the same benchmark.
struct ThreadState {
uint32_t tid;
Random rnd;
SharedState* shared;
ThreadState(uint32_t index, SharedState* _shared)
: tid(index), rnd(1000 + index), shared(_shared) {}
};
} // namespace
class CacheBench {
public:
CacheBench() : num_threads_(FLAGS_threads) {
if (FLAGS_use_clock_cache) {
cache_ = NewClockCache(FLAGS_cache_size, FLAGS_num_shard_bits);
if (!cache_) {
fprintf(stderr, "Clock cache not supported.\n");
exit(1);
}
} else {
cache_ = NewLRUCache(FLAGS_cache_size, FLAGS_num_shard_bits);
}
}
~CacheBench() {}
void PopulateCache() {
Random rnd(1);
for (int64_t i = 0; i < FLAGS_cache_size; i++) {
uint64_t rand_key = rnd.Next() % FLAGS_max_key;
// Cast uint64* to be char*, data would be copied to cache
Slice key(reinterpret_cast<char*>(&rand_key), 8);
// do insert
cache_->Insert(key, new char[10], 1, &deleter);
}
}
bool Run() {
rocksdb::Env* env = rocksdb::Env::Default();
PrintEnv();
SharedState shared(this);
std::vector<ThreadState*> threads(num_threads_);
for (uint32_t i = 0; i < num_threads_; i++) {
threads[i] = new ThreadState(i, &shared);
env->StartThread(ThreadBody, threads[i]);
}
{
MutexLock l(shared.GetMutex());
while (!shared.AllInitialized()) {
shared.GetCondVar()->Wait();
}
// Record start time
uint64_t start_time = env->NowMicros();
// Start all threads
shared.SetStart();
shared.GetCondVar()->SignalAll();
// Wait threads to complete
while (!shared.AllDone()) {
shared.GetCondVar()->Wait();
}
// Record end time
uint64_t end_time = env->NowMicros();
double elapsed = static_cast<double>(end_time - start_time) * 1e-6;
uint32_t qps = static_cast<uint32_t>(
static_cast<double>(FLAGS_threads * FLAGS_ops_per_thread) / elapsed);
fprintf(stdout, "Complete in %.3f s; QPS = %u\n", elapsed, qps);
}
return true;
}
private:
std::shared_ptr<Cache> cache_;
uint32_t num_threads_;
static void ThreadBody(void* v) {
ThreadState* thread = reinterpret_cast<ThreadState*>(v);
SharedState* shared = thread->shared;
{
MutexLock l(shared->GetMutex());
shared->IncInitialized();
if (shared->AllInitialized()) {
shared->GetCondVar()->SignalAll();
}
while (!shared->Started()) {
shared->GetCondVar()->Wait();
}
}
thread->shared->GetCacheBench()->OperateCache(thread);
{
MutexLock l(shared->GetMutex());
shared->IncDone();
if (shared->AllDone()) {
shared->GetCondVar()->SignalAll();
}
}
}
void OperateCache(ThreadState* thread) {
for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
uint64_t rand_key = thread->rnd.Next() % FLAGS_max_key;
// Cast uint64* to be char*, data would be copied to cache
Slice key(reinterpret_cast<char*>(&rand_key), 8);
int32_t prob_op = thread->rnd.Uniform(100);
if (prob_op >= 0 && prob_op < FLAGS_insert_percent) {
// do insert
cache_->Insert(key, new char[10], 1, &deleter);
} else if (prob_op -= FLAGS_insert_percent &&
prob_op < FLAGS_lookup_percent) {
// do lookup
auto handle = cache_->Lookup(key);
if (handle) {
cache_->Release(handle);
}
} else if (prob_op -= FLAGS_lookup_percent &&
prob_op < FLAGS_erase_percent) {
// do erase
cache_->Erase(key);
}
}
}
void PrintEnv() const {
printf("RocksDB version : %d.%d\n", kMajorVersion, kMinorVersion);
printf("Number of threads : %d\n", FLAGS_threads);
printf("Ops per thread : %" PRIu64 "\n", FLAGS_ops_per_thread);
printf("Cache size : %" PRIu64 "\n", FLAGS_cache_size);
printf("Num shard bits : %d\n", FLAGS_num_shard_bits);
printf("Max key : %" PRIu64 "\n", FLAGS_max_key);
printf("Populate cache : %d\n", FLAGS_populate_cache);
printf("Insert percentage : %d%%\n", FLAGS_insert_percent);
printf("Lookup percentage : %d%%\n", FLAGS_lookup_percent);
printf("Erase percentage : %d%%\n", FLAGS_erase_percent);
printf("----------------------------\n");
}
};
} // namespace rocksdb
int main(int argc, char** argv) {
ParseCommandLineFlags(&argc, &argv, true);
if (FLAGS_threads <= 0) {
fprintf(stderr, "threads number <= 0\n");
exit(1);
}
rocksdb::CacheBench bench;
if (FLAGS_populate_cache) {
bench.PopulateCache();
}
if (bench.Run()) {
return 0;
} else {
return 1;
}
}
#endif // GFLAGS

703
cache/cache_test.cc vendored Normal file
View File

@@ -0,0 +1,703 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "rocksdb/cache.h"
#include <forward_list>
#include <functional>
#include <iostream>
#include <string>
#include <vector>
#include "cache/clock_cache.h"
#include "cache/lru_cache.h"
#include "util/coding.h"
#include "util/string_util.h"
#include "util/testharness.h"
namespace rocksdb {
// Conversions between numeric keys/values and the types expected by Cache.
static std::string EncodeKey(int k) {
std::string result;
PutFixed32(&result, k);
return result;
}
static int DecodeKey(const Slice& k) {
assert(k.size() == 4);
return DecodeFixed32(k.data());
}
static void* EncodeValue(uintptr_t v) { return reinterpret_cast<void*>(v); }
static int DecodeValue(void* v) {
return static_cast<int>(reinterpret_cast<uintptr_t>(v));
}
const std::string kLRU = "lru";
const std::string kClock = "clock";
void dumbDeleter(const Slice& key, void* value) {}
void eraseDeleter(const Slice& key, void* value) {
Cache* cache = reinterpret_cast<Cache*>(value);
cache->Erase("foo");
}
class CacheTest : public testing::TestWithParam<std::string> {
public:
static CacheTest* current_;
static void Deleter(const Slice& key, void* v) {
current_->deleted_keys_.push_back(DecodeKey(key));
current_->deleted_values_.push_back(DecodeValue(v));
}
static const int kCacheSize = 1000;
static const int kNumShardBits = 4;
static const int kCacheSize2 = 100;
static const int kNumShardBits2 = 2;
std::vector<int> deleted_keys_;
std::vector<int> deleted_values_;
shared_ptr<Cache> cache_;
shared_ptr<Cache> cache2_;
CacheTest()
: cache_(NewCache(kCacheSize, kNumShardBits, false)),
cache2_(NewCache(kCacheSize2, kNumShardBits2, false)) {
current_ = this;
}
~CacheTest() {
}
std::shared_ptr<Cache> NewCache(size_t capacity) {
auto type = GetParam();
if (type == kLRU) {
return NewLRUCache(capacity);
}
if (type == kClock) {
return NewClockCache(capacity);
}
return nullptr;
}
std::shared_ptr<Cache> NewCache(size_t capacity, int num_shard_bits,
bool strict_capacity_limit) {
auto type = GetParam();
if (type == kLRU) {
return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit);
}
if (type == kClock) {
return NewClockCache(capacity, num_shard_bits, strict_capacity_limit);
}
return nullptr;
}
int Lookup(shared_ptr<Cache> cache, int key) {
Cache::Handle* handle = cache->Lookup(EncodeKey(key));
const int r = (handle == nullptr) ? -1 : DecodeValue(cache->Value(handle));
if (handle != nullptr) {
cache->Release(handle);
}
return r;
}
void Insert(shared_ptr<Cache> cache, int key, int value, int charge = 1) {
cache->Insert(EncodeKey(key), EncodeValue(value), charge,
&CacheTest::Deleter);
}
void Erase(shared_ptr<Cache> cache, int key) {
cache->Erase(EncodeKey(key));
}
int Lookup(int key) {
return Lookup(cache_, key);
}
void Insert(int key, int value, int charge = 1) {
Insert(cache_, key, value, charge);
}
void Erase(int key) {
Erase(cache_, key);
}
int Lookup2(int key) {
return Lookup(cache2_, key);
}
void Insert2(int key, int value, int charge = 1) {
Insert(cache2_, key, value, charge);
}
void Erase2(int key) {
Erase(cache2_, key);
}
};
CacheTest* CacheTest::current_;
TEST_P(CacheTest, UsageTest) {
// cache is shared_ptr and will be automatically cleaned up.
const uint64_t kCapacity = 100000;
auto cache = NewCache(kCapacity, 8, false);
size_t usage = 0;
char value[10] = "abcdef";
// make sure everything will be cached
for (int i = 1; i < 100; ++i) {
std::string key(i, 'a');
auto kv_size = key.size() + 5;
cache->Insert(key, reinterpret_cast<void*>(value), kv_size, dumbDeleter);
usage += kv_size;
ASSERT_EQ(usage, cache->GetUsage());
}
// make sure the cache will be overloaded
for (uint64_t i = 1; i < kCapacity; ++i) {
auto key = ToString(i);
cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
dumbDeleter);
}
// the usage should be close to the capacity
ASSERT_GT(kCapacity, cache->GetUsage());
ASSERT_LT(kCapacity * 0.95, cache->GetUsage());
}
TEST_P(CacheTest, PinnedUsageTest) {
// cache is shared_ptr and will be automatically cleaned up.
const uint64_t kCapacity = 100000;
auto cache = NewCache(kCapacity, 8, false);
size_t pinned_usage = 0;
char value[10] = "abcdef";
std::forward_list<Cache::Handle*> unreleased_handles;
// Add entries. Unpin some of them after insertion. Then, pin some of them
// again. Check GetPinnedUsage().
for (int i = 1; i < 100; ++i) {
std::string key(i, 'a');
auto kv_size = key.size() + 5;
Cache::Handle* handle;
cache->Insert(key, reinterpret_cast<void*>(value), kv_size, dumbDeleter,
&handle);
pinned_usage += kv_size;
ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
if (i % 2 == 0) {
cache->Release(handle);
pinned_usage -= kv_size;
ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
} else {
unreleased_handles.push_front(handle);
}
if (i % 3 == 0) {
unreleased_handles.push_front(cache->Lookup(key));
// If i % 2 == 0, then the entry was unpinned before Lookup, so pinned
// usage increased
if (i % 2 == 0) {
pinned_usage += kv_size;
}
ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
}
}
// check that overloading the cache does not change the pinned usage
for (uint64_t i = 1; i < 2 * kCapacity; ++i) {
auto key = ToString(i);
cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
dumbDeleter);
}
ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
// release handles for pinned entries to prevent memory leaks
for (auto handle : unreleased_handles) {
cache->Release(handle);
}
}
TEST_P(CacheTest, HitAndMiss) {
ASSERT_EQ(-1, Lookup(100));
Insert(100, 101);
ASSERT_EQ(101, Lookup(100));
ASSERT_EQ(-1, Lookup(200));
ASSERT_EQ(-1, Lookup(300));
Insert(200, 201);
ASSERT_EQ(101, Lookup(100));
ASSERT_EQ(201, Lookup(200));
ASSERT_EQ(-1, Lookup(300));
Insert(100, 102);
ASSERT_EQ(102, Lookup(100));
ASSERT_EQ(201, Lookup(200));
ASSERT_EQ(-1, Lookup(300));
ASSERT_EQ(1U, deleted_keys_.size());
ASSERT_EQ(100, deleted_keys_[0]);
ASSERT_EQ(101, deleted_values_[0]);
}
TEST_P(CacheTest, InsertSameKey) {
Insert(1, 1);
Insert(1, 2);
ASSERT_EQ(2, Lookup(1));
}
TEST_P(CacheTest, Erase) {
Erase(200);
ASSERT_EQ(0U, deleted_keys_.size());
Insert(100, 101);
Insert(200, 201);
Erase(100);
ASSERT_EQ(-1, Lookup(100));
ASSERT_EQ(201, Lookup(200));
ASSERT_EQ(1U, deleted_keys_.size());
ASSERT_EQ(100, deleted_keys_[0]);
ASSERT_EQ(101, deleted_values_[0]);
Erase(100);
ASSERT_EQ(-1, Lookup(100));
ASSERT_EQ(201, Lookup(200));
ASSERT_EQ(1U, deleted_keys_.size());
}
TEST_P(CacheTest, EntriesArePinned) {
Insert(100, 101);
Cache::Handle* h1 = cache_->Lookup(EncodeKey(100));
ASSERT_EQ(101, DecodeValue(cache_->Value(h1)));
ASSERT_EQ(1U, cache_->GetUsage());
Insert(100, 102);
Cache::Handle* h2 = cache_->Lookup(EncodeKey(100));
ASSERT_EQ(102, DecodeValue(cache_->Value(h2)));
ASSERT_EQ(0U, deleted_keys_.size());
ASSERT_EQ(2U, cache_->GetUsage());
cache_->Release(h1);
ASSERT_EQ(1U, deleted_keys_.size());
ASSERT_EQ(100, deleted_keys_[0]);
ASSERT_EQ(101, deleted_values_[0]);
ASSERT_EQ(1U, cache_->GetUsage());
Erase(100);
ASSERT_EQ(-1, Lookup(100));
ASSERT_EQ(1U, deleted_keys_.size());
ASSERT_EQ(1U, cache_->GetUsage());
cache_->Release(h2);
ASSERT_EQ(2U, deleted_keys_.size());
ASSERT_EQ(100, deleted_keys_[1]);
ASSERT_EQ(102, deleted_values_[1]);
ASSERT_EQ(0U, cache_->GetUsage());
}
TEST_P(CacheTest, EvictionPolicy) {
Insert(100, 101);
Insert(200, 201);
// Frequently used entry must be kept around
for (int i = 0; i < kCacheSize + 100; i++) {
Insert(1000+i, 2000+i);
ASSERT_EQ(101, Lookup(100));
}
ASSERT_EQ(101, Lookup(100));
ASSERT_EQ(-1, Lookup(200));
}
TEST_P(CacheTest, ExternalRefPinsEntries) {
Insert(100, 101);
Cache::Handle* h = cache_->Lookup(EncodeKey(100));
ASSERT_TRUE(cache_->Ref(h));
ASSERT_EQ(101, DecodeValue(cache_->Value(h)));
ASSERT_EQ(1U, cache_->GetUsage());
for (int i = 0; i < 3; ++i) {
if (i > 0) {
// First release (i == 1) corresponds to Ref(), second release (i == 2)
// corresponds to Lookup(). Then, since all external refs are released,
// the below insertions should push out the cache entry.
cache_->Release(h);
}
// double cache size because the usage bit in block cache prevents 100 from
// being evicted in the first kCacheSize iterations
for (int j = 0; j < 2 * kCacheSize + 100; j++) {
Insert(1000 + j, 2000 + j);
}
if (i < 2) {
ASSERT_EQ(101, Lookup(100));
}
}
ASSERT_EQ(-1, Lookup(100));
}
TEST_P(CacheTest, EvictionPolicyRef) {
Insert(100, 101);
Insert(101, 102);
Insert(102, 103);
Insert(103, 104);
Insert(200, 101);
Insert(201, 102);
Insert(202, 103);
Insert(203, 104);
Cache::Handle* h201 = cache_->Lookup(EncodeKey(200));
Cache::Handle* h202 = cache_->Lookup(EncodeKey(201));
Cache::Handle* h203 = cache_->Lookup(EncodeKey(202));
Cache::Handle* h204 = cache_->Lookup(EncodeKey(203));
Insert(300, 101);
Insert(301, 102);
Insert(302, 103);
Insert(303, 104);
// Insert entries much more than Cache capacity
for (int i = 0; i < kCacheSize + 100; i++) {
Insert(1000 + i, 2000 + i);
}
// Check whether the entries inserted in the beginning
// are evicted. Ones without extra ref are evicted and
// those with are not.
ASSERT_EQ(-1, Lookup(100));
ASSERT_EQ(-1, Lookup(101));
ASSERT_EQ(-1, Lookup(102));
ASSERT_EQ(-1, Lookup(103));
ASSERT_EQ(-1, Lookup(300));
ASSERT_EQ(-1, Lookup(301));
ASSERT_EQ(-1, Lookup(302));
ASSERT_EQ(-1, Lookup(303));
ASSERT_EQ(101, Lookup(200));
ASSERT_EQ(102, Lookup(201));
ASSERT_EQ(103, Lookup(202));
ASSERT_EQ(104, Lookup(203));
// Cleaning up all the handles
cache_->Release(h201);
cache_->Release(h202);
cache_->Release(h203);
cache_->Release(h204);
}
TEST_P(CacheTest, EvictEmptyCache) {
// Insert item large than capacity to trigger eviction on empty cache.
auto cache = NewCache(1, 0, false);
ASSERT_OK(cache->Insert("foo", nullptr, 10, dumbDeleter));
}
TEST_P(CacheTest, EraseFromDeleter) {
// Have deleter which will erase item from cache, which will re-enter
// the cache at that point.
std::shared_ptr<Cache> cache = NewCache(10, 0, false);
ASSERT_OK(cache->Insert("foo", nullptr, 1, dumbDeleter));
ASSERT_OK(cache->Insert("bar", cache.get(), 1, eraseDeleter));
cache->Erase("bar");
ASSERT_EQ(nullptr, cache->Lookup("foo"));
ASSERT_EQ(nullptr, cache->Lookup("bar"));
}
TEST_P(CacheTest, ErasedHandleState) {
// insert a key and get two handles
Insert(100, 1000);
Cache::Handle* h1 = cache_->Lookup(EncodeKey(100));
Cache::Handle* h2 = cache_->Lookup(EncodeKey(100));
ASSERT_EQ(h1, h2);
ASSERT_EQ(DecodeValue(cache_->Value(h1)), 1000);
ASSERT_EQ(DecodeValue(cache_->Value(h2)), 1000);
// delete the key from the cache
Erase(100);
// can no longer find in the cache
ASSERT_EQ(-1, Lookup(100));
// release one handle
cache_->Release(h1);
// still can't find in cache
ASSERT_EQ(-1, Lookup(100));
cache_->Release(h2);
}
TEST_P(CacheTest, HeavyEntries) {
// Add a bunch of light and heavy entries and then count the combined
// size of items still in the cache, which must be approximately the
// same as the total capacity.
const int kLight = 1;
const int kHeavy = 10;
int added = 0;
int index = 0;
while (added < 2*kCacheSize) {
const int weight = (index & 1) ? kLight : kHeavy;
Insert(index, 1000+index, weight);
added += weight;
index++;
}
int cached_weight = 0;
for (int i = 0; i < index; i++) {
const int weight = (i & 1 ? kLight : kHeavy);
int r = Lookup(i);
if (r >= 0) {
cached_weight += weight;
ASSERT_EQ(1000+i, r);
}
}
ASSERT_LE(cached_weight, kCacheSize + kCacheSize/10);
}
TEST_P(CacheTest, NewId) {
uint64_t a = cache_->NewId();
uint64_t b = cache_->NewId();
ASSERT_NE(a, b);
}
class Value {
public:
explicit Value(size_t v) : v_(v) { }
size_t v_;
};
namespace {
void deleter(const Slice& key, void* value) {
delete static_cast<Value *>(value);
}
} // namespace
TEST_P(CacheTest, ReleaseAndErase) {
std::shared_ptr<Cache> cache = NewCache(5, 0, false);
Cache::Handle* handle;
Status s = cache->Insert(EncodeKey(100), EncodeValue(100), 1,
&CacheTest::Deleter, &handle);
ASSERT_TRUE(s.ok());
ASSERT_EQ(5U, cache->GetCapacity());
ASSERT_EQ(1U, cache->GetUsage());
ASSERT_EQ(0U, deleted_keys_.size());
auto erased = cache->Release(handle, true);
ASSERT_TRUE(erased);
// This tests that deleter has been called
ASSERT_EQ(1U, deleted_keys_.size());
}
TEST_P(CacheTest, ReleaseWithoutErase) {
std::shared_ptr<Cache> cache = NewCache(5, 0, false);
Cache::Handle* handle;
Status s = cache->Insert(EncodeKey(100), EncodeValue(100), 1,
&CacheTest::Deleter, &handle);
ASSERT_TRUE(s.ok());
ASSERT_EQ(5U, cache->GetCapacity());
ASSERT_EQ(1U, cache->GetUsage());
ASSERT_EQ(0U, deleted_keys_.size());
auto erased = cache->Release(handle);
ASSERT_FALSE(erased);
// This tests that deleter is not called. When cache has free capacity it is
// not expected to immediately erase the released items.
ASSERT_EQ(0U, deleted_keys_.size());
}
TEST_P(CacheTest, SetCapacity) {
// test1: increase capacity
// lets create a cache with capacity 5,
// then, insert 5 elements, then increase capacity
// to 10, returned capacity should be 10, usage=5
std::shared_ptr<Cache> cache = NewCache(5, 0, false);
std::vector<Cache::Handle*> handles(10);
// Insert 5 entries, but not releasing.
for (size_t i = 0; i < 5; i++) {
std::string key = ToString(i+1);
Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
ASSERT_TRUE(s.ok());
}
ASSERT_EQ(5U, cache->GetCapacity());
ASSERT_EQ(5U, cache->GetUsage());
cache->SetCapacity(10);
ASSERT_EQ(10U, cache->GetCapacity());
ASSERT_EQ(5U, cache->GetUsage());
// test2: decrease capacity
// insert 5 more elements to cache, then release 5,
// then decrease capacity to 7, final capacity should be 7
// and usage should be 7
for (size_t i = 5; i < 10; i++) {
std::string key = ToString(i+1);
Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
ASSERT_TRUE(s.ok());
}
ASSERT_EQ(10U, cache->GetCapacity());
ASSERT_EQ(10U, cache->GetUsage());
for (size_t i = 0; i < 5; i++) {
cache->Release(handles[i]);
}
ASSERT_EQ(10U, cache->GetCapacity());
ASSERT_EQ(10U, cache->GetUsage());
cache->SetCapacity(7);
ASSERT_EQ(7, cache->GetCapacity());
ASSERT_EQ(7, cache->GetUsage());
// release remaining 5 to keep valgrind happy
for (size_t i = 5; i < 10; i++) {
cache->Release(handles[i]);
}
}
TEST_P(CacheTest, SetStrictCapacityLimit) {
// test1: set the flag to false. Insert more keys than capacity. See if they
// all go through.
std::shared_ptr<Cache> cache = NewLRUCache(5, 0, false);
std::vector<Cache::Handle*> handles(10);
Status s;
for (size_t i = 0; i < 10; i++) {
std::string key = ToString(i + 1);
s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
ASSERT_OK(s);
ASSERT_NE(nullptr, handles[i]);
}
// test2: set the flag to true. Insert and check if it fails.
std::string extra_key = "extra";
Value* extra_value = new Value(0);
cache->SetStrictCapacityLimit(true);
Cache::Handle* handle;
s = cache->Insert(extra_key, extra_value, 1, &deleter, &handle);
ASSERT_TRUE(s.IsIncomplete());
ASSERT_EQ(nullptr, handle);
for (size_t i = 0; i < 10; i++) {
cache->Release(handles[i]);
}
// test3: init with flag being true.
std::shared_ptr<Cache> cache2 = NewLRUCache(5, 0, true);
for (size_t i = 0; i < 5; i++) {
std::string key = ToString(i + 1);
s = cache2->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
ASSERT_OK(s);
ASSERT_NE(nullptr, handles[i]);
}
s = cache2->Insert(extra_key, extra_value, 1, &deleter, &handle);
ASSERT_TRUE(s.IsIncomplete());
ASSERT_EQ(nullptr, handle);
// test insert without handle
s = cache2->Insert(extra_key, extra_value, 1, &deleter);
// AS if the key have been inserted into cache but get evicted immediately.
ASSERT_OK(s);
ASSERT_EQ(5, cache->GetUsage());
ASSERT_EQ(nullptr, cache2->Lookup(extra_key));
for (size_t i = 0; i < 5; i++) {
cache2->Release(handles[i]);
}
}
TEST_P(CacheTest, OverCapacity) {
size_t n = 10;
// a LRUCache with n entries and one shard only
std::shared_ptr<Cache> cache = NewCache(n, 0, false);
std::vector<Cache::Handle*> handles(n+1);
// Insert n+1 entries, but not releasing.
for (size_t i = 0; i < n + 1; i++) {
std::string key = ToString(i+1);
Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
ASSERT_TRUE(s.ok());
}
// Guess what's in the cache now?
for (size_t i = 0; i < n + 1; i++) {
std::string key = ToString(i+1);
auto h = cache->Lookup(key);
ASSERT_TRUE(h != nullptr);
if (h) cache->Release(h);
}
// the cache is over capacity since nothing could be evicted
ASSERT_EQ(n + 1U, cache->GetUsage());
for (size_t i = 0; i < n + 1; i++) {
cache->Release(handles[i]);
}
// Make sure eviction is triggered.
cache->SetCapacity(n);
// cache is under capacity now since elements were released
ASSERT_EQ(n, cache->GetUsage());
// element 0 is evicted and the rest is there
// This is consistent with the LRU policy since the element 0
// was released first
for (size_t i = 0; i < n + 1; i++) {
std::string key = ToString(i+1);
auto h = cache->Lookup(key);
if (h) {
ASSERT_NE(i, 0U);
cache->Release(h);
} else {
ASSERT_EQ(i, 0U);
}
}
}
namespace {
std::vector<std::pair<int, int>> callback_state;
void callback(void* entry, size_t charge) {
callback_state.push_back({DecodeValue(entry), static_cast<int>(charge)});
}
};
TEST_P(CacheTest, ApplyToAllCacheEntiresTest) {
std::vector<std::pair<int, int>> inserted;
callback_state.clear();
for (int i = 0; i < 10; ++i) {
Insert(i, i * 2, i + 1);
inserted.push_back({i * 2, i + 1});
}
cache_->ApplyToAllCacheEntries(callback, true);
std::sort(inserted.begin(), inserted.end());
std::sort(callback_state.begin(), callback_state.end());
ASSERT_TRUE(inserted == callback_state);
}
TEST_P(CacheTest, DefaultShardBits) {
// test1: set the flag to false. Insert more keys than capacity. See if they
// all go through.
std::shared_ptr<Cache> cache = NewCache(16 * 1024L * 1024L);
ShardedCache* sc = dynamic_cast<ShardedCache*>(cache.get());
ASSERT_EQ(5, sc->GetNumShardBits());
cache = NewLRUCache(511 * 1024L, -1, true);
sc = dynamic_cast<ShardedCache*>(cache.get());
ASSERT_EQ(0, sc->GetNumShardBits());
cache = NewLRUCache(1024L * 1024L * 1024L, -1, true);
sc = dynamic_cast<ShardedCache*>(cache.get());
ASSERT_EQ(6, sc->GetNumShardBits());
}
#ifdef SUPPORT_CLOCK_CACHE
shared_ptr<Cache> (*new_clock_cache_func)(size_t, int, bool) = NewClockCache;
INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest,
testing::Values(kLRU, kClock));
#else
INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest, testing::Values(kLRU));
#endif // SUPPORT_CLOCK_CACHE
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

729
cache/clock_cache.cc vendored Normal file
View File

@@ -0,0 +1,729 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "cache/clock_cache.h"
#ifndef SUPPORT_CLOCK_CACHE
namespace rocksdb {
std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
bool strict_capacity_limit) {
// Clock cache not supported.
return nullptr;
}
} // namespace rocksdb
#else
#include <assert.h>
#include <atomic>
#include <deque>
// "tbb/concurrent_hash_map.h" requires RTTI if exception is enabled.
// Disable it so users can chooose to disable RTTI.
#ifndef ROCKSDB_USE_RTTI
#define TBB_USE_EXCEPTIONS 0
#endif
#include "tbb/concurrent_hash_map.h"
#include "cache/sharded_cache.h"
#include "port/port.h"
#include "util/autovector.h"
#include "util/mutexlock.h"
namespace rocksdb {
namespace {
// An implementation of the Cache interface based on CLOCK algorithm, with
// better concurrent performance than LRUCache. The idea of CLOCK algorithm
// is to maintain all cache entries in a circular list, and an iterator
// (the "head") pointing to the last examined entry. Eviction starts from the
// current head. Each entry is given a second chance before eviction, if it
// has been access since last examine. In contrast to LRU, no modification
// to the internal data-structure (except for flipping the usage bit) needs
// to be done upon lookup. This gives us oppertunity to implement a cache
// with better concurrency.
//
// Each cache entry is represented by a cache handle, and all the handles
// are arranged in a circular list, as describe above. Upon erase of an entry,
// we never remove the handle. Instead, the handle is put into a recycle bin
// to be re-use. This is to avoid memory dealocation, which is hard to deal
// with in concurrent environment.
//
// The cache also maintains a concurrent hash map for lookup. Any concurrent
// hash map implementation should do the work. We currently use
// tbb::concurrent_hash_map because it supports concurrent erase.
//
// Each cache handle has the following flags and counters, which are squeeze
// in an atomic interger, to make sure the handle always be in a consistent
// state:
//
// * In-cache bit: whether the entry is reference by the cache itself. If
// an entry is in cache, its key would also be available in the hash map.
// * Usage bit: whether the entry has been access by user since last
// examine for eviction. Can be reset by eviction.
// * Reference count: reference count by user.
//
// An entry can be reference only when it's in cache. An entry can be evicted
// only when it is in cache, has no usage since last examine, and reference
// count is zero.
//
// The follow figure shows a possible layout of the cache. Boxes represents
// cache handles and numbers in each box being in-cache bit, usage bit and
// reference count respectively.
//
// hash map:
// +-------+--------+
// | key | handle |
// +-------+--------+
// | "foo" | 5 |-------------------------------------+
// +-------+--------+ |
// | "bar" | 2 |--+ |
// +-------+--------+ | |
// | |
// head | |
// | | |
// circular list: | | |
// +-------+ +-------+ +-------+ +-------+ +-------+ +-------
// |(0,0,0)|---|(1,1,0)|---|(0,0,0)|---|(0,1,3)|---|(1,0,0)|---| ...
// +-------+ +-------+ +-------+ +-------+ +-------+ +-------
// | |
// +-------+ +-----------+
// | |
// +---+---+
// recycle bin: | 1 | 3 |
// +---+---+
//
// Suppose we try to insert "baz" into the cache at this point and the cache is
// full. The cache will first look for entries to evict, starting from where
// head points to (the second entry). It resets usage bit of the second entry,
// skips the third and fourth entry since they are not in cache, and finally
// evict the fifth entry ("foo"). It looks at recycle bin for available handle,
// grabs handle 3, and insert the key into the handle. The following figure
// shows the resulting layout.
//
// hash map:
// +-------+--------+
// | key | handle |
// +-------+--------+
// | "baz" | 3 |-------------+
// +-------+--------+ |
// | "bar" | 2 |--+ |
// +-------+--------+ | |
// | |
// | | head
// | | |
// circular list: | | |
// +-------+ +-------+ +-------+ +-------+ +-------+ +-------
// |(0,0,0)|---|(1,0,0)|---|(1,0,0)|---|(0,1,3)|---|(0,0,0)|---| ...
// +-------+ +-------+ +-------+ +-------+ +-------+ +-------
// | |
// +-------+ +-----------------------------------+
// | |
// +---+---+
// recycle bin: | 1 | 5 |
// +---+---+
//
// A global mutex guards the circular list, the head, and the recycle bin.
// We additionally require that modifying the hash map needs to hold the mutex.
// As such, Modifying the cache (such as Insert() and Erase()) require to
// hold the mutex. Lookup() only access the hash map and the flags associated
// with each handle, and don't require explicit locking. Release() has to
// acquire the mutex only when it releases the last reference to the entry and
// the entry has been erased from cache explicitly. A future improvement could
// be to remove the mutex completely.
//
// Benchmark:
// We run readrandom db_bench on a test DB of size 13GB, with size of each
// level:
//
// Level Files Size(MB)
// -------------------------
// L0 1 0.01
// L1 18 17.32
// L2 230 182.94
// L3 1186 1833.63
// L4 4602 8140.30
//
// We test with both 32 and 16 read threads, with 2GB cache size (the whole DB
// doesn't fits in) and 64GB cache size (the whole DB can fit in cache), and
// whether to put index and filter blocks in block cache. The benchmark runs
// with
// with RocksDB 4.10. We got the following result:
//
// Threads Cache Cache ClockCache LRUCache
// Size Index/Filter Throughput(MB/s) Hit Throughput(MB/s) Hit
// 32 2GB yes 466.7 85.9% 433.7 86.5%
// 32 2GB no 529.9 72.7% 532.7 73.9%
// 32 64GB yes 649.9 99.9% 507.9 99.9%
// 32 64GB no 740.4 99.9% 662.8 99.9%
// 16 2GB yes 278.4 85.9% 283.4 86.5%
// 16 2GB no 318.6 72.7% 335.8 73.9%
// 16 64GB yes 391.9 99.9% 353.3 99.9%
// 16 64GB no 433.8 99.8% 419.4 99.8%
// Cache entry meta data.
struct CacheHandle {
Slice key;
uint32_t hash;
void* value;
size_t charge;
void (*deleter)(const Slice&, void* value);
// Flags and counters associated with the cache handle:
// lowest bit: n-cache bit
// second lowest bit: usage bit
// the rest bits: reference count
// The handle is unused when flags equals to 0. The thread decreases the count
// to 0 is responsible to put the handle back to recycle_ and cleanup memory.
std::atomic<uint32_t> flags;
CacheHandle() = default;
CacheHandle(const CacheHandle& a) { *this = a; }
CacheHandle(const Slice& k, void* v,
void (*del)(const Slice& key, void* value))
: key(k), value(v), deleter(del) {}
CacheHandle& operator=(const CacheHandle& a) {
// Only copy members needed for deletion.
key = a.key;
value = a.value;
deleter = a.deleter;
return *this;
}
};
// Key of hash map. We store hash value with the key for convenience.
struct CacheKey {
Slice key;
uint32_t hash_value;
CacheKey() = default;
CacheKey(const Slice& k, uint32_t h) {
key = k;
hash_value = h;
}
static bool equal(const CacheKey& a, const CacheKey& b) {
return a.hash_value == b.hash_value && a.key == b.key;
}
static size_t hash(const CacheKey& a) {
return static_cast<size_t>(a.hash_value);
}
};
struct CleanupContext {
// List of values to be deleted, along with the key and deleter.
autovector<CacheHandle> to_delete_value;
// List of keys to be deleted.
autovector<const char*> to_delete_key;
};
// A cache shard which maintains its own CLOCK cache.
class ClockCacheShard : public CacheShard {
public:
// Hash map type.
typedef tbb::concurrent_hash_map<CacheKey, CacheHandle*, CacheKey> HashTable;
ClockCacheShard();
~ClockCacheShard();
// Interfaces
virtual void SetCapacity(size_t capacity) override;
virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
virtual Status Insert(const Slice& key, uint32_t hash, void* value,
size_t charge,
void (*deleter)(const Slice& key, void* value),
Cache::Handle** handle,
Cache::Priority priority) override;
virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
// If the entry in in cache, increase reference count and return true.
// Return false otherwise.
//
// Not necessary to hold mutex_ before being called.
virtual bool Ref(Cache::Handle* handle) override;
virtual bool Release(Cache::Handle* handle,
bool force_erase = false) override;
virtual void Erase(const Slice& key, uint32_t hash) override;
bool EraseAndConfirm(const Slice& key, uint32_t hash,
CleanupContext* context);
virtual size_t GetUsage() const override;
virtual size_t GetPinnedUsage() const override;
virtual void EraseUnRefEntries() override;
virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
bool thread_safe) override;
private:
static const uint32_t kInCacheBit = 1;
static const uint32_t kUsageBit = 2;
static const uint32_t kRefsOffset = 2;
static const uint32_t kOneRef = 1 << kRefsOffset;
// Helper functions to extract cache handle flags and counters.
static bool InCache(uint32_t flags) { return flags & kInCacheBit; }
static bool HasUsage(uint32_t flags) { return flags & kUsageBit; }
static uint32_t CountRefs(uint32_t flags) { return flags >> kRefsOffset; }
// Decrease reference count of the entry. If this decreases the count to 0,
// recycle the entry. If set_usage is true, also set the usage bit.
//
// returns true if a value is erased.
//
// Not necessary to hold mutex_ before being called.
bool Unref(CacheHandle* handle, bool set_usage, CleanupContext* context);
// Unset in-cache bit of the entry. Recycle the handle if necessary.
//
// returns true if a value is erased.
//
// Has to hold mutex_ before being called.
bool UnsetInCache(CacheHandle* handle, CleanupContext* context);
// Put the handle back to recycle_ list, and put the value associated with
// it into to-be-deleted list. It doesn't cleanup the key as it might be
// reused by another handle.
//
// Has to hold mutex_ before being called.
void RecycleHandle(CacheHandle* handle, CleanupContext* context);
// Delete keys and values in to-be-deleted list. Call the method without
// holding mutex, as destructors can be expensive.
void Cleanup(const CleanupContext& context);
// Examine the handle for eviction. If the handle is in cache, usage bit is
// not set, and referece count is 0, evict it from cache. Otherwise unset
// the usage bit.
//
// Has to hold mutex_ before being called.
bool TryEvict(CacheHandle* value, CleanupContext* context);
// Scan through the circular list, evict entries until we get enough capacity
// for new cache entry of specific size. Return true if success, false
// otherwise.
//
// Has to hold mutex_ before being called.
bool EvictFromCache(size_t charge, CleanupContext* context);
CacheHandle* Insert(const Slice& key, uint32_t hash, void* value,
size_t change,
void (*deleter)(const Slice& key, void* value),
bool hold_reference, CleanupContext* context);
// Guards list_, head_, and recycle_. In addition, updating table_ also has
// to hold the mutex, to avoid the cache being in inconsistent state.
mutable port::Mutex mutex_;
// The circular list of cache handles. Initially the list is empty. Once a
// handle is needed by insertion, and no more handles are available in
// recycle bin, one more handle is appended to the end.
//
// We use std::deque for the circular list because we want to make sure
// pointers to handles are valid through out the life-cycle of the cache
// (in contrast to std::vector), and be able to grow the list (in contrast
// to statically allocated arrays).
std::deque<CacheHandle> list_;
// Pointer to the next handle in the circular list to be examine for
// eviction.
size_t head_;
// Recycle bin of cache handles.
autovector<CacheHandle*> recycle_;
// Maximum cache size.
std::atomic<size_t> capacity_;
// Current total size of the cache.
std::atomic<size_t> usage_;
// Total un-released cache size.
std::atomic<size_t> pinned_usage_;
// Whether allow insert into cache if cache is full.
std::atomic<bool> strict_capacity_limit_;
// Hash table (tbb::concurrent_hash_map) for lookup.
HashTable table_;
};
ClockCacheShard::ClockCacheShard()
: head_(0), usage_(0), pinned_usage_(0), strict_capacity_limit_(false) {}
ClockCacheShard::~ClockCacheShard() {
for (auto& handle : list_) {
uint32_t flags = handle.flags.load(std::memory_order_relaxed);
if (InCache(flags) || CountRefs(flags) > 0) {
(*handle.deleter)(handle.key, handle.value);
delete[] handle.key.data();
}
}
}
size_t ClockCacheShard::GetUsage() const {
return usage_.load(std::memory_order_relaxed);
}
size_t ClockCacheShard::GetPinnedUsage() const {
return pinned_usage_.load(std::memory_order_relaxed);
}
void ClockCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
bool thread_safe) {
if (thread_safe) {
mutex_.Lock();
}
for (auto& handle : list_) {
// Use relaxed semantics instead of acquire semantics since we are either
// holding mutex, or don't have thread safe requirement.
uint32_t flags = handle.flags.load(std::memory_order_relaxed);
if (InCache(flags)) {
callback(handle.value, handle.charge);
}
}
if (thread_safe) {
mutex_.Unlock();
}
}
void ClockCacheShard::RecycleHandle(CacheHandle* handle,
CleanupContext* context) {
mutex_.AssertHeld();
assert(!InCache(handle->flags) && CountRefs(handle->flags) == 0);
context->to_delete_key.push_back(handle->key.data());
context->to_delete_value.emplace_back(*handle);
handle->key.clear();
handle->value = nullptr;
handle->deleter = nullptr;
recycle_.push_back(handle);
usage_.fetch_sub(handle->charge, std::memory_order_relaxed);
}
void ClockCacheShard::Cleanup(const CleanupContext& context) {
for (const CacheHandle& handle : context.to_delete_value) {
if (handle.deleter) {
(*handle.deleter)(handle.key, handle.value);
}
}
for (const char* key : context.to_delete_key) {
delete[] key;
}
}
bool ClockCacheShard::Ref(Cache::Handle* h) {
auto handle = reinterpret_cast<CacheHandle*>(h);
// CAS loop to increase reference count.
uint32_t flags = handle->flags.load(std::memory_order_relaxed);
while (InCache(flags)) {
// Use acquire semantics on success, as further operations on the cache
// entry has to be order after reference count is increased.
if (handle->flags.compare_exchange_weak(flags, flags + kOneRef,
std::memory_order_acquire,
std::memory_order_relaxed)) {
if (CountRefs(flags) == 0) {
// No reference count before the operation.
pinned_usage_.fetch_add(handle->charge, std::memory_order_relaxed);
}
return true;
}
}
return false;
}
bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
CleanupContext* context) {
if (set_usage) {
handle->flags.fetch_or(kUsageBit, std::memory_order_relaxed);
}
// Use acquire-release semantics as previous operations on the cache entry
// has to be order before reference count is decreased, and potential cleanup
// of the entry has to be order after.
uint32_t flags = handle->flags.fetch_sub(kOneRef, std::memory_order_acq_rel);
assert(CountRefs(flags) > 0);
if (CountRefs(flags) == 1) {
// this is the last reference.
pinned_usage_.fetch_sub(handle->charge, std::memory_order_relaxed);
// Cleanup if it is the last reference.
if (!InCache(flags)) {
MutexLock l(&mutex_);
RecycleHandle(handle, context);
}
}
return context->to_delete_value.size();
}
bool ClockCacheShard::UnsetInCache(CacheHandle* handle,
CleanupContext* context) {
mutex_.AssertHeld();
// Use acquire-release semantics as previous operations on the cache entry
// has to be order before reference count is decreased, and potential cleanup
// of the entry has to be order after.
uint32_t flags =
handle->flags.fetch_and(~kInCacheBit, std::memory_order_acq_rel);
// Cleanup if it is the last reference.
if (InCache(flags) && CountRefs(flags) == 0) {
RecycleHandle(handle, context);
}
return context->to_delete_value.size();
}
bool ClockCacheShard::TryEvict(CacheHandle* handle, CleanupContext* context) {
mutex_.AssertHeld();
uint32_t flags = kInCacheBit;
if (handle->flags.compare_exchange_strong(flags, 0, std::memory_order_acquire,
std::memory_order_relaxed)) {
bool erased __attribute__((__unused__)) =
table_.erase(CacheKey(handle->key, handle->hash));
assert(erased);
RecycleHandle(handle, context);
return true;
}
handle->flags.fetch_and(~kUsageBit, std::memory_order_relaxed);
return false;
}
bool ClockCacheShard::EvictFromCache(size_t charge, CleanupContext* context) {
size_t usage = usage_.load(std::memory_order_relaxed);
size_t capacity = capacity_.load(std::memory_order_relaxed);
if (usage == 0) {
return charge <= capacity;
}
size_t new_head = head_;
bool second_iteration = false;
while (usage + charge > capacity) {
assert(new_head < list_.size());
if (TryEvict(&list_[new_head], context)) {
usage = usage_.load(std::memory_order_relaxed);
}
new_head = (new_head + 1 >= list_.size()) ? 0 : new_head + 1;
if (new_head == head_) {
if (second_iteration) {
return false;
} else {
second_iteration = true;
}
}
}
head_ = new_head;
return true;
}
void ClockCacheShard::SetCapacity(size_t capacity) {
CleanupContext context;
{
MutexLock l(&mutex_);
capacity_.store(capacity, std::memory_order_relaxed);
EvictFromCache(0, &context);
}
Cleanup(context);
}
void ClockCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
strict_capacity_limit_.store(strict_capacity_limit,
std::memory_order_relaxed);
}
CacheHandle* ClockCacheShard::Insert(
const Slice& key, uint32_t hash, void* value, size_t charge,
void (*deleter)(const Slice& key, void* value), bool hold_reference,
CleanupContext* context) {
MutexLock l(&mutex_);
bool success = EvictFromCache(charge, context);
bool strict = strict_capacity_limit_.load(std::memory_order_relaxed);
if (!success && (strict || !hold_reference)) {
context->to_delete_key.push_back(key.data());
if (!hold_reference) {
context->to_delete_value.emplace_back(key, value, deleter);
}
return nullptr;
}
// Grab available handle from recycle bin. If recycle bin is empty, create
// and append new handle to end of circular list.
CacheHandle* handle = nullptr;
if (!recycle_.empty()) {
handle = recycle_.back();
recycle_.pop_back();
} else {
list_.emplace_back();
handle = &list_.back();
}
// Fill handle.
handle->key = key;
handle->hash = hash;
handle->value = value;
handle->charge = charge;
handle->deleter = deleter;
uint32_t flags = hold_reference ? kInCacheBit + kOneRef : kInCacheBit;
handle->flags.store(flags, std::memory_order_relaxed);
HashTable::accessor accessor;
if (table_.find(accessor, CacheKey(key, hash))) {
CacheHandle* existing_handle = accessor->second;
table_.erase(accessor);
UnsetInCache(existing_handle, context);
}
table_.insert(HashTable::value_type(CacheKey(key, hash), handle));
if (hold_reference) {
pinned_usage_.fetch_add(charge, std::memory_order_relaxed);
}
usage_.fetch_add(charge, std::memory_order_relaxed);
return handle;
}
Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
size_t charge,
void (*deleter)(const Slice& key, void* value),
Cache::Handle** out_handle,
Cache::Priority priority) {
CleanupContext context;
HashTable::accessor accessor;
char* key_data = new char[key.size()];
memcpy(key_data, key.data(), key.size());
Slice key_copy(key_data, key.size());
CacheHandle* handle = Insert(key_copy, hash, value, charge, deleter,
out_handle != nullptr, &context);
Status s;
if (out_handle != nullptr) {
if (handle == nullptr) {
s = Status::Incomplete("Insert failed due to LRU cache being full.");
} else {
*out_handle = reinterpret_cast<Cache::Handle*>(handle);
}
}
Cleanup(context);
return s;
}
Cache::Handle* ClockCacheShard::Lookup(const Slice& key, uint32_t hash) {
HashTable::const_accessor accessor;
if (!table_.find(accessor, CacheKey(key, hash))) {
return nullptr;
}
CacheHandle* handle = accessor->second;
accessor.release();
// Ref() could fail if another thread sneak in and evict/erase the cache
// entry before we are able to hold reference.
if (!Ref(reinterpret_cast<Cache::Handle*>(handle))) {
return nullptr;
}
// Double check the key since the handle may now representing another key
// if other threads sneak in, evict/erase the entry and re-used the handle
// for another cache entry.
if (hash != handle->hash || key != handle->key) {
CleanupContext context;
Unref(handle, false, &context);
// It is possible Unref() delete the entry, so we need to cleanup.
Cleanup(context);
return nullptr;
}
return reinterpret_cast<Cache::Handle*>(handle);
}
bool ClockCacheShard::Release(Cache::Handle* h, bool force_erase) {
CleanupContext context;
CacheHandle* handle = reinterpret_cast<CacheHandle*>(h);
bool erased = Unref(handle, true, &context);
if (force_erase && !erased) {
erased = EraseAndConfirm(handle->key, handle->hash, &context);
}
Cleanup(context);
return erased;
}
void ClockCacheShard::Erase(const Slice& key, uint32_t hash) {
CleanupContext context;
EraseAndConfirm(key, hash, &context);
Cleanup(context);
}
bool ClockCacheShard::EraseAndConfirm(const Slice& key, uint32_t hash,
CleanupContext* context) {
MutexLock l(&mutex_);
HashTable::accessor accessor;
bool erased = false;
if (table_.find(accessor, CacheKey(key, hash))) {
CacheHandle* handle = accessor->second;
table_.erase(accessor);
erased = UnsetInCache(handle, context);
}
return erased;
}
void ClockCacheShard::EraseUnRefEntries() {
CleanupContext context;
{
MutexLock l(&mutex_);
table_.clear();
for (auto& handle : list_) {
UnsetInCache(&handle, &context);
}
}
Cleanup(context);
}
class ClockCache : public ShardedCache {
public:
ClockCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit)
: ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
int num_shards = 1 << num_shard_bits;
shards_ = new ClockCacheShard[num_shards];
SetCapacity(capacity);
SetStrictCapacityLimit(strict_capacity_limit);
}
virtual ~ClockCache() { delete[] shards_; }
virtual const char* Name() const override { return "ClockCache"; }
virtual CacheShard* GetShard(int shard) override {
return reinterpret_cast<CacheShard*>(&shards_[shard]);
}
virtual const CacheShard* GetShard(int shard) const override {
return reinterpret_cast<CacheShard*>(&shards_[shard]);
}
virtual void* Value(Handle* handle) override {
return reinterpret_cast<const CacheHandle*>(handle)->value;
}
virtual size_t GetCharge(Handle* handle) const override {
return reinterpret_cast<const CacheHandle*>(handle)->charge;
}
virtual uint32_t GetHash(Handle* handle) const override {
return reinterpret_cast<const CacheHandle*>(handle)->hash;
}
virtual void DisownData() override { shards_ = nullptr; }
private:
ClockCacheShard* shards_;
};
} // end anonymous namespace
std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
bool strict_capacity_limit) {
if (num_shard_bits < 0) {
num_shard_bits = GetDefaultCacheShardBits(capacity);
}
return std::make_shared<ClockCache>(capacity, num_shard_bits,
strict_capacity_limit);
}
} // namespace rocksdb
#endif // SUPPORT_CLOCK_CACHE

16
cache/clock_cache.h vendored Normal file
View File

@@ -0,0 +1,16 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include "rocksdb/cache.h"
#if defined(TBB) && !defined(ROCKSDB_LITE)
#define SUPPORT_CLOCK_CACHE
#endif

530
cache/lru_cache.cc vendored Normal file
View File

@@ -0,0 +1,530 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include "cache/lru_cache.h"
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include "util/mutexlock.h"
namespace rocksdb {
LRUHandleTable::LRUHandleTable() : list_(nullptr), length_(0), elems_(0) {
Resize();
}
LRUHandleTable::~LRUHandleTable() {
ApplyToAllCacheEntries([](LRUHandle* h) {
if (h->refs == 1) {
h->Free();
}
});
delete[] list_;
}
LRUHandle* LRUHandleTable::Lookup(const Slice& key, uint32_t hash) {
return *FindPointer(key, hash);
}
LRUHandle* LRUHandleTable::Insert(LRUHandle* h) {
LRUHandle** ptr = FindPointer(h->key(), h->hash);
LRUHandle* old = *ptr;
h->next_hash = (old == nullptr ? nullptr : old->next_hash);
*ptr = h;
if (old == nullptr) {
++elems_;
if (elems_ > length_) {
// Since each cache entry is fairly large, we aim for a small
// average linked list length (<= 1).
Resize();
}
}
return old;
}
LRUHandle* LRUHandleTable::Remove(const Slice& key, uint32_t hash) {
LRUHandle** ptr = FindPointer(key, hash);
LRUHandle* result = *ptr;
if (result != nullptr) {
*ptr = result->next_hash;
--elems_;
}
return result;
}
LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) {
LRUHandle** ptr = &list_[hash & (length_ - 1)];
while (*ptr != nullptr && ((*ptr)->hash != hash || key != (*ptr)->key())) {
ptr = &(*ptr)->next_hash;
}
return ptr;
}
void LRUHandleTable::Resize() {
uint32_t new_length = 16;
while (new_length < elems_ * 1.5) {
new_length *= 2;
}
LRUHandle** new_list = new LRUHandle*[new_length];
memset(new_list, 0, sizeof(new_list[0]) * new_length);
uint32_t count = 0;
for (uint32_t i = 0; i < length_; i++) {
LRUHandle* h = list_[i];
while (h != nullptr) {
LRUHandle* next = h->next_hash;
uint32_t hash = h->hash;
LRUHandle** ptr = &new_list[hash & (new_length - 1)];
h->next_hash = *ptr;
*ptr = h;
h = next;
count++;
}
}
assert(elems_ == count);
delete[] list_;
list_ = new_list;
length_ = new_length;
}
LRUCacheShard::LRUCacheShard()
: high_pri_pool_usage_(0), usage_(0), lru_usage_(0) {
// Make empty circular linked list
lru_.next = &lru_;
lru_.prev = &lru_;
lru_low_pri_ = &lru_;
}
LRUCacheShard::~LRUCacheShard() {}
bool LRUCacheShard::Unref(LRUHandle* e) {
assert(e->refs > 0);
e->refs--;
return e->refs == 0;
}
// Call deleter and free
void LRUCacheShard::EraseUnRefEntries() {
autovector<LRUHandle*> last_reference_list;
{
MutexLock l(&mutex_);
while (lru_.next != &lru_) {
LRUHandle* old = lru_.next;
assert(old->InCache());
assert(old->refs ==
1); // LRU list contains elements which may be evicted
LRU_Remove(old);
table_.Remove(old->key(), old->hash);
old->SetInCache(false);
Unref(old);
usage_ -= old->charge;
last_reference_list.push_back(old);
}
}
for (auto entry : last_reference_list) {
entry->Free();
}
}
void LRUCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
bool thread_safe) {
if (thread_safe) {
mutex_.Lock();
}
table_.ApplyToAllCacheEntries(
[callback](LRUHandle* h) { callback(h->value, h->charge); });
if (thread_safe) {
mutex_.Unlock();
}
}
void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
*lru = &lru_;
*lru_low_pri = lru_low_pri_;
}
size_t LRUCacheShard::TEST_GetLRUSize() {
LRUHandle* lru_handle = lru_.next;
size_t lru_size = 0;
while (lru_handle != &lru_) {
lru_size++;
lru_handle = lru_handle->next;
}
return lru_size;
}
void LRUCacheShard::LRU_Remove(LRUHandle* e) {
assert(e->next != nullptr);
assert(e->prev != nullptr);
if (lru_low_pri_ == e) {
lru_low_pri_ = e->prev;
}
e->next->prev = e->prev;
e->prev->next = e->next;
e->prev = e->next = nullptr;
lru_usage_ -= e->charge;
if (e->InHighPriPool()) {
assert(high_pri_pool_usage_ >= e->charge);
high_pri_pool_usage_ -= e->charge;
}
}
void LRUCacheShard::LRU_Insert(LRUHandle* e) {
assert(e->next == nullptr);
assert(e->prev == nullptr);
if (high_pri_pool_ratio_ > 0 && e->IsHighPri()) {
// Inset "e" to head of LRU list.
e->next = &lru_;
e->prev = lru_.prev;
e->prev->next = e;
e->next->prev = e;
e->SetInHighPriPool(true);
high_pri_pool_usage_ += e->charge;
MaintainPoolSize();
} else {
// Insert "e" to the head of low-pri pool. Note that when
// high_pri_pool_ratio is 0, head of low-pri pool is also head of LRU list.
e->next = lru_low_pri_->next;
e->prev = lru_low_pri_;
e->prev->next = e;
e->next->prev = e;
e->SetInHighPriPool(false);
lru_low_pri_ = e;
}
lru_usage_ += e->charge;
}
void LRUCacheShard::MaintainPoolSize() {
while (high_pri_pool_usage_ > high_pri_pool_capacity_) {
// Overflow last entry in high-pri pool to low-pri pool.
lru_low_pri_ = lru_low_pri_->next;
assert(lru_low_pri_ != &lru_);
lru_low_pri_->SetInHighPriPool(false);
high_pri_pool_usage_ -= lru_low_pri_->charge;
}
}
void LRUCacheShard::EvictFromLRU(size_t charge,
autovector<LRUHandle*>* deleted) {
while (usage_ + charge > capacity_ && lru_.next != &lru_) {
LRUHandle* old = lru_.next;
assert(old->InCache());
assert(old->refs == 1); // LRU list contains elements which may be evicted
LRU_Remove(old);
table_.Remove(old->key(), old->hash);
old->SetInCache(false);
Unref(old);
usage_ -= old->charge;
deleted->push_back(old);
}
}
void* LRUCacheShard::operator new(size_t size) {
return port::cacheline_aligned_alloc(size);
}
void* LRUCacheShard::operator new[](size_t size) {
return port::cacheline_aligned_alloc(size);
}
void LRUCacheShard::operator delete(void *memblock) {
port::cacheline_aligned_free(memblock);
}
void LRUCacheShard::operator delete[](void* memblock) {
port::cacheline_aligned_free(memblock);
}
void LRUCacheShard::SetCapacity(size_t capacity) {
autovector<LRUHandle*> last_reference_list;
{
MutexLock l(&mutex_);
capacity_ = capacity;
high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
EvictFromLRU(0, &last_reference_list);
}
// we free the entries here outside of mutex for
// performance reasons
for (auto entry : last_reference_list) {
entry->Free();
}
}
void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
MutexLock l(&mutex_);
strict_capacity_limit_ = strict_capacity_limit;
}
Cache::Handle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash) {
MutexLock l(&mutex_);
LRUHandle* e = table_.Lookup(key, hash);
if (e != nullptr) {
assert(e->InCache());
if (e->refs == 1) {
LRU_Remove(e);
}
e->refs++;
}
return reinterpret_cast<Cache::Handle*>(e);
}
bool LRUCacheShard::Ref(Cache::Handle* h) {
LRUHandle* handle = reinterpret_cast<LRUHandle*>(h);
MutexLock l(&mutex_);
if (handle->InCache() && handle->refs == 1) {
LRU_Remove(handle);
}
handle->refs++;
return true;
}
void LRUCacheShard::SetHighPriorityPoolRatio(double high_pri_pool_ratio) {
MutexLock l(&mutex_);
high_pri_pool_ratio_ = high_pri_pool_ratio;
high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
MaintainPoolSize();
}
bool LRUCacheShard::Release(Cache::Handle* handle, bool force_erase) {
if (handle == nullptr) {
return false;
}
LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
bool last_reference = false;
{
MutexLock l(&mutex_);
last_reference = Unref(e);
if (last_reference) {
usage_ -= e->charge;
}
if (e->refs == 1 && e->InCache()) {
// The item is still in cache, and nobody else holds a reference to it
if (usage_ > capacity_ || force_erase) {
// the cache is full
// The LRU list must be empty since the cache is full
assert(!(usage_ > capacity_) || lru_.next == &lru_);
// take this opportunity and remove the item
table_.Remove(e->key(), e->hash);
e->SetInCache(false);
Unref(e);
usage_ -= e->charge;
last_reference = true;
} else {
// put the item on the list to be potentially freed
LRU_Insert(e);
}
}
}
// free outside of mutex
if (last_reference) {
e->Free();
}
return last_reference;
}
Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
size_t charge,
void (*deleter)(const Slice& key, void* value),
Cache::Handle** handle, Cache::Priority priority) {
// Allocate the memory here outside of the mutex
// If the cache is full, we'll have to release it
// It shouldn't happen very often though.
LRUHandle* e = reinterpret_cast<LRUHandle*>(
new char[sizeof(LRUHandle) - 1 + key.size()]);
Status s;
autovector<LRUHandle*> last_reference_list;
e->value = value;
e->deleter = deleter;
e->charge = charge;
e->key_length = key.size();
e->hash = hash;
e->refs = (handle == nullptr
? 1
: 2); // One from LRUCache, one for the returned handle
e->next = e->prev = nullptr;
e->SetInCache(true);
e->SetPriority(priority);
memcpy(e->key_data, key.data(), key.size());
{
MutexLock l(&mutex_);
// Free the space following strict LRU policy until enough space
// is freed or the lru list is empty
EvictFromLRU(charge, &last_reference_list);
if (usage_ - lru_usage_ + charge > capacity_ &&
(strict_capacity_limit_ || handle == nullptr)) {
if (handle == nullptr) {
// Don't insert the entry but still return ok, as if the entry inserted
// into cache and get evicted immediately.
last_reference_list.push_back(e);
} else {
delete[] reinterpret_cast<char*>(e);
*handle = nullptr;
s = Status::Incomplete("Insert failed due to LRU cache being full.");
}
} else {
// insert into the cache
// note that the cache might get larger than its capacity if not enough
// space was freed
LRUHandle* old = table_.Insert(e);
usage_ += e->charge;
if (old != nullptr) {
old->SetInCache(false);
if (Unref(old)) {
usage_ -= old->charge;
// old is on LRU because it's in cache and its reference count
// was just 1 (Unref returned 0)
LRU_Remove(old);
last_reference_list.push_back(old);
}
}
if (handle == nullptr) {
LRU_Insert(e);
} else {
*handle = reinterpret_cast<Cache::Handle*>(e);
}
s = Status::OK();
}
}
// we free the entries here outside of mutex for
// performance reasons
for (auto entry : last_reference_list) {
entry->Free();
}
return s;
}
void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
LRUHandle* e;
bool last_reference = false;
{
MutexLock l(&mutex_);
e = table_.Remove(key, hash);
if (e != nullptr) {
last_reference = Unref(e);
if (last_reference) {
usage_ -= e->charge;
}
if (last_reference && e->InCache()) {
LRU_Remove(e);
}
e->SetInCache(false);
}
}
// mutex not held here
// last_reference will only be true if e != nullptr
if (last_reference) {
e->Free();
}
}
size_t LRUCacheShard::GetUsage() const {
MutexLock l(&mutex_);
return usage_;
}
size_t LRUCacheShard::GetPinnedUsage() const {
MutexLock l(&mutex_);
assert(usage_ >= lru_usage_);
return usage_ - lru_usage_;
}
std::string LRUCacheShard::GetPrintableOptions() const {
const int kBufferSize = 200;
char buffer[kBufferSize];
{
MutexLock l(&mutex_);
snprintf(buffer, kBufferSize, " high_pri_pool_ratio: %.3lf\n",
high_pri_pool_ratio_);
}
return std::string(buffer);
}
LRUCache::LRUCache(size_t capacity, int num_shard_bits,
bool strict_capacity_limit, double high_pri_pool_ratio)
: ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
num_shards_ = 1 << num_shard_bits;
shards_ = new LRUCacheShard[num_shards_];
SetCapacity(capacity);
SetStrictCapacityLimit(strict_capacity_limit);
for (int i = 0; i < num_shards_; i++) {
shards_[i].SetHighPriorityPoolRatio(high_pri_pool_ratio);
}
}
LRUCache::~LRUCache() { delete[] shards_; }
CacheShard* LRUCache::GetShard(int shard) {
return reinterpret_cast<CacheShard*>(&shards_[shard]);
}
const CacheShard* LRUCache::GetShard(int shard) const {
return reinterpret_cast<CacheShard*>(&shards_[shard]);
}
void* LRUCache::Value(Handle* handle) {
return reinterpret_cast<const LRUHandle*>(handle)->value;
}
size_t LRUCache::GetCharge(Handle* handle) const {
return reinterpret_cast<const LRUHandle*>(handle)->charge;
}
uint32_t LRUCache::GetHash(Handle* handle) const {
return reinterpret_cast<const LRUHandle*>(handle)->hash;
}
void LRUCache::DisownData() {
// Do not drop data if compile with ASAN to suppress leak warning.
#ifndef __SANITIZE_ADDRESS__
shards_ = nullptr;
#endif // !__SANITIZE_ADDRESS__
}
size_t LRUCache::TEST_GetLRUSize() {
size_t lru_size_of_all_shards = 0;
for (int i = 0; i < num_shards_; i++) {
lru_size_of_all_shards += shards_[i].TEST_GetLRUSize();
}
return lru_size_of_all_shards;
}
std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
bool strict_capacity_limit,
double high_pri_pool_ratio) {
if (num_shard_bits >= 20) {
return nullptr; // the cache cannot be sharded into too many fine pieces
}
if (high_pri_pool_ratio < 0.0 || high_pri_pool_ratio > 1.0) {
// invalid high_pri_pool_ratio
return nullptr;
}
if (num_shard_bits < 0) {
num_shard_bits = GetDefaultCacheShardBits(capacity);
}
return std::make_shared<LRUCache>(capacity, num_shard_bits,
strict_capacity_limit, high_pri_pool_ratio);
}
} // namespace rocksdb

302
cache/lru_cache.h vendored Normal file
View File

@@ -0,0 +1,302 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include <string>
#include "cache/sharded_cache.h"
#include "port/port.h"
#include "util/autovector.h"
namespace rocksdb {
// LRU cache implementation
// An entry is a variable length heap-allocated structure.
// Entries are referenced by cache and/or by any external entity.
// The cache keeps all its entries in table. Some elements
// are also stored on LRU list.
//
// LRUHandle can be in these states:
// 1. Referenced externally AND in hash table.
// In that case the entry is *not* in the LRU. (refs > 1 && in_cache == true)
// 2. Not referenced externally and in hash table. In that case the entry is
// in the LRU and can be freed. (refs == 1 && in_cache == true)
// 3. Referenced externally and not in hash table. In that case the entry is
// in not on LRU and not in table. (refs >= 1 && in_cache == false)
//
// All newly created LRUHandles are in state 1. If you call
// LRUCacheShard::Release
// on entry in state 1, it will go into state 2. To move from state 1 to
// state 3, either call LRUCacheShard::Erase or LRUCacheShard::Insert with the
// same key.
// To move from state 2 to state 1, use LRUCacheShard::Lookup.
// Before destruction, make sure that no handles are in state 1. This means
// that any successful LRUCacheShard::Lookup/LRUCacheShard::Insert have a
// matching
// RUCache::Release (to move into state 2) or LRUCacheShard::Erase (for state 3)
struct LRUHandle {
void* value;
void (*deleter)(const Slice&, void* value);
LRUHandle* next_hash;
LRUHandle* next;
LRUHandle* prev;
size_t charge; // TODO(opt): Only allow uint32_t?
size_t key_length;
uint32_t refs; // a number of refs to this entry
// cache itself is counted as 1
// Include the following flags:
// in_cache: whether this entry is referenced by the hash table.
// is_high_pri: whether this entry is high priority entry.
// in_high_pro_pool: whether this entry is in high-pri pool.
char flags;
uint32_t hash; // Hash of key(); used for fast sharding and comparisons
char key_data[1]; // Beginning of key
Slice key() const {
// For cheaper lookups, we allow a temporary Handle object
// to store a pointer to a key in "value".
if (next == this) {
return *(reinterpret_cast<Slice*>(value));
} else {
return Slice(key_data, key_length);
}
}
bool InCache() { return flags & 1; }
bool IsHighPri() { return flags & 2; }
bool InHighPriPool() { return flags & 4; }
void SetInCache(bool in_cache) {
if (in_cache) {
flags |= 1;
} else {
flags &= ~1;
}
}
void SetPriority(Cache::Priority priority) {
if (priority == Cache::Priority::HIGH) {
flags |= 2;
} else {
flags &= ~2;
}
}
void SetInHighPriPool(bool in_high_pri_pool) {
if (in_high_pri_pool) {
flags |= 4;
} else {
flags &= ~4;
}
}
void Free() {
assert((refs == 1 && InCache()) || (refs == 0 && !InCache()));
if (deleter) {
(*deleter)(key(), value);
}
delete[] reinterpret_cast<char*>(this);
}
};
// We provide our own simple hash table since it removes a whole bunch
// of porting hacks and is also faster than some of the built-in hash
// table implementations in some of the compiler/runtime combinations
// we have tested. E.g., readrandom speeds up by ~5% over the g++
// 4.4.3's builtin hashtable.
class LRUHandleTable {
public:
LRUHandleTable();
~LRUHandleTable();
LRUHandle* Lookup(const Slice& key, uint32_t hash);
LRUHandle* Insert(LRUHandle* h);
LRUHandle* Remove(const Slice& key, uint32_t hash);
template <typename T>
void ApplyToAllCacheEntries(T func) {
for (uint32_t i = 0; i < length_; i++) {
LRUHandle* h = list_[i];
while (h != nullptr) {
auto n = h->next_hash;
assert(h->InCache());
func(h);
h = n;
}
}
}
private:
// Return a pointer to slot that points to a cache entry that
// matches key/hash. If there is no such cache entry, return a
// pointer to the trailing slot in the corresponding linked list.
LRUHandle** FindPointer(const Slice& key, uint32_t hash);
void Resize();
// The table consists of an array of buckets where each bucket is
// a linked list of cache entries that hash into the bucket.
LRUHandle** list_;
uint32_t length_;
uint32_t elems_;
};
// A single shard of sharded cache.
class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
public:
LRUCacheShard();
virtual ~LRUCacheShard();
// Separate from constructor so caller can easily make an array of LRUCache
// if current usage is more than new capacity, the function will attempt to
// free the needed space
virtual void SetCapacity(size_t capacity) override;
// Set the flag to reject insertion if cache if full.
virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
// Set percentage of capacity reserved for high-pri cache entries.
void SetHighPriorityPoolRatio(double high_pri_pool_ratio);
// Like Cache methods, but with an extra "hash" parameter.
virtual Status Insert(const Slice& key, uint32_t hash, void* value,
size_t charge,
void (*deleter)(const Slice& key, void* value),
Cache::Handle** handle,
Cache::Priority priority) override;
virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
virtual bool Ref(Cache::Handle* handle) override;
virtual bool Release(Cache::Handle* handle,
bool force_erase = false) override;
virtual void Erase(const Slice& key, uint32_t hash) override;
// Although in some platforms the update of size_t is atomic, to make sure
// GetUsage() and GetPinnedUsage() work correctly under any platform, we'll
// protect them with mutex_.
virtual size_t GetUsage() const override;
virtual size_t GetPinnedUsage() const override;
virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
bool thread_safe) override;
virtual void EraseUnRefEntries() override;
virtual std::string GetPrintableOptions() const override;
void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri);
// Retrieves number of elements in LRU, for unit test purpose only
// not threadsafe
size_t TEST_GetLRUSize();
// Overloading to aligned it to cache line size
void* operator new(size_t);
void* operator new[](size_t);
void operator delete(void *);
void operator delete[](void*);
private:
void LRU_Remove(LRUHandle* e);
void LRU_Insert(LRUHandle* e);
// Overflow the last entry in high-pri pool to low-pri pool until size of
// high-pri pool is no larger than the size specify by high_pri_pool_pct.
void MaintainPoolSize();
// Just reduce the reference count by 1.
// Return true if last reference
bool Unref(LRUHandle* e);
// Free some space following strict LRU policy until enough space
// to hold (usage_ + charge) is freed or the lru list is empty
// This function is not thread safe - it needs to be executed while
// holding the mutex_
void EvictFromLRU(size_t charge, autovector<LRUHandle*>* deleted);
// Initialized before use.
size_t capacity_;
// Memory size for entries in high-pri pool.
size_t high_pri_pool_usage_;
// Whether to reject insertion if cache reaches its full capacity.
bool strict_capacity_limit_;
// Ratio of capacity reserved for high priority cache entries.
double high_pri_pool_ratio_;
// High-pri pool size, equals to capacity * high_pri_pool_ratio.
// Remember the value to avoid recomputing each time.
double high_pri_pool_capacity_;
// Dummy head of LRU list.
// lru.prev is newest entry, lru.next is oldest entry.
// LRU contains items which can be evicted, ie reference only by cache
LRUHandle lru_;
// Pointer to head of low-pri pool in LRU list.
LRUHandle* lru_low_pri_;
// ------------^^^^^^^^^^^^^-----------
// Not frequently modified data members
// ------------------------------------
//
// We separate data members that are updated frequently from the ones that
// are not frequently updated so that they don't share the same cache line
// which will lead into false cache sharing
//
// ------------------------------------
// Frequently modified data members
// ------------vvvvvvvvvvvvv-----------
LRUHandleTable table_;
// Memory size for entries residing in the cache
size_t usage_;
// Memory size for entries residing only in the LRU list
size_t lru_usage_;
// mutex_ protects the following state.
// We don't count mutex_ as the cache's internal state so semantically we
// don't mind mutex_ invoking the non-const actions.
mutable port::Mutex mutex_;
};
class LRUCache : public ShardedCache {
public:
LRUCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
double high_pri_pool_ratio);
virtual ~LRUCache();
virtual const char* Name() const override { return "LRUCache"; }
virtual CacheShard* GetShard(int shard) override;
virtual const CacheShard* GetShard(int shard) const override;
virtual void* Value(Handle* handle) override;
virtual size_t GetCharge(Handle* handle) const override;
virtual uint32_t GetHash(Handle* handle) const override;
virtual void DisownData() override;
// Retrieves number of elements in LRU, for unit test purpose only
size_t TEST_GetLRUSize();
private:
LRUCacheShard* shards_;
int num_shards_ = 0;
};
} // namespace rocksdb

172
cache/lru_cache_test.cc vendored Normal file
View File

@@ -0,0 +1,172 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "cache/lru_cache.h"
#include <string>
#include <vector>
#include "util/testharness.h"
namespace rocksdb {
class LRUCacheTest : public testing::Test {
public:
LRUCacheTest() {}
~LRUCacheTest() {}
void NewCache(size_t capacity, double high_pri_pool_ratio = 0.0) {
cache_.reset(
#if defined(_MSC_VER)
#pragma warning(push)
#pragma warning(disable: 4316) // We've validated the alignment with the new operators
#endif
new LRUCacheShard()
#if defined(_MSC_VER)
#pragma warning(pop)
#endif
);
cache_->SetCapacity(capacity);
cache_->SetStrictCapacityLimit(false);
cache_->SetHighPriorityPoolRatio(high_pri_pool_ratio);
}
void Insert(const std::string& key,
Cache::Priority priority = Cache::Priority::LOW) {
cache_->Insert(key, 0 /*hash*/, nullptr /*value*/, 1 /*charge*/,
nullptr /*deleter*/, nullptr /*handle*/, priority);
}
void Insert(char key, Cache::Priority priority = Cache::Priority::LOW) {
Insert(std::string(1, key), priority);
}
bool Lookup(const std::string& key) {
auto handle = cache_->Lookup(key, 0 /*hash*/);
if (handle) {
cache_->Release(handle);
return true;
}
return false;
}
bool Lookup(char key) { return Lookup(std::string(1, key)); }
void Erase(const std::string& key) { cache_->Erase(key, 0 /*hash*/); }
void ValidateLRUList(std::vector<std::string> keys,
size_t num_high_pri_pool_keys = 0) {
LRUHandle* lru;
LRUHandle* lru_low_pri;
cache_->TEST_GetLRUList(&lru, &lru_low_pri);
LRUHandle* iter = lru;
bool in_high_pri_pool = false;
size_t high_pri_pool_keys = 0;
if (iter == lru_low_pri) {
in_high_pri_pool = true;
}
for (const auto& key : keys) {
iter = iter->next;
ASSERT_NE(lru, iter);
ASSERT_EQ(key, iter->key().ToString());
ASSERT_EQ(in_high_pri_pool, iter->InHighPriPool());
if (in_high_pri_pool) {
high_pri_pool_keys++;
}
if (iter == lru_low_pri) {
ASSERT_FALSE(in_high_pri_pool);
in_high_pri_pool = true;
}
}
ASSERT_EQ(lru, iter->next);
ASSERT_TRUE(in_high_pri_pool);
ASSERT_EQ(num_high_pri_pool_keys, high_pri_pool_keys);
}
private:
std::unique_ptr<LRUCacheShard> cache_;
};
TEST_F(LRUCacheTest, BasicLRU) {
NewCache(5);
for (char ch = 'a'; ch <= 'e'; ch++) {
Insert(ch);
}
ValidateLRUList({"a", "b", "c", "d", "e"});
for (char ch = 'x'; ch <= 'z'; ch++) {
Insert(ch);
}
ValidateLRUList({"d", "e", "x", "y", "z"});
ASSERT_FALSE(Lookup("b"));
ValidateLRUList({"d", "e", "x", "y", "z"});
ASSERT_TRUE(Lookup("e"));
ValidateLRUList({"d", "x", "y", "z", "e"});
ASSERT_TRUE(Lookup("z"));
ValidateLRUList({"d", "x", "y", "e", "z"});
Erase("x");
ValidateLRUList({"d", "y", "e", "z"});
ASSERT_TRUE(Lookup("d"));
ValidateLRUList({"y", "e", "z", "d"});
Insert("u");
ValidateLRUList({"y", "e", "z", "d", "u"});
Insert("v");
ValidateLRUList({"e", "z", "d", "u", "v"});
}
TEST_F(LRUCacheTest, MidPointInsertion) {
// Allocate 2 cache entries to high-pri pool.
NewCache(5, 0.45);
Insert("a", Cache::Priority::LOW);
Insert("b", Cache::Priority::LOW);
Insert("c", Cache::Priority::LOW);
ValidateLRUList({"a", "b", "c"}, 0);
// Low-pri entries can take high-pri pool capacity if available
Insert("u", Cache::Priority::LOW);
Insert("v", Cache::Priority::LOW);
ValidateLRUList({"a", "b", "c", "u", "v"}, 0);
Insert("X", Cache::Priority::HIGH);
Insert("Y", Cache::Priority::HIGH);
ValidateLRUList({"c", "u", "v", "X", "Y"}, 2);
// High-pri entries can overflow to low-pri pool.
Insert("Z", Cache::Priority::HIGH);
ValidateLRUList({"u", "v", "X", "Y", "Z"}, 2);
// Low-pri entries will be inserted to head of low-pri pool.
Insert("a", Cache::Priority::LOW);
ValidateLRUList({"v", "X", "a", "Y", "Z"}, 2);
// Low-pri entries will be inserted to head of low-pri pool after lookup.
ASSERT_TRUE(Lookup("v"));
ValidateLRUList({"X", "a", "v", "Y", "Z"}, 2);
// High-pri entries will be inserted to the head of the list after lookup.
ASSERT_TRUE(Lookup("X"));
ValidateLRUList({"a", "v", "Y", "Z", "X"}, 2);
ASSERT_TRUE(Lookup("Z"));
ValidateLRUList({"a", "v", "Y", "X", "Z"}, 2);
Erase("Y");
ValidateLRUList({"a", "v", "X", "Z"}, 2);
Erase("X");
ValidateLRUList({"a", "v", "Z"}, 1);
Insert("d", Cache::Priority::LOW);
Insert("e", Cache::Priority::LOW);
ValidateLRUList({"a", "v", "d", "e", "Z"}, 1);
Insert("f", Cache::Priority::LOW);
Insert("g", Cache::Priority::LOW);
ValidateLRUList({"d", "e", "f", "g", "Z"}, 1);
ASSERT_TRUE(Lookup("d"));
ValidateLRUList({"e", "f", "g", "d", "Z"}, 1);
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

161
cache/sharded_cache.cc vendored Normal file
View File

@@ -0,0 +1,161 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include "cache/sharded_cache.h"
#include <string>
#include "util/mutexlock.h"
namespace rocksdb {
ShardedCache::ShardedCache(size_t capacity, int num_shard_bits,
bool strict_capacity_limit)
: num_shard_bits_(num_shard_bits),
capacity_(capacity),
strict_capacity_limit_(strict_capacity_limit),
last_id_(1) {}
void ShardedCache::SetCapacity(size_t capacity) {
int num_shards = 1 << num_shard_bits_;
const size_t per_shard = (capacity + (num_shards - 1)) / num_shards;
MutexLock l(&capacity_mutex_);
for (int s = 0; s < num_shards; s++) {
GetShard(s)->SetCapacity(per_shard);
}
capacity_ = capacity;
}
void ShardedCache::SetStrictCapacityLimit(bool strict_capacity_limit) {
int num_shards = 1 << num_shard_bits_;
MutexLock l(&capacity_mutex_);
for (int s = 0; s < num_shards; s++) {
GetShard(s)->SetStrictCapacityLimit(strict_capacity_limit);
}
strict_capacity_limit_ = strict_capacity_limit;
}
Status ShardedCache::Insert(const Slice& key, void* value, size_t charge,
void (*deleter)(const Slice& key, void* value),
Handle** handle, Priority priority) {
uint32_t hash = HashSlice(key);
return GetShard(Shard(hash))
->Insert(key, hash, value, charge, deleter, handle, priority);
}
Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* stats) {
uint32_t hash = HashSlice(key);
return GetShard(Shard(hash))->Lookup(key, hash);
}
bool ShardedCache::Ref(Handle* handle) {
uint32_t hash = GetHash(handle);
return GetShard(Shard(hash))->Ref(handle);
}
bool ShardedCache::Release(Handle* handle, bool force_erase) {
uint32_t hash = GetHash(handle);
return GetShard(Shard(hash))->Release(handle, force_erase);
}
void ShardedCache::Erase(const Slice& key) {
uint32_t hash = HashSlice(key);
GetShard(Shard(hash))->Erase(key, hash);
}
uint64_t ShardedCache::NewId() {
return last_id_.fetch_add(1, std::memory_order_relaxed);
}
size_t ShardedCache::GetCapacity() const {
MutexLock l(&capacity_mutex_);
return capacity_;
}
bool ShardedCache::HasStrictCapacityLimit() const {
MutexLock l(&capacity_mutex_);
return strict_capacity_limit_;
}
size_t ShardedCache::GetUsage() const {
// We will not lock the cache when getting the usage from shards.
int num_shards = 1 << num_shard_bits_;
size_t usage = 0;
for (int s = 0; s < num_shards; s++) {
usage += GetShard(s)->GetUsage();
}
return usage;
}
size_t ShardedCache::GetUsage(Handle* handle) const {
return GetCharge(handle);
}
size_t ShardedCache::GetPinnedUsage() const {
// We will not lock the cache when getting the usage from shards.
int num_shards = 1 << num_shard_bits_;
size_t usage = 0;
for (int s = 0; s < num_shards; s++) {
usage += GetShard(s)->GetPinnedUsage();
}
return usage;
}
void ShardedCache::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
bool thread_safe) {
int num_shards = 1 << num_shard_bits_;
for (int s = 0; s < num_shards; s++) {
GetShard(s)->ApplyToAllCacheEntries(callback, thread_safe);
}
}
void ShardedCache::EraseUnRefEntries() {
int num_shards = 1 << num_shard_bits_;
for (int s = 0; s < num_shards; s++) {
GetShard(s)->EraseUnRefEntries();
}
}
std::string ShardedCache::GetPrintableOptions() const {
std::string ret;
ret.reserve(20000);
const int kBufferSize = 200;
char buffer[kBufferSize];
{
MutexLock l(&capacity_mutex_);
snprintf(buffer, kBufferSize, " capacity : %" ROCKSDB_PRIszt "\n",
capacity_);
ret.append(buffer);
snprintf(buffer, kBufferSize, " num_shard_bits : %d\n", num_shard_bits_);
ret.append(buffer);
snprintf(buffer, kBufferSize, " strict_capacity_limit : %d\n",
strict_capacity_limit_);
ret.append(buffer);
}
ret.append(GetShard(0)->GetPrintableOptions());
return ret;
}
int GetDefaultCacheShardBits(size_t capacity) {
int num_shard_bits = 0;
size_t min_shard_size = 512L * 1024L; // Every shard is at least 512KB.
size_t num_shards = capacity / min_shard_size;
while (num_shards >>= 1) {
if (++num_shard_bits >= 6) {
// No more than 6.
return num_shard_bits;
}
}
return num_shard_bits;
}
} // namespace rocksdb

102
cache/sharded_cache.h vendored Normal file
View File

@@ -0,0 +1,102 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include <atomic>
#include <string>
#include "port/port.h"
#include "rocksdb/cache.h"
#include "util/hash.h"
namespace rocksdb {
// Single cache shard interface.
class CacheShard {
public:
CacheShard() = default;
virtual ~CacheShard() = default;
virtual Status Insert(const Slice& key, uint32_t hash, void* value,
size_t charge,
void (*deleter)(const Slice& key, void* value),
Cache::Handle** handle, Cache::Priority priority) = 0;
virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) = 0;
virtual bool Ref(Cache::Handle* handle) = 0;
virtual bool Release(Cache::Handle* handle, bool force_erase = false) = 0;
virtual void Erase(const Slice& key, uint32_t hash) = 0;
virtual void SetCapacity(size_t capacity) = 0;
virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
virtual size_t GetUsage() const = 0;
virtual size_t GetPinnedUsage() const = 0;
virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
bool thread_safe) = 0;
virtual void EraseUnRefEntries() = 0;
virtual std::string GetPrintableOptions() const { return ""; }
};
// Generic cache interface which shards cache by hash of keys. 2^num_shard_bits
// shards will be created, with capacity split evenly to each of the shards.
// Keys are sharded by the highest num_shard_bits bits of hash value.
class ShardedCache : public Cache {
public:
ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit);
virtual ~ShardedCache() = default;
virtual const char* Name() const override = 0;
virtual CacheShard* GetShard(int shard) = 0;
virtual const CacheShard* GetShard(int shard) const = 0;
virtual void* Value(Handle* handle) override = 0;
virtual size_t GetCharge(Handle* handle) const = 0;
virtual uint32_t GetHash(Handle* handle) const = 0;
virtual void DisownData() override = 0;
virtual void SetCapacity(size_t capacity) override;
virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
virtual Status Insert(const Slice& key, void* value, size_t charge,
void (*deleter)(const Slice& key, void* value),
Handle** handle, Priority priority) override;
virtual Handle* Lookup(const Slice& key, Statistics* stats) override;
virtual bool Ref(Handle* handle) override;
virtual bool Release(Handle* handle, bool force_erase = false) override;
virtual void Erase(const Slice& key) override;
virtual uint64_t NewId() override;
virtual size_t GetCapacity() const override;
virtual bool HasStrictCapacityLimit() const override;
virtual size_t GetUsage() const override;
virtual size_t GetUsage(Handle* handle) const override;
virtual size_t GetPinnedUsage() const override;
virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
bool thread_safe) override;
virtual void EraseUnRefEntries() override;
virtual std::string GetPrintableOptions() const override;
int GetNumShardBits() const { return num_shard_bits_; }
private:
static inline uint32_t HashSlice(const Slice& s) {
return Hash(s.data(), s.size(), 0);
}
uint32_t Shard(uint32_t hash) {
// Note, hash >> 32 yields hash in gcc, not the zero we expect!
return (num_shard_bits_ > 0) ? (hash >> (32 - num_shard_bits_)) : 0;
}
int num_shard_bits_;
mutable port::Mutex capacity_mutex_;
size_t capacity_;
bool strict_capacity_limit_;
std::atomic<uint64_t> last_id_;
};
extern int GetDefaultCacheShardBits(size_t capacity);
} // namespace rocksdb

3
cmake/RocksDBConfig.cmake.in Normal file
View File

@@ -0,0 +1,3 @@
@PACKAGE_INIT@
include("${CMAKE_CURRENT_LIST_DIR}/RocksDBTargets.cmake")
check_required_components(RocksDB)

21
cmake/modules/FindJeMalloc.cmake Normal file
View File

@@ -0,0 +1,21 @@
# - Find JeMalloc library
# Find the native JeMalloc includes and library
#
# JEMALLOC_INCLUDE_DIR - where to find jemalloc.h, etc.
# JEMALLOC_LIBRARIES - List of libraries when using jemalloc.
# JEMALLOC_FOUND - True if jemalloc found.
find_path(JEMALLOC_INCLUDE_DIR
NAMES jemalloc/jemalloc.h
HINTS ${JEMALLOC_ROOT_DIR}/include)
find_library(JEMALLOC_LIBRARIES
NAMES jemalloc
HINTS ${JEMALLOC_ROOT_DIR}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(jemalloc DEFAULT_MSG JEMALLOC_LIBRARIES JEMALLOC_INCLUDE_DIR)
mark_as_advanced(
JEMALLOC_LIBRARIES
JEMALLOC_INCLUDE_DIR)

21
cmake/modules/Findbzip2.cmake Normal file
View File

@@ -0,0 +1,21 @@
# - Find Bzip2
# Find the bzip2 compression library and includes
#
# BZIP2_INCLUDE_DIR - where to find bzlib.h, etc.
# BZIP2_LIBRARIES - List of libraries when using bzip2.
# BZIP2_FOUND - True if bzip2 found.
find_path(BZIP2_INCLUDE_DIR
NAMES bzlib.h
HINTS ${BZIP2_ROOT_DIR}/include)
find_library(BZIP2_LIBRARIES
NAMES bz2
HINTS ${BZIP2_ROOT_DIR}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(bzip2 DEFAULT_MSG BZIP2_LIBRARIES BZIP2_INCLUDE_DIR)
mark_as_advanced(
BZIP2_LIBRARIES
BZIP2_INCLUDE_DIR)

21
cmake/modules/Findlz4.cmake Normal file
View File

@@ -0,0 +1,21 @@
# - Find Lz4
# Find the lz4 compression library and includes
#
# LZ4_INCLUDE_DIR - where to find lz4.h, etc.
# LZ4_LIBRARIES - List of libraries when using lz4.
# LZ4_FOUND - True if lz4 found.
find_path(LZ4_INCLUDE_DIR
NAMES lz4.h
HINTS ${LZ4_ROOT_DIR}/include)
find_library(LZ4_LIBRARIES
NAMES lz4
HINTS ${LZ4_ROOT_DIR}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(lz4 DEFAULT_MSG LZ4_LIBRARIES LZ4_INCLUDE_DIR)
mark_as_advanced(
LZ4_LIBRARIES
LZ4_INCLUDE_DIR)

21
cmake/modules/Findsnappy.cmake Normal file
View File

@@ -0,0 +1,21 @@
# - Find Snappy
# Find the snappy compression library and includes
#
# SNAPPY_INCLUDE_DIR - where to find snappy.h, etc.
# SNAPPY_LIBRARIES - List of libraries when using snappy.
# SNAPPY_FOUND - True if snappy found.
find_path(SNAPPY_INCLUDE_DIR
NAMES snappy.h
HINTS ${SNAPPY_ROOT_DIR}/include)
find_library(SNAPPY_LIBRARIES
NAMES snappy
HINTS ${SNAPPY_ROOT_DIR}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(snappy DEFAULT_MSG SNAPPY_LIBRARIES SNAPPY_INCLUDE_DIR)
mark_as_advanced(
SNAPPY_LIBRARIES
SNAPPY_INCLUDE_DIR)

21
cmake/modules/Findzlib.cmake Normal file
View File

@@ -0,0 +1,21 @@
# - Find zlib
# Find the zlib compression library and includes
#
# ZLIB_INCLUDE_DIR - where to find zlib.h, etc.
# ZLIB_LIBRARIES - List of libraries when using zlib.
# ZLIB_FOUND - True if zlib found.
find_path(ZLIB_INCLUDE_DIR
NAMES zlib.h
HINTS ${ZLIB_ROOT_DIR}/include)
find_library(ZLIB_LIBRARIES
NAMES z
HINTS ${ZLIB_ROOT_DIR}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(zlib DEFAULT_MSG ZLIB_LIBRARIES ZLIB_INCLUDE_DIR)
mark_as_advanced(
ZLIB_LIBRARIES
ZLIB_INCLUDE_DIR)

21
cmake/modules/Findzstd.cmake Normal file
View File

@@ -0,0 +1,21 @@
# - Find zstd
# Find the zstd compression library and includes
#
# ZSTD_INCLUDE_DIR - where to find zstd.h, etc.
# ZSTD_LIBRARIES - List of libraries when using zstd.
# ZSTD_FOUND - True if zstd found.
find_path(ZSTD_INCLUDE_DIR
NAMES zstd.h
HINTS ${ZSTD_ROOT_DIR}/include)
find_library(ZSTD_LIBRARIES
NAMES zstd
HINTS ${ZSTD_ROOT_DIR}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(zstd DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIR)
mark_as_advanced(
ZSTD_LIBRARIES
ZSTD_INCLUDE_DIR)

6
coverage/coverage_test.sh
View File

@@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Exit on error.
set -e
@@ -11,8 +11,8 @@ fi
ROOT=".."
# Fetch right version of gcov
if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
source $ROOT/build_tools/fbcode.gcc471.sh
GCOV=$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1/cc6c9dc/bin/gcov
source $ROOT/build_tools/fbcode_config.sh
GCOV=$GCC_BASE/bin/gcov
else
GCOV=$(which gcov)
fi

335
db/builder.cc
View File

@@ -1,7 +1,7 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
@@ -9,212 +9,201 @@
#include "db/builder.h"
#include <algorithm>
#include <deque>
#include <vector>
#include "db/compaction_iterator.h"
#include "db/dbformat.h"
#include "db/filename.h"
#include "db/event_helpers.h"
#include "db/internal_stats.h"
#include "db/merge_helper.h"
#include "db/table_cache.h"
#include "db/version_edit.h"
#include "monitoring/iostats_context_imp.h"
#include "monitoring/thread_status_util.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "rocksdb/options.h"
#include "rocksdb/table.h"
#include "table/block_based_table_builder.h"
#include "table/internal_iterator.h"
#include "util/file_reader_writer.h"
#include "util/filename.h"
#include "util/stop_watch.h"
#include "util/sync_point.h"
namespace rocksdb {
class TableFactory;
TableBuilder* NewTableBuilder(const Options& options,
const InternalKeyComparator& internal_comparator,
WritableFile* file,
CompressionType compression_type) {
return options.table_factory->NewTableBuilder(options, internal_comparator,
file, compression_type);
TableBuilder* NewTableBuilder(
const ImmutableCFOptions& ioptions,
const InternalKeyComparator& internal_comparator,
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories,
uint32_t column_family_id, const std::string& column_family_name,
WritableFileWriter* file, const CompressionType compression_type,
const CompressionOptions& compression_opts, int level,
const std::string* compression_dict, const bool skip_filters,
const uint64_t creation_time, const uint64_t oldest_key_time) {
assert((column_family_id ==
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
column_family_name.empty());
return ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(
ioptions, internal_comparator, int_tbl_prop_collector_factories,
compression_type, compression_opts, compression_dict, skip_filters,
column_family_name, level, creation_time, oldest_key_time),
column_family_id, file);
}
Status BuildTable(const std::string& dbname, Env* env, const Options& options,
const EnvOptions& soptions, TableCache* table_cache,
Iterator* iter, FileMetaData* meta,
const InternalKeyComparator& internal_comparator,
const SequenceNumber newest_snapshot,
const SequenceNumber earliest_seqno_in_memtable,
const CompressionType compression,
const Env::IOPriority io_priority) {
Status BuildTable(
const std::string& dbname, Env* env, const ImmutableCFOptions& ioptions,
const MutableCFOptions& mutable_cf_options, const EnvOptions& env_options,
TableCache* table_cache, InternalIterator* iter,
std::unique_ptr<InternalIterator> range_del_iter, FileMetaData* meta,
const InternalKeyComparator& internal_comparator,
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories,
uint32_t column_family_id, const std::string& column_family_name,
std::vector<SequenceNumber> snapshots,
SequenceNumber earliest_write_conflict_snapshot,
const CompressionType compression,
const CompressionOptions& compression_opts, bool paranoid_file_checks,
InternalStats* internal_stats, TableFileCreationReason reason,
EventLogger* event_logger, int job_id, const Env::IOPriority io_priority,
TableProperties* table_properties, int level, const uint64_t creation_time,
const uint64_t oldest_key_time) {
assert((column_family_id ==
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
column_family_name.empty());
// Reports the IOStats for flush for every following bytes.
const size_t kReportFlushIOStatsEvery = 1048576;
Status s;
meta->fd.file_size = 0;
meta->smallest_seqno = meta->largest_seqno = 0;
iter->SeekToFirst();
// If the sequence number of the smallest entry in the memtable is
// smaller than the most recent snapshot, then we do not trigger
// removal of duplicate/deleted keys as part of this builder.
bool purge = options.purge_redundant_kvs_while_flush;
if (earliest_seqno_in_memtable <= newest_snapshot) {
purge = false;
std::unique_ptr<RangeDelAggregator> range_del_agg(
new RangeDelAggregator(internal_comparator, snapshots));
s = range_del_agg->AddTombstones(std::move(range_del_iter));
if (!s.ok()) {
// may be non-ok if a range tombstone key is unparsable
return s;
}
std::string fname = TableFileName(options.db_paths, meta->fd.GetNumber(),
std::string fname = TableFileName(ioptions.db_paths, meta->fd.GetNumber(),
meta->fd.GetPathId());
if (iter->Valid()) {
unique_ptr<WritableFile> file;
s = env->NewWritableFile(fname, &file, soptions);
if (!s.ok()) {
return s;
}
file->SetIOPriority(io_priority);
#ifndef ROCKSDB_LITE
EventHelpers::NotifyTableFileCreationStarted(
ioptions.listeners, dbname, column_family_name, fname, job_id, reason);
#endif // !ROCKSDB_LITE
TableProperties tp;
TableBuilder* builder =
NewTableBuilder(options, internal_comparator, file.get(), compression);
// the first key is the smallest key
Slice key = iter->key();
meta->smallest.DecodeFrom(key);
meta->smallest_seqno = GetInternalKeySeqno(key);
meta->largest_seqno = meta->smallest_seqno;
MergeHelper merge(internal_comparator.user_comparator(),
options.merge_operator.get(), options.info_log.get(),
options.min_partial_merge_operands,
true /* internal key corruption is not ok */);
if (purge) {
// Ugly walkaround to avoid compiler error for release build
bool ok __attribute__((unused)) = true;
// Will write to builder if current key != prev key
ParsedInternalKey prev_ikey;
std::string prev_key;
bool is_first_key = true; // Also write if this is the very first key
while (iter->Valid()) {
bool iterator_at_next = false;
// Get current key
ParsedInternalKey this_ikey;
Slice key = iter->key();
Slice value = iter->value();
// In-memory key corruption is not ok;
// TODO: find a clean way to treat in memory key corruption
ok = ParseInternalKey(key, &this_ikey);
assert(ok);
assert(this_ikey.sequence >= earliest_seqno_in_memtable);
// If the key is the same as the previous key (and it is not the
// first key), then we skip it, since it is an older version.
// Otherwise we output the key and mark it as the "new" previous key.
if (!is_first_key && !internal_comparator.user_comparator()->Compare(
prev_ikey.user_key, this_ikey.user_key)) {
// seqno within the same key are in decreasing order
assert(this_ikey.sequence < prev_ikey.sequence);
} else {
is_first_key = false;
if (this_ikey.type == kTypeMerge) {
// TODO(tbd): Add a check here to prevent RocksDB from crash when
// reopening a DB w/o properly specifying the merge operator. But
// currently we observed a memory leak on failing in RocksDB
// recovery, so we decide to let it crash instead of causing
// memory leak for now before we have identified the real cause
// of the memory leak.
// Handle merge-type keys using the MergeHelper
// TODO: pass statistics to MergeUntil
merge.MergeUntil(iter, 0 /* don't worry about snapshot */);
iterator_at_next = true;
if (merge.IsSuccess()) {
// Merge completed correctly.
// Add the resulting merge key/value and continue to next
builder->Add(merge.key(), merge.value());
prev_key.assign(merge.key().data(), merge.key().size());
ok = ParseInternalKey(Slice(prev_key), &prev_ikey);
assert(ok);
} else {
// Merge did not find a Put/Delete.
// Can not compact these merges into a kValueType.
// Write them out one-by-one. (Proceed back() to front())
const std::deque<std::string>& keys = merge.keys();
const std::deque<std::string>& values = merge.values();
assert(keys.size() == values.size() && keys.size() >= 1);
std::deque<std::string>::const_reverse_iterator key_iter;
std::deque<std::string>::const_reverse_iterator value_iter;
for (key_iter=keys.rbegin(), value_iter = values.rbegin();
key_iter != keys.rend() && value_iter != values.rend();
++key_iter, ++value_iter) {
builder->Add(Slice(*key_iter), Slice(*value_iter));
}
// Sanity check. Both iterators should end at the same time
assert(key_iter == keys.rend() && value_iter == values.rend());
prev_key.assign(keys.front());
ok = ParseInternalKey(Slice(prev_key), &prev_ikey);
assert(ok);
}
} else {
// Handle Put/Delete-type keys by simply writing them
builder->Add(key, value);
prev_key.assign(key.data(), key.size());
ok = ParseInternalKey(Slice(prev_key), &prev_ikey);
assert(ok);
}
}
if (!iterator_at_next) iter->Next();
if (iter->Valid() || range_del_agg->ShouldAddTombstones()) {
TableBuilder* builder;
unique_ptr<WritableFileWriter> file_writer;
{
unique_ptr<WritableFile> file;
#ifndef NDEBUG
bool use_direct_writes = env_options.use_direct_writes;
TEST_SYNC_POINT_CALLBACK("BuildTable:create_file", &use_direct_writes);
#endif // !NDEBUG
s = NewWritableFile(env, fname, &file, env_options);
if (!s.ok()) {
EventHelpers::LogAndNotifyTableFileCreationFinished(
event_logger, ioptions.listeners, dbname, column_family_name, fname,
job_id, meta->fd, tp, reason, s);
return s;
}
file->SetIOPriority(io_priority);
// The last key is the largest key
meta->largest.DecodeFrom(Slice(prev_key));
SequenceNumber seqno = GetInternalKeySeqno(Slice(prev_key));
meta->smallest_seqno = std::min(meta->smallest_seqno, seqno);
meta->largest_seqno = std::max(meta->largest_seqno, seqno);
file_writer.reset(new WritableFileWriter(std::move(file), env_options,
ioptions.statistics));
builder = NewTableBuilder(
ioptions, internal_comparator, int_tbl_prop_collector_factories,
column_family_id, column_family_name, file_writer.get(), compression,
compression_opts, level, nullptr /* compression_dict */,
false /* skip_filters */, creation_time, oldest_key_time);
}
} else {
for (; iter->Valid(); iter->Next()) {
Slice key = iter->key();
meta->largest.DecodeFrom(key);
builder->Add(key, iter->value());
SequenceNumber seqno = GetInternalKeySeqno(key);
meta->smallest_seqno = std::min(meta->smallest_seqno, seqno);
meta->largest_seqno = std::max(meta->largest_seqno, seqno);
MergeHelper merge(env, internal_comparator.user_comparator(),
ioptions.merge_operator, nullptr, ioptions.info_log,
true /* internal key corruption is not ok */,
snapshots.empty() ? 0 : snapshots.back());
CompactionIterator c_iter(
iter, internal_comparator.user_comparator(), &merge, kMaxSequenceNumber,
&snapshots, earliest_write_conflict_snapshot, env,
true /* internal key corruption is not ok */, range_del_agg.get());
c_iter.SeekToFirst();
for (; c_iter.Valid(); c_iter.Next()) {
const Slice& key = c_iter.key();
const Slice& value = c_iter.value();
builder->Add(key, value);
meta->UpdateBoundaries(key, c_iter.ikey().sequence);
// TODO(noetzli): Update stats after flush, too.
if (io_priority == Env::IO_HIGH &&
IOSTATS(bytes_written) >= kReportFlushIOStatsEvery) {
ThreadStatusUtil::SetThreadOperationProperty(
ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written));
}
}
// nullptr for table_{min,max} so all range tombstones will be flushed
range_del_agg->AddToBuilder(builder, nullptr /* lower_bound */,
nullptr /* upper_bound */, meta);
// Finish and check for builder errors
if (s.ok()) {
s = builder->Finish();
if (s.ok()) {
meta->fd.file_size = builder->FileSize();
assert(meta->fd.GetFileSize() > 0);
}
} else {
bool empty = builder->NumEntries() == 0;
s = c_iter.status();
if (!s.ok() || empty) {
builder->Abandon();
} else {
s = builder->Finish();
}
if (s.ok() && !empty) {
uint64_t file_size = builder->FileSize();
meta->fd.file_size = file_size;
meta->marked_for_compaction = builder->NeedCompact();
assert(meta->fd.GetFileSize() > 0);
tp = builder->GetTableProperties();
if (table_properties) {
*table_properties = tp;
}
}
delete builder;
// Finish and check for file errors
if (s.ok() && !options.disableDataSync) {
if (options.use_fsync) {
StopWatch sw(env, options.statistics.get(), TABLE_SYNC_MICROS);
s = file->Fsync();
} else {
StopWatch sw(env, options.statistics.get(), TABLE_SYNC_MICROS);
s = file->Sync();
}
if (s.ok() && !empty) {
StopWatch sw(env, ioptions.statistics, TABLE_SYNC_MICROS);
s = file_writer->Sync(ioptions.use_fsync);
}
if (s.ok()) {
s = file->Close();
if (s.ok() && !empty) {
s = file_writer->Close();
}
if (s.ok()) {
if (s.ok() && !empty) {
// Verify that the table is usable
Iterator* it = table_cache->NewIterator(ReadOptions(), soptions,
internal_comparator, meta->fd);
// We set for_compaction to false and don't OptimizeForCompactionTableRead
// here because this is a special case after we finish the table building
// No matter whether use_direct_io_for_flush_and_compaction is true,
// we will regrad this verification as user reads since the goal is
// to cache it here for further user reads
std::unique_ptr<InternalIterator> it(table_cache->NewIterator(
ReadOptions(), env_options, internal_comparator, meta->fd,
nullptr /* range_del_agg */, nullptr,
(internal_stats == nullptr) ? nullptr
: internal_stats->GetFileReadHist(0),
false /* for_compaction */, nullptr /* arena */,
false /* skip_filter */, level));
s = it->status();
delete it;
if (s.ok() && paranoid_file_checks) {
for (it->SeekToFirst(); it->Valid(); it->Next()) {
}
s = it->status();
}
}
}
@@ -223,11 +212,15 @@ Status BuildTable(const std::string& dbname, Env* env, const Options& options,
s = iter->status();
}
if (s.ok() && meta->fd.GetFileSize() > 0) {
// Keep it
} else {
if (!s.ok() || meta->fd.GetFileSize() == 0) {
env->DeleteFile(fname);
}
// Output to event logger and fire events.
EventHelpers::LogAndNotifyTableFileCreationFinished(
event_logger, ioptions.listeners, dbname, column_family_name, fname,
job_id, meta->fd, tp, reason, s);
return s;
}

74
db/builder.h
View File

@@ -1,16 +1,25 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include <string>
#include <utility>
#include <vector>
#include "db/table_properties_collector.h"
#include "options/cf_options.h"
#include "rocksdb/comparator.h"
#include "rocksdb/env.h"
#include "rocksdb/status.h"
#include "rocksdb/types.h"
#include "rocksdb/listener.h"
#include "rocksdb/options.h"
#include "rocksdb/status.h"
#include "rocksdb/table_properties.h"
#include "rocksdb/types.h"
#include "table/scoped_arena_iterator.h"
#include "util/event_logger.h"
namespace rocksdb {
@@ -23,25 +32,52 @@ class Iterator;
class TableCache;
class VersionEdit;
class TableBuilder;
class WritableFile;
class WritableFileWriter;
class InternalStats;
class InternalIterator;
extern TableBuilder* NewTableBuilder(
const Options& options, const InternalKeyComparator& internal_comparator,
WritableFile* file, CompressionType compression_type);
// @param column_family_name Name of the column family that is also identified
// by column_family_id, or empty string if unknown. It must outlive the
// TableBuilder returned by this function.
// @param compression_dict Data for presetting the compression library's
// dictionary, or nullptr.
TableBuilder* NewTableBuilder(
const ImmutableCFOptions& options,
const InternalKeyComparator& internal_comparator,
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories,
uint32_t column_family_id, const std::string& column_family_name,
WritableFileWriter* file, const CompressionType compression_type,
const CompressionOptions& compression_opts, int level,
const std::string* compression_dict = nullptr,
const bool skip_filters = false, const uint64_t creation_time = 0,
const uint64_t oldest_key_time = 0);
// Build a Table file from the contents of *iter. The generated file
// will be named according to number specified in meta. On success, the rest of
// *meta will be filled with metadata about the generated table.
// If no data is present in *iter, meta->file_size will be set to
// zero, and no Table file will be produced.
extern Status BuildTable(const std::string& dbname, Env* env,
const Options& options, const EnvOptions& soptions,
TableCache* table_cache, Iterator* iter,
FileMetaData* meta,
const InternalKeyComparator& internal_comparator,
const SequenceNumber newest_snapshot,
const SequenceNumber earliest_seqno_in_memtable,
const CompressionType compression,
const Env::IOPriority io_priority = Env::IO_HIGH);
//
// @param column_family_name Name of the column family that is also identified
// by column_family_id, or empty string if unknown.
extern Status BuildTable(
const std::string& dbname, Env* env, const ImmutableCFOptions& options,
const MutableCFOptions& mutable_cf_options, const EnvOptions& env_options,
TableCache* table_cache, InternalIterator* iter,
std::unique_ptr<InternalIterator> range_del_iter, FileMetaData* meta,
const InternalKeyComparator& internal_comparator,
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories,
uint32_t column_family_id, const std::string& column_family_name,
std::vector<SequenceNumber> snapshots,
SequenceNumber earliest_write_conflict_snapshot,
const CompressionType compression,
const CompressionOptions& compression_opts, bool paranoid_file_checks,
InternalStats* internal_stats, TableFileCreationReason reason,
EventLogger* event_logger = nullptr, int job_id = 0,
const Env::IOPriority io_priority = Env::IO_HIGH,
TableProperties* table_properties = nullptr, int level = -1,
const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0);
} // namespace rocksdb

2168
db/c.cc
View File

File diff suppressed because it is too large Load Diff

990
db/c_test.c
View File

File diff suppressed because it is too large Load Diff

863
db/column_family.cc
View File

File diff suppressed because it is too large Load Diff

375
db/column_family.h
View File

@@ -1,7 +1,7 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
@@ -14,12 +14,16 @@
#include <vector>
#include <atomic>
#include "rocksdb/options.h"
#include "db/memtable_list.h"
#include "db/table_cache.h"
#include "db/table_properties_collector.h"
#include "db/write_batch_internal.h"
#include "db/write_controller.h"
#include "options/cf_options.h"
#include "rocksdb/compaction_job_stats.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "db/memtable_list.h"
#include "db/write_batch_internal.h"
#include "db/table_cache.h"
#include "rocksdb/options.h"
#include "util/thread_local.h"
namespace rocksdb {
@@ -35,6 +39,10 @@ class InternalStats;
class ColumnFamilyData;
class DBImpl;
class LogBuffer;
class InstrumentedMutex;
class InstrumentedMutexLock;
extern const double kIncSlowdownRatio;
// ColumnFamilyHandleImpl is the class that clients use to access different
// column families. It has non-trivial destructor, which gets called when client
@@ -42,17 +50,21 @@ class LogBuffer;
class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
public:
// create while holding the mutex
ColumnFamilyHandleImpl(ColumnFamilyData* cfd, DBImpl* db, port::Mutex* mutex);
ColumnFamilyHandleImpl(
ColumnFamilyData* cfd, DBImpl* db, InstrumentedMutex* mutex);
// destroy without mutex
virtual ~ColumnFamilyHandleImpl();
virtual ColumnFamilyData* cfd() const { return cfd_; }
virtual uint32_t GetID() const;
virtual uint32_t GetID() const override;
virtual const std::string& GetName() const override;
virtual Status GetDescriptor(ColumnFamilyDescriptor* desc) override;
virtual const Comparator* GetComparator() const override;
private:
ColumnFamilyData* cfd_;
DBImpl* db_;
port::Mutex* mutex_;
InstrumentedMutex* mutex_;
};
// Does not ref-count ColumnFamilyData
@@ -66,7 +78,7 @@ class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
ColumnFamilyHandleInternal()
: ColumnFamilyHandleImpl(nullptr, nullptr, nullptr) {}
void SetCFD(ColumnFamilyData* cfd) { internal_cfd_ = cfd; }
void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; }
virtual ColumnFamilyData* cfd() const override { return internal_cfd_; }
private:
@@ -75,23 +87,23 @@ class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
// holds references to memtable, all immutable memtables and version
struct SuperVersion {
// Accessing members of this class is not thread-safe and requires external
// synchronization (ie db mutex held or on write thread).
MemTable* mem;
MemTableListVersion* imm;
Version* current;
std::atomic<uint32_t> refs;
// We need to_delete because during Cleanup(), imm->Unref() returns
// all memtables that we need to free through this vector. We then
// delete all those memtables outside of mutex, during destruction
autovector<MemTable*> to_delete;
MutableCFOptions mutable_cf_options;
// Version number of the current SuperVersion
uint64_t version_number;
port::Mutex* db_mutex;
InstrumentedMutex* db_mutex;
// should be called outside the mutex
SuperVersion() = default;
~SuperVersion();
SuperVersion* Ref();
// If Unref() returns true, Cleanup() should be called with mutex held
// before deleting this SuperVersion.
bool Unref();
// call these two methods with db mutex held
@@ -110,15 +122,33 @@ struct SuperVersion {
static int dummy;
static void* const kSVInUse;
static void* const kSVObsolete;
private:
std::atomic<uint32_t> refs;
// We need to_delete because during Cleanup(), imm->Unref() returns
// all memtables that we need to free through this vector. We then
// delete all those memtables outside of mutex, during destruction
autovector<MemTable*> to_delete;
};
extern ColumnFamilyOptions SanitizeOptions(const InternalKeyComparator* icmp,
extern Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options);
extern Status CheckConcurrentWritesSupported(
const ColumnFamilyOptions& cf_options);
extern ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
const ColumnFamilyOptions& src);
// Wrap user defined table proproties collector factories `from cf_options`
// into internal ones in int_tbl_prop_collector_factories. Add a system internal
// one too.
extern void GetIntTblPropCollectorFactory(
const ImmutableCFOptions& ioptions,
std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories);
class ColumnFamilySet;
// This class keeps all the data that a column family needs. It's mosly dumb and
// used just to provide access to metadata.
// This class keeps all the data that a column family needs.
// Most methods require DB mutex held, unless otherwise noted
class ColumnFamilyData {
public:
@@ -129,45 +159,74 @@ class ColumnFamilyData {
// thread-safe
const std::string& GetName() const { return name_; }
void Ref() { ++refs_; }
// will just decrease reference count to 0, but will not delete it. returns
// true if the ref count was decreased to zero. in that case, it can be
// deleted by the caller immediatelly, or later, by calling
// FreeDeadColumnFamilies()
// Ref() can only be called from a context where the caller can guarantee
// that ColumnFamilyData is alive (while holding a non-zero ref already,
// holding a DB mutex, or as the leader in a write batch group).
void Ref() { refs_.fetch_add(1, std::memory_order_relaxed); }
// Unref decreases the reference count, but does not handle deletion
// when the count goes to 0. If this method returns true then the
// caller should delete the instance immediately, or later, by calling
// FreeDeadColumnFamilies(). Unref() can only be called while holding
// a DB mutex, or during single-threaded recovery.
bool Unref() {
assert(refs_ > 0);
return --refs_ == 0;
int old_refs = refs_.fetch_sub(1, std::memory_order_relaxed);
assert(old_refs > 0);
return old_refs == 1;
}
// This can only be called from single-threaded VersionSet::LogAndApply()
// SetDropped() can only be called under following conditions:
// 1) Holding a DB mutex,
// 2) from single-threaded write thread, AND
// 3) from single-threaded VersionSet::LogAndApply()
// After dropping column family no other operation on that column family
// will be executed. All the files and memory will be, however, kept around
// until client drops the column family handle. That way, client can still
// access data from dropped column family.
// Column family can be dropped and still alive. In that state:
// *) Column family is not included in the iteration.
// *) Compaction and flush is not executed on the dropped column family.
// *) Client can continue writing and reading from column family. However, all
// writes stay in the current memtable.
// *) Client can continue reading from column family. Writes will fail unless
// WriteOptions::ignore_missing_column_families is true
// When the dropped column family is unreferenced, then we:
// *) Remove column family from the linked list maintained by ColumnFamilySet
// *) delete all memory associated with that column family
// *) delete all the files associated with that column family
void SetDropped() {
// can't drop default CF
assert(id_ != 0);
dropped_ = true;
}
void SetDropped();
bool IsDropped() const { return dropped_; }
// thread-safe
int NumberLevels() const { return options_.num_levels; }
int NumberLevels() const { return ioptions_.num_levels; }
void SetLogNumber(uint64_t log_number) { log_number_ = log_number; }
uint64_t GetLogNumber() const { return log_number_; }
// thread-safe
const Options* options() const { return &options_; }
const EnvOptions* soptions() const;
const ImmutableCFOptions* ioptions() const { return &ioptions_; }
// REQUIRES: DB mutex held
// This returns the MutableCFOptions used by current SuperVersion
// You should use this API to reference MutableCFOptions most of the time.
const MutableCFOptions* GetCurrentMutableCFOptions() const {
return &(super_version_->mutable_cf_options);
}
// REQUIRES: DB mutex held
// This returns the latest MutableCFOptions, which may be not in effect yet.
const MutableCFOptions* GetLatestMutableCFOptions() const {
return &mutable_cf_options_;
}
// REQUIRES: DB mutex held
// Build ColumnFamiliesOptions with immutable options and latest mutable
// options.
ColumnFamilyOptions GetLatestCFOptions() const;
bool is_delete_range_supported() { return is_delete_range_supported_; }
#ifndef ROCKSDB_LITE
// REQUIRES: DB mutex held
Status SetOptions(
const std::unordered_map<std::string, std::string>& options_map);
#endif // ROCKSDB_LITE
InternalStats* internal_stats() { return internal_stats_.get(); }
@@ -175,18 +234,46 @@ class ColumnFamilyData {
MemTable* mem() { return mem_; }
Version* current() { return current_; }
Version* dummy_versions() { return dummy_versions_; }
void SetCurrent(Version* _current);
uint64_t GetNumLiveVersions() const; // REQUIRE: DB mutex held
uint64_t GetTotalSstFilesSize() const; // REQUIRE: DB mutex held
void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }
void SetCurrent(Version* current);
void CreateNewMemtable();
// calculate the oldest log needed for the durability of this column family
uint64_t OldestLogToKeep();
// See Memtable constructor for explanation of earliest_seq param.
MemTable* ConstructNewMemtable(const MutableCFOptions& mutable_cf_options,
SequenceNumber earliest_seq);
void CreateNewMemtable(const MutableCFOptions& mutable_cf_options,
SequenceNumber earliest_seq);
TableCache* table_cache() const { return table_cache_.get(); }
// See documentation in compaction_picker.h
Compaction* PickCompaction(LogBuffer* log_buffer);
Compaction* CompactRange(int input_level, int output_level,
// REQUIRES: DB mutex held
bool NeedsCompaction() const;
// REQUIRES: DB mutex held
Compaction* PickCompaction(const MutableCFOptions& mutable_options,
LogBuffer* log_buffer);
// Check if the passed range overlap with any running compactions.
// REQUIRES: DB mutex held
bool RangeOverlapWithCompaction(const Slice& smallest_user_key,
const Slice& largest_user_key,
int level) const;
// A flag to tell a manual compaction is to compact all levels together
// instad of for specific level.
static const int kCompactAllLevels;
// A flag to tell a manual compaction's output is base level.
static const int kCompactToBaseLevel;
// REQUIRES: DB mutex held
Compaction* CompactRange(const MutableCFOptions& mutable_cf_options,
int input_level, int output_level,
uint32_t output_path_id, const InternalKey* begin,
const InternalKey* end,
InternalKey** compaction_end);
const InternalKey* end, InternalKey** compaction_end,
bool* manual_conflict);
CompactionPicker* compaction_picker() { return compaction_picker_.get(); }
// thread-safe
@@ -198,14 +285,19 @@ class ColumnFamilyData {
return internal_comparator_;
}
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories() const {
return &int_tbl_prop_collector_factories_;
}
SuperVersion* GetSuperVersion() { return super_version_; }
// thread-safe
// Return a already referenced SuperVersion to be used safely.
SuperVersion* GetReferencedSuperVersion(port::Mutex* db_mutex);
SuperVersion* GetReferencedSuperVersion(InstrumentedMutex* db_mutex);
// thread-safe
// Get SuperVersion stored in thread local storage. If it does not exist,
// get a reference from a current SuperVersion.
SuperVersion* GetThreadLocalSuperVersion(port::Mutex* db_mutex);
SuperVersion* GetThreadLocalSuperVersion(InstrumentedMutex* db_mutex);
// Try to return SuperVersion back to thread local storage. Retrun true on
// success and false on failure. It fails when the thread local storage
// contains anything other than SuperVersion::kSVInUse flag.
@@ -218,66 +310,68 @@ class ColumnFamilyData {
// if its reference count is zero and needs deletion or nullptr if not
// As argument takes a pointer to allocated SuperVersion to enable
// the clients to allocate SuperVersion outside of mutex.
// IMPORTANT: Only call this from DBImpl::InstallSuperVersion()
SuperVersion* InstallSuperVersion(SuperVersion* new_superversion,
port::Mutex* db_mutex);
InstrumentedMutex* db_mutex,
const MutableCFOptions& mutable_cf_options);
SuperVersion* InstallSuperVersion(SuperVersion* new_superversion,
InstrumentedMutex* db_mutex);
void ResetThreadLocalSuperVersions();
// A Flag indicating whether write needs to slowdown because of there are
// too many number of level0 files.
bool NeedSlowdownForNumLevel0Files() const {
return need_slowdown_for_num_level0_files_;
}
bool NeedWaitForNumLevel0Files() const {
return need_wait_for_num_level0_files_;
}
bool NeedWaitForNumMemtables() const {
return need_wait_for_num_memtables_;
}
bool ExceedsSoftRateLimit() const {
return exceeds_soft_rate_limit_;
}
bool ExceedsHardRateLimit() const {
return exceeds_hard_rate_limit_;
}
private:
friend class ColumnFamilySet;
ColumnFamilyData(uint32_t id, const std::string& name,
Version* dummy_versions, Cache* table_cache,
const ColumnFamilyOptions& options,
const DBOptions* db_options,
const EnvOptions& storage_options,
ColumnFamilySet* column_family_set);
// Protected by DB mutex
void set_pending_flush(bool value) { pending_flush_ = value; }
void set_pending_compaction(bool value) { pending_compaction_ = value; }
bool pending_flush() { return pending_flush_; }
bool pending_compaction() { return pending_compaction_; }
// Recalculate some small conditions, which are changed only during
// compaction, adding new memtable and/or
// recalculation of compaction score. These values are used in
// DBImpl::MakeRoomForWrite function to decide, if it need to make
// a write stall
void RecalculateWriteStallConditions();
void RecalculateWriteStallRateLimitsConditions();
void RecalculateWriteStallConditions(
const MutableCFOptions& mutable_cf_options);
void set_initialized() { initialized_.store(true); }
bool initialized() const { return initialized_.load(); }
private:
friend class ColumnFamilySet;
ColumnFamilyData(uint32_t id, const std::string& name,
Version* dummy_versions, Cache* table_cache,
WriteBufferManager* write_buffer_manager,
const ColumnFamilyOptions& options,
const ImmutableDBOptions& db_options,
const EnvOptions& env_options,
ColumnFamilySet* column_family_set);
uint32_t id_;
const std::string name_;
Version* dummy_versions_; // Head of circular doubly-linked list of versions.
Version* current_; // == dummy_versions->prev_
int refs_; // outstanding references to ColumnFamilyData
std::atomic<int> refs_; // outstanding references to ColumnFamilyData
std::atomic<bool> initialized_;
bool dropped_; // true if client dropped it
const InternalKeyComparator internal_comparator_;
std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
int_tbl_prop_collector_factories_;
Options const options_;
const ColumnFamilyOptions initial_cf_options_;
const ImmutableCFOptions ioptions_;
MutableCFOptions mutable_cf_options_;
const bool is_delete_range_supported_;
std::unique_ptr<TableCache> table_cache_;
std::unique_ptr<InternalStats> internal_stats_;
WriteBufferManager* write_buffer_manager_;
MemTable* mem_;
MemTableList imm_;
SuperVersion* super_version_;
@@ -291,8 +385,9 @@ class ColumnFamilyData {
// This needs to be destructed before mutex_
std::unique_ptr<ThreadLocalPtr> local_sv_;
// pointers for a circular linked list. we use it to support iterations
// that can be concurrent with writes
// pointers for a circular linked list. we use it to support iterations over
// all column families that are alive (note: dropped column families can also
// be alive as long as client holds a reference)
ColumnFamilyData* next_;
ColumnFamilyData* prev_;
@@ -301,46 +396,43 @@ class ColumnFamilyData {
// recovered from
uint64_t log_number_;
// A flag indicating whether we should delay writes because
// we have too many level 0 files
bool need_slowdown_for_num_level0_files_;
// These 4 variables are updated only after compaction,
// adding new memtable, flushing memtables to files
// and/or add recalculation of compaction score.
// That's why theirs values are cached in ColumnFamilyData.
// Recalculation is made by RecalculateWriteStallConditions and
// RecalculateWriteStallRateLimitsConditions function. They are used
// in DBImpl::MakeRoomForWrite function to decide, if it need
// to sleep during write operation
bool need_wait_for_num_memtables_;
bool need_wait_for_num_level0_files_;
bool exceeds_hard_rate_limit_;
bool exceeds_soft_rate_limit_;
// An object that keeps all the compaction stats
// and picks the next compaction
std::unique_ptr<CompactionPicker> compaction_picker_;
ColumnFamilySet* column_family_set_;
std::unique_ptr<WriteControllerToken> write_controller_token_;
// If true --> this ColumnFamily is currently present in DBImpl::flush_queue_
bool pending_flush_;
// If true --> this ColumnFamily is currently present in
// DBImpl::compaction_queue_
bool pending_compaction_;
uint64_t prev_compaction_needed_bytes_;
// if the database was opened with 2pc enabled
bool allow_2pc_;
};
// ColumnFamilySet has interesting thread-safety requirements
// * CreateColumnFamily() or RemoveColumnFamily() -- need to protect by DB
// mutex. Inside, column_family_data_ and column_families_ will be protected
// by Lock() and Unlock(). CreateColumnFamily() should ONLY be called from
// VersionSet::LogAndApply() in the normal runtime. It is also called
// during Recovery and in DumpManifest(). RemoveColumnFamily() is called
// from ColumnFamilyData destructor
// * CreateColumnFamily() or RemoveColumnFamily() -- need to be protected by DB
// mutex AND executed in the write thread.
// CreateColumnFamily() should ONLY be called from VersionSet::LogAndApply() AND
// single-threaded write thread. It is also called during Recovery and in
// DumpManifest().
// RemoveColumnFamily() is only called from SetDropped(). DB mutex needs to be
// held and it needs to be executed from the write thread. SetDropped() also
// guarantees that it will be called only from single-threaded LogAndApply(),
// but this condition is not that important.
// * Iteration -- hold DB mutex, but you can release it in the body of
// iteration. If you release DB mutex in body, reference the column
// family before the mutex and unreference after you unlock, since the column
// family might get dropped when the DB mutex is released
// * GetDefault() -- thread safe
// * GetColumnFamily() -- either inside of DB mutex or call Lock() <-> Unlock()
// * GetColumnFamily() -- either inside of DB mutex or from a write thread
// * GetNextColumnFamilyID(), GetMaxColumnFamily(), UpdateMaxColumnFamily(),
// NumberOfColumnFamilies -- inside of DB mutex
class ColumnFamilySet {
@@ -351,10 +443,13 @@ class ColumnFamilySet {
explicit iterator(ColumnFamilyData* cfd)
: current_(cfd) {}
iterator& operator++() {
// dummy is never dead or dropped, so this will never be infinite
// dropped column families might still be included in this iteration
// (we're only removing them when client drops the last reference to the
// column family).
// dummy is never dead, so this will never be infinite
do {
current_ = current_->next_;
} while (current_->refs_ == 0 || current_->IsDropped());
} while (current_->refs_.load(std::memory_order_relaxed) == 0);
return *this;
}
bool operator!=(const iterator& other) {
@@ -366,8 +461,11 @@ class ColumnFamilySet {
ColumnFamilyData* current_;
};
ColumnFamilySet(const std::string& dbname, const DBOptions* db_options,
const EnvOptions& storage_options, Cache* table_cache);
ColumnFamilySet(const std::string& dbname,
const ImmutableDBOptions* db_options,
const EnvOptions& env_options, Cache* table_cache,
WriteBufferManager* write_buffer_manager,
WriteController* write_controller);
~ColumnFamilySet();
ColumnFamilyData* GetDefault() const;
@@ -390,13 +488,12 @@ class ColumnFamilySet {
iterator begin() { return iterator(dummy_cfd_->next_); }
iterator end() { return iterator(dummy_cfd_); }
void Lock();
void Unlock();
// REQUIRES: DB mutex held
// Don't call while iterating over ColumnFamilySet
void FreeDeadColumnFamilies();
Cache* get_table_cache() { return table_cache_; }
private:
friend class ColumnFamilyData;
// helper function that gets called from cfd destructor
@@ -404,9 +501,12 @@ class ColumnFamilySet {
void RemoveColumnFamily(ColumnFamilyData* cfd);
// column_families_ and column_family_data_ need to be protected:
// * when mutating: 1. DB mutex locked first, 2. spinlock locked second
// * when reading, either: 1. lock DB mutex, or 2. lock spinlock
// (if both, respect the ordering to avoid deadlock!)
// * when mutating both conditions have to be satisfied:
// 1. DB mutex locked
// 2. thread currently in single-threaded write thread
// * when reading, at least one condition needs to be satisfied:
// 1. DB mutex locked
// 2. accessed from a single-threaded write thread
std::unordered_map<std::string, uint32_t> column_families_;
std::unordered_map<uint32_t, ColumnFamilyData*> column_family_data_;
@@ -419,10 +519,11 @@ class ColumnFamilySet {
ColumnFamilyData* default_cfd_cache_;
const std::string db_name_;
const DBOptions* const db_options_;
const EnvOptions storage_options_;
const ImmutableDBOptions* const db_options_;
const EnvOptions env_options_;
Cache* table_cache_;
std::atomic_flag spin_lock_;
WriteBufferManager* write_buffer_manager_;
WriteController* write_controller_;
};
// We use ColumnFamilyMemTablesImpl to provide WriteBatch a way to access
@@ -432,23 +533,36 @@ class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables {
explicit ColumnFamilyMemTablesImpl(ColumnFamilySet* column_family_set)
: column_family_set_(column_family_set), current_(nullptr) {}
// Constructs a ColumnFamilyMemTablesImpl equivalent to one constructed
// with the arguments used to construct *orig.
explicit ColumnFamilyMemTablesImpl(ColumnFamilyMemTablesImpl* orig)
: column_family_set_(orig->column_family_set_), current_(nullptr) {}
// sets current_ to ColumnFamilyData with column_family_id
// returns false if column family doesn't exist
// REQUIRES: use this function of DBImpl::column_family_memtables_ should be
// under a DB mutex OR from a write thread
bool Seek(uint32_t column_family_id) override;
// Returns log number of the selected column family
// REQUIRES: under a DB mutex OR from a write thread
uint64_t GetLogNumber() const override;
// REQUIRES: Seek() called first
// REQUIRES: use this function of DBImpl::column_family_memtables_ should be
// under a DB mutex OR from a write thread
virtual MemTable* GetMemTable() const override;
// Returns options for selected column family
// REQUIRES: Seek() called first
virtual const Options* GetOptions() const override;
// Returns column family handle for the selected column family
// REQUIRES: use this function of DBImpl::column_family_memtables_ should be
// under a DB mutex OR from a write thread
virtual ColumnFamilyHandle* GetColumnFamilyHandle() override;
// Cannot be called while another thread is calling Seek().
// REQUIRES: use this function of DBImpl::column_family_memtables_ should be
// under a DB mutex OR from a write thread
virtual ColumnFamilyData* current() override { return current_; }
private:
ColumnFamilySet* column_family_set_;
ColumnFamilyData* current_;
@@ -457,4 +571,7 @@ class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables {
extern uint32_t GetColumnFamilyID(ColumnFamilyHandle* column_family);
extern const Comparator* GetColumnFamilyUserComparator(
ColumnFamilyHandle* column_family);
} // namespace rocksdb

2490
db/column_family_test.cc
View File

File diff suppressed because it is too large Load Diff

328
db/compact_files_test.cc Normal file
View File

@@ -0,0 +1,328 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include <mutex>
#include <string>
#include <thread>
#include <vector>
#include "db/db_impl.h"
#include "port/port.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "util/string_util.h"
#include "util/sync_point.h"
#include "util/testharness.h"
namespace rocksdb {
class CompactFilesTest : public testing::Test {
public:
CompactFilesTest() {
env_ = Env::Default();
db_name_ = test::TmpDir(env_) + "/compact_files_test";
}
std::string db_name_;
Env* env_;
};
// A class which remembers the name of each flushed file.
class FlushedFileCollector : public EventListener {
public:
FlushedFileCollector() {}
~FlushedFileCollector() {}
virtual void OnFlushCompleted(
DB* db, const FlushJobInfo& info) override {
std::lock_guard<std::mutex> lock(mutex_);
flushed_files_.push_back(info.file_path);
}
std::vector<std::string> GetFlushedFiles() {
std::lock_guard<std::mutex> lock(mutex_);
std::vector<std::string> result;
for (auto fname : flushed_files_) {
result.push_back(fname);
}
return result;
}
void ClearFlushedFiles() {
std::lock_guard<std::mutex> lock(mutex_);
flushed_files_.clear();
}
private:
std::vector<std::string> flushed_files_;
std::mutex mutex_;
};
TEST_F(CompactFilesTest, L0ConflictsFiles) {
Options options;
// to trigger compaction more easily
const int kWriteBufferSize = 10000;
const int kLevel0Trigger = 2;
options.create_if_missing = true;
options.compaction_style = kCompactionStyleLevel;
// Small slowdown and stop trigger for experimental purpose.
options.level0_slowdown_writes_trigger = 20;
options.level0_stop_writes_trigger = 20;
options.level0_stop_writes_trigger = 20;
options.write_buffer_size = kWriteBufferSize;
options.level0_file_num_compaction_trigger = kLevel0Trigger;
options.compression = kNoCompression;
DB* db = nullptr;
DestroyDB(db_name_, options);
Status s = DB::Open(options, db_name_, &db);
assert(s.ok());
assert(db);
rocksdb::SyncPoint::GetInstance()->LoadDependency({
{"CompactFilesImpl:0", "BackgroundCallCompaction:0"},
{"BackgroundCallCompaction:1", "CompactFilesImpl:1"},
});
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
// create couple files
// Background compaction starts and waits in BackgroundCallCompaction:0
for (int i = 0; i < kLevel0Trigger * 4; ++i) {
db->Put(WriteOptions(), ToString(i), "");
db->Put(WriteOptions(), ToString(100 - i), "");
db->Flush(FlushOptions());
}
rocksdb::ColumnFamilyMetaData meta;
db->GetColumnFamilyMetaData(&meta);
std::string file1;
for (auto& file : meta.levels[0].files) {
ASSERT_EQ(0, meta.levels[0].level);
if (file1 == "") {
file1 = file.db_path + "/" + file.name;
} else {
std::string file2 = file.db_path + "/" + file.name;
// Another thread starts a compact files and creates an L0 compaction
// The background compaction then notices that there is an L0 compaction
// already in progress and doesn't do an L0 compaction
// Once the background compaction finishes, the compact files finishes
ASSERT_OK(
db->CompactFiles(rocksdb::CompactionOptions(), {file1, file2}, 0));
break;
}
}
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
delete db;
}
TEST_F(CompactFilesTest, ObsoleteFiles) {
Options options;
// to trigger compaction more easily
const int kWriteBufferSize = 65536;
options.create_if_missing = true;
// Disable RocksDB background compaction.
options.compaction_style = kCompactionStyleNone;
options.level0_slowdown_writes_trigger = (1 << 30);
options.level0_stop_writes_trigger = (1 << 30);
options.write_buffer_size = kWriteBufferSize;
options.max_write_buffer_number = 2;
options.compression = kNoCompression;
// Add listener
FlushedFileCollector* collector = new FlushedFileCollector();
options.listeners.emplace_back(collector);
DB* db = nullptr;
DestroyDB(db_name_, options);
Status s = DB::Open(options, db_name_, &db);
assert(s.ok());
assert(db);
// create couple files
for (int i = 1000; i < 2000; ++i) {
db->Put(WriteOptions(), ToString(i),
std::string(kWriteBufferSize / 10, 'a' + (i % 26)));
}
auto l0_files = collector->GetFlushedFiles();
ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files, 1));
reinterpret_cast<DBImpl*>(db)->TEST_WaitForCompact();
// verify all compaction input files are deleted
for (auto fname : l0_files) {
ASSERT_EQ(Status::NotFound(), env_->FileExists(fname));
}
delete db;
}
TEST_F(CompactFilesTest, NotCutOutputOnLevel0) {
Options options;
options.create_if_missing = true;
// Disable RocksDB background compaction.
options.compaction_style = kCompactionStyleNone;
options.level0_slowdown_writes_trigger = 1000;
options.level0_stop_writes_trigger = 1000;
options.write_buffer_size = 65536;
options.max_write_buffer_number = 2;
options.compression = kNoCompression;
options.max_compaction_bytes = 5000;
// Add listener
FlushedFileCollector* collector = new FlushedFileCollector();
options.listeners.emplace_back(collector);
DB* db = nullptr;
DestroyDB(db_name_, options);
Status s = DB::Open(options, db_name_, &db);
assert(s.ok());
assert(db);
// create couple files
for (int i = 0; i < 500; ++i) {
db->Put(WriteOptions(), ToString(i), std::string(1000, 'a' + (i % 26)));
}
reinterpret_cast<DBImpl*>(db)->TEST_WaitForFlushMemTable();
auto l0_files_1 = collector->GetFlushedFiles();
collector->ClearFlushedFiles();
for (int i = 0; i < 500; ++i) {
db->Put(WriteOptions(), ToString(i), std::string(1000, 'a' + (i % 26)));
}
reinterpret_cast<DBImpl*>(db)->TEST_WaitForFlushMemTable();
auto l0_files_2 = collector->GetFlushedFiles();
ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files_1, 0));
ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files_2, 0));
// no assertion failure
delete db;
}
TEST_F(CompactFilesTest, CapturingPendingFiles) {
Options options;
options.create_if_missing = true;
// Disable RocksDB background compaction.
options.compaction_style = kCompactionStyleNone;
// Always do full scans for obsolete files (needed to reproduce the issue).
options.delete_obsolete_files_period_micros = 0;
// Add listener.
FlushedFileCollector* collector = new FlushedFileCollector();
options.listeners.emplace_back(collector);
DB* db = nullptr;
DestroyDB(db_name_, options);
Status s = DB::Open(options, db_name_, &db);
assert(s.ok());
assert(db);
// Create 5 files.
for (int i = 0; i < 5; ++i) {
db->Put(WriteOptions(), "key" + ToString(i), "value");
db->Flush(FlushOptions());
}
auto l0_files = collector->GetFlushedFiles();
EXPECT_EQ(5, l0_files.size());
rocksdb::SyncPoint::GetInstance()->LoadDependency({
{"CompactFilesImpl:2", "CompactFilesTest.CapturingPendingFiles:0"},
{"CompactFilesTest.CapturingPendingFiles:1", "CompactFilesImpl:3"},
});
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
// Start compacting files.
rocksdb::port::Thread compaction_thread(
[&] { EXPECT_OK(db->CompactFiles(CompactionOptions(), l0_files, 1)); });
// In the meantime flush another file.
TEST_SYNC_POINT("CompactFilesTest.CapturingPendingFiles:0");
db->Put(WriteOptions(), "key5", "value");
db->Flush(FlushOptions());
TEST_SYNC_POINT("CompactFilesTest.CapturingPendingFiles:1");
compaction_thread.join();
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
delete db;
// Make sure we can reopen the DB.
s = DB::Open(options, db_name_, &db);
ASSERT_TRUE(s.ok());
assert(db);
delete db;
}
TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
class FilterWithGet : public CompactionFilter {
public:
virtual bool Filter(int level, const Slice& key, const Slice& value,
std::string* new_value,
bool* value_changed) const override {
if (db_ == nullptr) {
return true;
}
std::string res;
db_->Get(ReadOptions(), "", &res);
return true;
}
void SetDB(DB* db) {
db_ = db;
}
virtual const char* Name() const override { return "FilterWithGet"; }
private:
DB* db_;
};
std::shared_ptr<FilterWithGet> cf(new FilterWithGet());
Options options;
options.create_if_missing = true;
options.compaction_filter = cf.get();
DB* db = nullptr;
DestroyDB(db_name_, options);
Status s = DB::Open(options, db_name_, &db);
ASSERT_OK(s);
cf->SetDB(db);
// Write one L0 file
db->Put(WriteOptions(), "K1", "V1");
db->Flush(FlushOptions());
// Compact all L0 files using CompactFiles
rocksdb::ColumnFamilyMetaData meta;
db->GetColumnFamilyMetaData(&meta);
for (auto& file : meta.levels[0].files) {
std::string fname = file.db_path + "/" + file.name;
ASSERT_OK(
db->CompactFiles(rocksdb::CompactionOptions(), {fname}, 0));
}
delete db;
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
#include <stdio.h>
int main(int argc, char** argv) {
fprintf(stderr,
"SKIPPED as DBImpl::CompactFiles is not supported in ROCKSDB_LITE\n");
return 0;
}
#endif // !ROCKSDB_LITE

166
db/compacted_db_impl.cc Normal file
View File

@@ -0,0 +1,166 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include "db/compacted_db_impl.h"
#include "db/db_impl.h"
#include "db/version_set.h"
#include "table/get_context.h"
namespace rocksdb {
extern void MarkKeyMayExist(void* arg);
extern bool SaveValue(void* arg, const ParsedInternalKey& parsed_key,
const Slice& v, bool hit_and_return);
CompactedDBImpl::CompactedDBImpl(
const DBOptions& options, const std::string& dbname)
: DBImpl(options, dbname) {
}
CompactedDBImpl::~CompactedDBImpl() {
}
size_t CompactedDBImpl::FindFile(const Slice& key) {
size_t left = 0;
size_t right = files_.num_files - 1;
while (left < right) {
size_t mid = (left + right) >> 1;
const FdWithKeyRange& f = files_.files[mid];
if (user_comparator_->Compare(ExtractUserKey(f.largest_key), key) < 0) {
// Key at "mid.largest" is < "target". Therefore all
// files at or before "mid" are uninteresting.
left = mid + 1;
} else {
// Key at "mid.largest" is >= "target". Therefore all files
// after "mid" are uninteresting.
right = mid;
}
}
return right;
}
Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
const Slice& key, PinnableSlice* value) {
GetContext get_context(user_comparator_, nullptr, nullptr, nullptr,
GetContext::kNotFound, key, value, nullptr, nullptr,
nullptr, nullptr);
LookupKey lkey(key, kMaxSequenceNumber);
files_.files[FindFile(key)].fd.table_reader->Get(
options, lkey.internal_key(), &get_context);
if (get_context.State() == GetContext::kFound) {
return Status::OK();
}
return Status::NotFound();
}
std::vector<Status> CompactedDBImpl::MultiGet(const ReadOptions& options,
const std::vector<ColumnFamilyHandle*>&,
const std::vector<Slice>& keys, std::vector<std::string>* values) {
autovector<TableReader*, 16> reader_list;
for (const auto& key : keys) {
const FdWithKeyRange& f = files_.files[FindFile(key)];
if (user_comparator_->Compare(key, ExtractUserKey(f.smallest_key)) < 0) {
reader_list.push_back(nullptr);
} else {
LookupKey lkey(key, kMaxSequenceNumber);
f.fd.table_reader->Prepare(lkey.internal_key());
reader_list.push_back(f.fd.table_reader);
}
}
std::vector<Status> statuses(keys.size(), Status::NotFound());
values->resize(keys.size());
int idx = 0;
for (auto* r : reader_list) {
if (r != nullptr) {
PinnableSlice pinnable_val;
std::string& value = (*values)[idx];
GetContext get_context(user_comparator_, nullptr, nullptr, nullptr,
GetContext::kNotFound, keys[idx], &pinnable_val,
nullptr, nullptr, nullptr, nullptr);
LookupKey lkey(keys[idx], kMaxSequenceNumber);
r->Get(options, lkey.internal_key(), &get_context);
value.assign(pinnable_val.data(), pinnable_val.size());
if (get_context.State() == GetContext::kFound) {
statuses[idx] = Status::OK();
}
}
++idx;
}
return statuses;
}
Status CompactedDBImpl::Init(const Options& options) {
mutex_.Lock();
ColumnFamilyDescriptor cf(kDefaultColumnFamilyName,
ColumnFamilyOptions(options));
Status s = Recover({cf}, true /* read only */, false, true);
if (s.ok()) {
cfd_ = reinterpret_cast<ColumnFamilyHandleImpl*>(
DefaultColumnFamily())->cfd();
delete cfd_->InstallSuperVersion(new SuperVersion(), &mutex_);
}
mutex_.Unlock();
if (!s.ok()) {
return s;
}
NewThreadStatusCfInfo(cfd_);
version_ = cfd_->GetSuperVersion()->current;
user_comparator_ = cfd_->user_comparator();
auto* vstorage = version_->storage_info();
if (vstorage->num_non_empty_levels() == 0) {
return Status::NotSupported("no file exists");
}
const LevelFilesBrief& l0 = vstorage->LevelFilesBrief(0);
// L0 should not have files
if (l0.num_files > 1) {
return Status::NotSupported("L0 contain more than 1 file");
}
if (l0.num_files == 1) {
if (vstorage->num_non_empty_levels() > 1) {
return Status::NotSupported("Both L0 and other level contain files");
}
files_ = l0;
return Status::OK();
}
for (int i = 1; i < vstorage->num_non_empty_levels() - 1; ++i) {
if (vstorage->LevelFilesBrief(i).num_files > 0) {
return Status::NotSupported("Other levels also contain files");
}
}
int level = vstorage->num_non_empty_levels() - 1;
if (vstorage->LevelFilesBrief(level).num_files > 0) {
files_ = vstorage->LevelFilesBrief(level);
return Status::OK();
}
return Status::NotSupported("no file exists");
}
Status CompactedDBImpl::Open(const Options& options,
const std::string& dbname, DB** dbptr) {
*dbptr = nullptr;
if (options.max_open_files != -1) {
return Status::InvalidArgument("require max_open_files = -1");
}
if (options.merge_operator.get() != nullptr) {
return Status::InvalidArgument("merge operator is not supported");
}
DBOptions db_options(options);
std::unique_ptr<CompactedDBImpl> db(new CompactedDBImpl(db_options, dbname));
Status s = db->Init(options);
if (s.ok()) {
ROCKS_LOG_INFO(db->immutable_db_options_.info_log,
"Opened the db as fully compacted mode");
LogFlush(db->immutable_db_options_.info_log);
*dbptr = db.release();
}
return s;
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

102
db/compacted_db_impl.h Normal file
View File

@@ -0,0 +1,102 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#ifndef ROCKSDB_LITE
#include "db/db_impl.h"
#include <vector>
#include <string>
namespace rocksdb {
class CompactedDBImpl : public DBImpl {
public:
CompactedDBImpl(const DBOptions& options, const std::string& dbname);
virtual ~CompactedDBImpl();
static Status Open(const Options& options, const std::string& dbname,
DB** dbptr);
// Implementations of the DB interface
using DB::Get;
virtual Status Get(const ReadOptions& options,
ColumnFamilyHandle* column_family, const Slice& key,
PinnableSlice* value) override;
using DB::MultiGet;
virtual std::vector<Status> MultiGet(
const ReadOptions& options,
const std::vector<ColumnFamilyHandle*>&,
const std::vector<Slice>& keys, std::vector<std::string>* values)
override;
using DBImpl::Put;
virtual Status Put(const WriteOptions& options,
ColumnFamilyHandle* column_family, const Slice& key,
const Slice& value) override {
return Status::NotSupported("Not supported in compacted db mode.");
}
using DBImpl::Merge;
virtual Status Merge(const WriteOptions& options,
ColumnFamilyHandle* column_family, const Slice& key,
const Slice& value) override {
return Status::NotSupported("Not supported in compacted db mode.");
}
using DBImpl::Delete;
virtual Status Delete(const WriteOptions& options,
ColumnFamilyHandle* column_family,
const Slice& key) override {
return Status::NotSupported("Not supported in compacted db mode.");
}
virtual Status Write(const WriteOptions& options,
WriteBatch* updates) override {
return Status::NotSupported("Not supported in compacted db mode.");
}
using DBImpl::CompactRange;
virtual Status CompactRange(const CompactRangeOptions& options,
ColumnFamilyHandle* column_family,
const Slice* begin, const Slice* end) override {
return Status::NotSupported("Not supported in compacted db mode.");
}
virtual Status DisableFileDeletions() override {
return Status::NotSupported("Not supported in compacted db mode.");
}
virtual Status EnableFileDeletions(bool force) override {
return Status::NotSupported("Not supported in compacted db mode.");
}
virtual Status GetLiveFiles(std::vector<std::string>&,
uint64_t* manifest_file_size,
bool flush_memtable = true) override {
return Status::NotSupported("Not supported in compacted db mode.");
}
using DBImpl::Flush;
virtual Status Flush(const FlushOptions& options,
ColumnFamilyHandle* column_family) override {
return Status::NotSupported("Not supported in compacted db mode.");
}
using DB::IngestExternalFile;
virtual Status IngestExternalFile(
ColumnFamilyHandle* column_family,
const std::vector<std::string>& external_files,
const IngestExternalFileOptions& ingestion_options) override {
return Status::NotSupported("Not supported in compacted db mode.");
}
private:
friend class DB;
inline size_t FindFile(const Slice& key);
Status Init(const Options& options);
ColumnFamilyData* cfd_;
Version* version_;
const Comparator* user_comparator_;
LevelFilesBrief files_;
// No copying allowed
CompactedDBImpl(const CompactedDBImpl&);
void operator=(const CompactedDBImpl&);
};
}
#endif // ROCKSDB_LITE

495
db/compaction.cc
View File

@@ -1,7 +1,7 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
@@ -9,12 +9,17 @@
#include "db/compaction.h"
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include <inttypes.h>
#include <vector>
#include "db/column_family.h"
#include "util/logging.h"
#include "rocksdb/compaction_filter.h"
#include "util/string_util.h"
#include "util/sync_point.h"
namespace rocksdb {
@@ -26,51 +31,170 @@ uint64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
return sum;
}
Compaction::Compaction(Version* input_version, int start_level, int out_level,
uint64_t target_file_size,
uint64_t max_grandparent_overlap_bytes,
uint32_t output_path_id,
CompressionType output_compression, bool seek_compaction,
bool deletion_compaction)
: start_level_(start_level),
output_level_(out_level),
max_output_file_size_(target_file_size),
max_grandparent_overlap_bytes_(max_grandparent_overlap_bytes),
input_version_(input_version),
number_levels_(input_version_->NumberLevels()),
cfd_(input_version_->cfd_),
output_path_id_(output_path_id),
output_compression_(output_compression),
seek_compaction_(seek_compaction),
deletion_compaction_(deletion_compaction),
grandparent_index_(0),
seen_key_(false),
overlapped_bytes_(0),
base_index_(-1),
parent_index_(-1),
score_(0),
bottommost_level_(false),
is_full_compaction_(false),
is_manual_compaction_(false),
level_ptrs_(std::vector<size_t>(number_levels_)) {
void Compaction::SetInputVersion(Version* _input_version) {
input_version_ = _input_version;
cfd_ = input_version_->cfd();
cfd_->Ref();
input_version_->Ref();
edit_ = new VersionEdit();
edit_->SetColumnFamily(cfd_->GetID());
for (int i = 0; i < number_levels_; i++) {
level_ptrs_[i] = 0;
}
int num_levels = output_level_ - start_level_ + 1;
input_levels_.resize(num_levels);
inputs_.resize(num_levels);
for (int i = 0; i < num_levels; ++i) {
inputs_[i].level = start_level_ + i;
edit_.SetColumnFamily(cfd_->GetID());
}
void Compaction::GetBoundaryKeys(
VersionStorageInfo* vstorage,
const std::vector<CompactionInputFiles>& inputs, Slice* smallest_user_key,
Slice* largest_user_key) {
bool initialized = false;
const Comparator* ucmp = vstorage->InternalComparator()->user_comparator();
for (size_t i = 0; i < inputs.size(); ++i) {
if (inputs[i].files.empty()) {
continue;
}
if (inputs[i].level == 0) {
// we need to consider all files on level 0
for (const auto* f : inputs[i].files) {
const Slice& start_user_key = f->smallest.user_key();
if (!initialized ||
ucmp->Compare(start_user_key, *smallest_user_key) < 0) {
*smallest_user_key = start_user_key;
}
const Slice& end_user_key = f->largest.user_key();
if (!initialized ||
ucmp->Compare(end_user_key, *largest_user_key) > 0) {
*largest_user_key = end_user_key;
}
initialized = true;
}
} else {
// we only need to consider the first and last file
const Slice& start_user_key = inputs[i].files[0]->smallest.user_key();
if (!initialized ||
ucmp->Compare(start_user_key, *smallest_user_key) < 0) {
*smallest_user_key = start_user_key;
}
const Slice& end_user_key = inputs[i].files.back()->largest.user_key();
if (!initialized || ucmp->Compare(end_user_key, *largest_user_key) > 0) {
*largest_user_key = end_user_key;
}
initialized = true;
}
}
}
// helper function to determine if compaction is creating files at the
// bottommost level
bool Compaction::IsBottommostLevel(
int output_level, VersionStorageInfo* vstorage,
const std::vector<CompactionInputFiles>& inputs) {
if (inputs[0].level == 0 &&
inputs[0].files.back() != vstorage->LevelFiles(0).back()) {
return false;
}
Slice smallest_key, largest_key;
GetBoundaryKeys(vstorage, inputs, &smallest_key, &largest_key);
// Checks whether there are files living beyond the output_level.
// If lower levels have files, it checks for overlap between files
// if the compaction process and those files.
// Bottomlevel optimizations can be made if there are no files in
// lower levels or if there is no overlap with the files in
// the lower levels.
for (int i = output_level + 1; i < vstorage->num_levels(); i++) {
// It is not the bottommost level if there are files in higher
// levels when the output level is 0 or if there are files in
// higher levels which overlap with files to be compacted.
// output_level == 0 means that we want it to be considered
// s the bottommost level only if the last file on the level
// is a part of the files to be compacted - this is verified by
// the first if condition in this function
if (vstorage->NumLevelFiles(i) > 0 &&
(output_level == 0 ||
vstorage->OverlapInLevel(i, &smallest_key, &largest_key))) {
return false;
}
}
return true;
}
// test function to validate the functionality of IsBottommostLevel()
// function -- determines if compaction with inputs and storage is bottommost
bool Compaction::TEST_IsBottommostLevel(
int output_level, VersionStorageInfo* vstorage,
const std::vector<CompactionInputFiles>& inputs) {
return IsBottommostLevel(output_level, vstorage, inputs);
}
bool Compaction::IsFullCompaction(
VersionStorageInfo* vstorage,
const std::vector<CompactionInputFiles>& inputs) {
size_t num_files_in_compaction = 0;
size_t total_num_files = 0;
for (int l = 0; l < vstorage->num_levels(); l++) {
total_num_files += vstorage->NumLevelFiles(l);
}
for (size_t i = 0; i < inputs.size(); i++) {
num_files_in_compaction += inputs[i].size();
}
return num_files_in_compaction == total_num_files;
}
Compaction::Compaction(VersionStorageInfo* vstorage,
const ImmutableCFOptions& _immutable_cf_options,
const MutableCFOptions& _mutable_cf_options,
std::vector<CompactionInputFiles> _inputs,
int _output_level, uint64_t _target_file_size,
uint64_t _max_compaction_bytes, uint32_t _output_path_id,
CompressionType _compression,
std::vector<FileMetaData*> _grandparents,
bool _manual_compaction, double _score,
bool _deletion_compaction,
CompactionReason _compaction_reason)
: input_vstorage_(vstorage),
start_level_(_inputs[0].level),
output_level_(_output_level),
max_output_file_size_(_target_file_size),
max_compaction_bytes_(_max_compaction_bytes),
immutable_cf_options_(_immutable_cf_options),
mutable_cf_options_(_mutable_cf_options),
input_version_(nullptr),
number_levels_(vstorage->num_levels()),
cfd_(nullptr),
output_path_id_(_output_path_id),
output_compression_(_compression),
deletion_compaction_(_deletion_compaction),
inputs_(std::move(_inputs)),
grandparents_(std::move(_grandparents)),
score_(_score),
bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
is_manual_compaction_(_manual_compaction),
is_trivial_move_(false),
compaction_reason_(_compaction_reason) {
MarkFilesBeingCompacted(true);
if (is_manual_compaction_) {
compaction_reason_ = CompactionReason::kManualCompaction;
}
#ifndef NDEBUG
for (size_t i = 1; i < inputs_.size(); ++i) {
assert(inputs_[i].level > inputs_[i - 1].level);
}
#endif
// setup input_levels_
{
input_levels_.resize(num_input_levels());
for (size_t which = 0; which < num_input_levels(); which++) {
DoGenerateLevelFilesBrief(&input_levels_[which], inputs_[which].files,
&arena_);
}
}
GetBoundaryKeys(vstorage, inputs_, &smallest_user_key_, &largest_user_key_);
}
Compaction::~Compaction() {
delete edit_;
if (input_version_ != nullptr) {
input_version_->Unref();
}
@@ -81,11 +205,17 @@ Compaction::~Compaction() {
}
}
void Compaction::GenerateFileLevels() {
input_levels_.resize(num_input_levels());
for (int which = 0; which < num_input_levels(); which++) {
DoGenerateFileLevel(&input_levels_[which], inputs_[which].files, &arena_);
bool Compaction::InputCompressionMatchesOutput() const {
int base_level = input_vstorage_->base_level();
bool matches = (GetCompressionType(immutable_cf_options_, input_vstorage_,
mutable_cf_options_, start_level_,
base_level) == output_compression_);
if (matches) {
TEST_SYNC_POINT("Compaction::InputCompressionMatchesOutput:Matches");
return true;
}
TEST_SYNC_POINT("Compaction::InputCompressionMatchesOutput:DidntMatch");
return matches;
}
bool Compaction::IsTrivialMove() const {
@@ -93,129 +223,151 @@ bool Compaction::IsTrivialMove() const {
// Otherwise, the move could create a parent file that will require
// a very expensive merge later on.
// If start_level_== output_level_, the purpose is to force compaction
// filter to be applied to that level, and thus cannot be a trivia move.
return (start_level_ != output_level_ &&
num_input_levels() == 2 &&
num_input_files(0) == 1 &&
num_input_files(1) == 0 &&
TotalFileSize(grandparents_) <= max_grandparent_overlap_bytes_);
}
// filter to be applied to that level, and thus cannot be a trivial move.
void Compaction::AddInputDeletions(VersionEdit* edit) {
for (int which = 0; which < num_input_levels(); which++) {
for (size_t i = 0; i < inputs_[which].size(); i++) {
edit->DeleteFile(level(which), inputs_[which][i]->fd.GetNumber());
// Check if start level have files with overlapping ranges
if (start_level_ == 0 && input_vstorage_->level0_non_overlapping() == false) {
// We cannot move files from L0 to L1 if the files are overlapping
return false;
}
if (is_manual_compaction_ &&
(immutable_cf_options_.compaction_filter != nullptr ||
immutable_cf_options_.compaction_filter_factory != nullptr)) {
// This is a manual compaction and we have a compaction filter that should
// be executed, we cannot do a trivial move
return false;
}
// Used in universal compaction, where trivial move can be done if the
// input files are non overlapping
if ((immutable_cf_options_.compaction_options_universal.allow_trivial_move) &&
(output_level_ != 0)) {
return is_trivial_move_;
}
if (!(start_level_ != output_level_ && num_input_levels() == 1 &&
input(0, 0)->fd.GetPathId() == output_path_id() &&
InputCompressionMatchesOutput())) {
return false;
}
// assert inputs_.size() == 1
for (const auto& file : inputs_.front().files) {
std::vector<FileMetaData*> file_grand_parents;
if (output_level_ + 1 >= number_levels_) {
continue;
}
input_vstorage_->GetOverlappingInputs(output_level_ + 1, &file->smallest,
&file->largest, &file_grand_parents);
const auto compaction_size =
file->fd.GetFileSize() + TotalFileSize(file_grand_parents);
if (compaction_size > max_compaction_bytes_) {
return false;
}
}
}
bool Compaction::KeyNotExistsBeyondOutputLevel(const Slice& user_key) {
assert(cfd_->options()->compaction_style != kCompactionStyleFIFO);
if (cfd_->options()->compaction_style == kCompactionStyleUniversal) {
return bottommost_level_;
}
// Maybe use binary search to find right entry instead of linear search?
const Comparator* user_cmp = cfd_->user_comparator();
for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
const std::vector<FileMetaData*>& files = input_version_->files_[lvl];
for (; level_ptrs_[lvl] < files.size(); ) {
FileMetaData* f = files[level_ptrs_[lvl]];
if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
// We've advanced far enough
if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
// Key falls in this file's range, so definitely
// exists beyond output level
return false;
}
break;
}
level_ptrs_[lvl]++;
}
}
return true;
}
bool Compaction::ShouldStopBefore(const Slice& internal_key) {
// Scan to find earliest grandparent file that contains key.
const InternalKeyComparator* icmp = &cfd_->internal_comparator();
while (grandparent_index_ < grandparents_.size() &&
icmp->Compare(internal_key,
grandparents_[grandparent_index_]->largest.Encode()) > 0) {
if (seen_key_) {
overlapped_bytes_ += grandparents_[grandparent_index_]->fd.GetFileSize();
void Compaction::AddInputDeletions(VersionEdit* out_edit) {
for (size_t which = 0; which < num_input_levels(); which++) {
for (size_t i = 0; i < inputs_[which].size(); i++) {
out_edit->DeleteFile(level(which), inputs_[which][i]->fd.GetNumber());
}
assert(grandparent_index_ + 1 >= grandparents_.size() ||
icmp->Compare(grandparents_[grandparent_index_]->largest.Encode(),
grandparents_[grandparent_index_+1]->smallest.Encode())
< 0);
grandparent_index_++;
}
seen_key_ = true;
}
if (overlapped_bytes_ > max_grandparent_overlap_bytes_) {
// Too much overlap for current output; start new output
overlapped_bytes_ = 0;
bool Compaction::KeyNotExistsBeyondOutputLevel(
const Slice& user_key, std::vector<size_t>* level_ptrs) const {
assert(input_version_ != nullptr);
assert(level_ptrs != nullptr);
assert(level_ptrs->size() == static_cast<size_t>(number_levels_));
if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
if (output_level_ == 0) {
return false;
}
// Maybe use binary search to find right entry instead of linear search?
const Comparator* user_cmp = cfd_->user_comparator();
for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
const std::vector<FileMetaData*>& files =
input_vstorage_->LevelFiles(lvl);
for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) {
auto* f = files[level_ptrs->at(lvl)];
if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
// We've advanced far enough
if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
// Key falls in this file's range, so definitely
// exists beyond output level
return false;
}
break;
}
}
}
return true;
} else {
return false;
return bottommost_level_;
}
}
// Mark (or clear) each file that is being compacted
void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
for (int i = 0; i < num_input_levels(); i++) {
for (unsigned int j = 0; j < inputs_[i].size(); j++) {
assert(mark_as_compacted ? !inputs_[i][j]->being_compacted :
inputs_[i][j]->being_compacted);
for (size_t i = 0; i < num_input_levels(); i++) {
for (size_t j = 0; j < inputs_[i].size(); j++) {
assert(mark_as_compacted ? !inputs_[i][j]->being_compacted
: inputs_[i][j]->being_compacted);
inputs_[i][j]->being_compacted = mark_as_compacted;
}
}
}
// Is this compaction producing files at the bottommost level?
void Compaction::SetupBottomMostLevel(bool is_manual) {
assert(cfd_->options()->compaction_style != kCompactionStyleFIFO);
if (cfd_->options()->compaction_style == kCompactionStyleUniversal) {
// If universal compaction style is used and manual
// compaction is occuring, then we are guaranteed that
// all files will be picked in a single compaction
// run. We can safely set bottommost_level_ = true.
// If it is not manual compaction, then bottommost_level_
// is already set when the Compaction was created.
if (is_manual) {
bottommost_level_ = true;
// Sample output:
// If compacting 3 L0 files, 2 L3 files and 1 L4 file, and outputting to L5,
// print: "3@0 + 2@3 + 1@4 files to L5"
const char* Compaction::InputLevelSummary(
InputLevelSummaryBuffer* scratch) const {
int len = 0;
bool is_first = true;
for (auto& input_level : inputs_) {
if (input_level.empty()) {
continue;
}
return;
}
bottommost_level_ = true;
// checks whether there are files living beyond the output_level.
for (int i = output_level_ + 1; i < number_levels_; i++) {
if (input_version_->NumLevelFiles(i) > 0) {
bottommost_level_ = false;
break;
if (!is_first) {
len +=
snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, " + ");
} else {
is_first = false;
}
len += snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len,
"%" ROCKSDB_PRIszt "@%d", input_level.size(),
input_level.level);
}
snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len,
" files to L%d", output_level());
return scratch->buffer;
}
void Compaction::ReleaseInputs() {
if (input_version_ != nullptr) {
input_version_->Unref();
input_version_ = nullptr;
}
if (cfd_ != nullptr) {
if (cfd_->Unref()) {
delete cfd_;
uint64_t Compaction::CalculateTotalInputSize() const {
uint64_t size = 0;
for (auto& input_level : inputs_) {
for (auto f : input_level.files) {
size += f->fd.GetFileSize();
}
cfd_ = nullptr;
}
return size;
}
void Compaction::ReleaseCompactionFiles(Status status) {
MarkFilesBeingCompacted(false);
cfd_->compaction_picker()->ReleaseCompactionFiles(this, status);
}
void Compaction::ResetNextCompactionIndex() {
input_version_->ResetNextCompactionIndex(start_level_);
assert(input_version_ != nullptr);
input_vstorage_->ResetNextCompactionIndex(start_level_);
}
namespace {
@@ -223,7 +375,7 @@ int InputSummary(const std::vector<FileMetaData*>& files, char* output,
int len) {
*output = '\0';
int write = 0;
for (unsigned int i = 0; i < files.size(); i++) {
for (size_t i = 0; i < files.size(); i++) {
int sz = len - write;
int ret;
char sztxt[16];
@@ -240,22 +392,21 @@ int InputSummary(const std::vector<FileMetaData*>& files, char* output,
void Compaction::Summary(char* output, int len) {
int write =
snprintf(output, len, "Base version %" PRIu64
" Base level %d, seek compaction:%d, inputs: [",
input_version_->GetVersionNumber(),
start_level_, seek_compaction_);
snprintf(output, len, "Base version %" PRIu64 " Base level %d, inputs: [",
input_version_->GetVersionNumber(), start_level_);
if (write < 0 || write >= len) {
return;
}
for (int level = 0; level < num_input_levels(); ++level) {
if (level > 0) {
for (size_t level_iter = 0; level_iter < num_input_levels(); ++level_iter) {
if (level_iter > 0) {
write += snprintf(output + write, len - write, "], [");
if (write < 0 || write >= len) {
return;
}
}
write += InputSummary(inputs_[level].files, output + write, len - write);
write +=
InputSummary(inputs_[level_iter].files, output + write, len - write);
if (write < 0 || write >= len) {
return;
}
@@ -264,22 +415,66 @@ void Compaction::Summary(char* output, int len) {
snprintf(output + write, len - write, "]");
}
uint64_t Compaction::OutputFilePreallocationSize() {
uint64_t Compaction::OutputFilePreallocationSize() const {
uint64_t preallocation_size = 0;
if (cfd_->options()->compaction_style == kCompactionStyleLevel) {
preallocation_size =
cfd_->compaction_picker()->MaxFileSizeForLevel(output_level());
if (max_output_file_size_ != port::kMaxUint64 &&
(cfd_->ioptions()->compaction_style == kCompactionStyleLevel ||
output_level() > 0)) {
preallocation_size = max_output_file_size_;
} else {
for (int level = 0; level < num_input_levels(); ++level) {
for (const auto& f : inputs_[level].files) {
preallocation_size += f->fd.GetFileSize();
for (const auto& level_files : inputs_) {
for (const auto& file : level_files.files) {
preallocation_size += file->fd.GetFileSize();
}
}
}
// Over-estimate slightly so we don't end up just barely crossing
// the threshold
return preallocation_size * 1.1;
return preallocation_size + (preallocation_size / 10);
}
std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {
if (!cfd_->ioptions()->compaction_filter_factory) {
return nullptr;
}
CompactionFilter::Context context;
context.is_full_compaction = is_full_compaction_;
context.is_manual_compaction = is_manual_compaction_;
context.column_family_id = cfd_->GetID();
return cfd_->ioptions()->compaction_filter_factory->CreateCompactionFilter(
context);
}
bool Compaction::IsOutputLevelEmpty() const {
return inputs_.back().level != output_level_ || inputs_.back().empty();
}
bool Compaction::ShouldFormSubcompactions() const {
if (immutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) {
return false;
}
if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
return start_level_ == 0 && output_level_ > 0 && !IsOutputLevelEmpty();
} else if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
return number_levels_ > 1 && output_level_ > 0;
} else {
return false;
}
}
uint64_t Compaction::MaxInputFileCreationTime() const {
uint64_t max_creation_time = 0;
for (const auto& file : inputs_[0].files) {
if (file->fd.table_reader != nullptr &&
file->fd.table_reader->GetTableProperties() != nullptr) {
uint64_t creation_time =
file->fd.table_reader->GetTableProperties()->creation_time;
max_creation_time = std::max(max_creation_time, creation_time);
}
}
return max_creation_time;
}
} // namespace rocksdb

296
db/compaction.h
View File

@@ -1,16 +1,17 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include "db/version_set.h"
#include "options/cf_options.h"
#include "util/arena.h"
#include "util/autovector.h"
#include "db/version_set.h"
namespace rocksdb {
@@ -22,15 +23,28 @@ struct CompactionInputFiles {
inline bool empty() const { return files.empty(); }
inline size_t size() const { return files.size(); }
inline void clear() { files.clear(); }
inline FileMetaData* operator[](int i) const { return files[i]; }
inline FileMetaData* operator[](size_t i) const { return files[i]; }
};
class Version;
class ColumnFamilyData;
class VersionStorageInfo;
class CompactionFilter;
// A Compaction encapsulates information about a compaction.
class Compaction {
public:
Compaction(VersionStorageInfo* input_version,
const ImmutableCFOptions& immutable_cf_options,
const MutableCFOptions& mutable_cf_options,
std::vector<CompactionInputFiles> inputs, int output_level,
uint64_t target_file_size, uint64_t max_compaction_bytes,
uint32_t output_path_id, CompressionType compression,
std::vector<FileMetaData*> grandparents,
bool manual_compaction = false, double score = -1,
bool deletion_compaction = false,
CompactionReason compaction_reason = CompactionReason::kUnknown);
// No copying allowed
Compaction(const Compaction&) = delete;
void operator=(const Compaction&) = delete;
@@ -39,25 +53,27 @@ class Compaction {
// Returns the level associated to the specified compaction input level.
// If compaction_input_level is not specified, then input_level is set to 0.
int level(int compaction_input_level = 0) const {
int level(size_t compaction_input_level = 0) const {
return inputs_[compaction_input_level].level;
}
int start_level() const { return start_level_; }
// Outputs will go to this level
int output_level() const { return output_level_; }
// Returns the number of input levels in this compaction.
int num_input_levels() const { return inputs_.size(); }
size_t num_input_levels() const { return inputs_.size(); }
// Return the object that holds the edits to the descriptor done
// by this compaction.
VersionEdit* edit() const { return edit_; }
VersionEdit* edit() { return &edit_; }
// Returns the number of input files associated to the specified
// compaction input level.
// The function will return 0 if when "compaction_input_level" < 0
// or "compaction_input_level" >= "num_input_levels()".
int num_input_files(size_t compaction_input_level) const {
size_t num_input_files(size_t compaction_input_level) const {
if (compaction_input_level < inputs_.size()) {
return inputs_[compaction_input_level].size();
}
@@ -74,7 +90,7 @@ class Compaction {
// specified compaction input level.
// REQUIREMENT: "compaction_input_level" must be >= 0 and
// < "input_levels()"
FileMetaData* input(size_t compaction_input_level, int i) const {
FileMetaData* input(size_t compaction_input_level, size_t i) const {
assert(compaction_input_level < inputs_.size());
return inputs_[compaction_input_level][i];
}
@@ -83,55 +99,47 @@ class Compaction {
// input level.
// REQUIREMENT: "compaction_input_level" must be >= 0 and
// < "input_levels()"
std::vector<FileMetaData*>* const inputs(size_t compaction_input_level) {
const std::vector<FileMetaData*>* inputs(
size_t compaction_input_level) const {
assert(compaction_input_level < inputs_.size());
return &inputs_[compaction_input_level].files;
}
// Returns the FileLevel of the specified compaction input level.
FileLevel* input_levels(int compaction_input_level) {
const std::vector<CompactionInputFiles>* inputs() { return &inputs_; }
// Returns the LevelFilesBrief of the specified compaction input level.
const LevelFilesBrief* input_levels(size_t compaction_input_level) const {
return &input_levels_[compaction_input_level];
}
// Maximum size of files to build during this compaction.
uint64_t MaxOutputFileSize() const { return max_output_file_size_; }
uint64_t max_output_file_size() const { return max_output_file_size_; }
// What compression for output
CompressionType OutputCompressionType() const { return output_compression_; }
CompressionType output_compression() const { return output_compression_; }
// Whether need to write output file to second DB path.
uint32_t GetOutputPathId() const { return output_path_id_; }
// Generate input_levels_ from inputs_
// Should be called when inputs_ is stable
void GenerateFileLevels();
uint32_t output_path_id() const { return output_path_id_; }
// Is this a trivial compaction that can be implemented by just
// moving a single input file to the next level (no merging or splitting)
bool IsTrivialMove() const;
// If true, then the comaction can be done by simply deleting input files.
bool IsDeletionCompaction() const {
return deletion_compaction_;
}
// If true, then the compaction can be done by simply deleting input files.
bool deletion_compaction() const { return deletion_compaction_; }
// Add all inputs to this compaction as delete operations to *edit.
void AddInputDeletions(VersionEdit* edit);
// Returns true if the available information we have guarantees that
// the input "user_key" does not exist in any level beyond "output_level()".
bool KeyNotExistsBeyondOutputLevel(const Slice& user_key);
// Returns true iff we should stop building the current output
// before processing "internal_key".
bool ShouldStopBefore(const Slice& internal_key);
// Release the input version for the compaction, once the compaction
// is successful.
void ReleaseInputs();
bool KeyNotExistsBeyondOutputLevel(const Slice& user_key,
std::vector<size_t>* level_ptrs) const;
// Clear all files to indicate that they are not being compacted
// Delete this compaction from the list of running compactions.
//
// Requirement: DB mutex held
void ReleaseCompactionFiles(Status status);
// Returns the summary of the compaction in "output" with maximum "len"
@@ -143,92 +151,172 @@ class Compaction {
double score() const { return score_; }
// Is this compaction creating a file in the bottom most level?
bool BottomMostLevel() { return bottommost_level_; }
bool bottommost_level() const { return bottommost_level_; }
// Does this compaction include all sst files?
bool IsFullCompaction() { return is_full_compaction_; }
bool is_full_compaction() const { return is_full_compaction_; }
// Was this compaction triggered manually by the client?
bool IsManualCompaction() { return is_manual_compaction_; }
bool is_manual_compaction() const { return is_manual_compaction_; }
// Used when allow_trivial_move option is set in
// Universal compaction. If all the input files are
// non overlapping, then is_trivial_move_ variable
// will be set true, else false
void set_is_trivial_move(bool trivial_move) {
is_trivial_move_ = trivial_move;
}
// Used when allow_trivial_move option is set in
// Universal compaction. Returns true, if the input files
// are non-overlapping and can be trivially moved.
bool is_trivial_move() const { return is_trivial_move_; }
// How many total levels are there?
int number_levels() const { return number_levels_; }
// Return the ImmutableCFOptions that should be used throughout the compaction
// procedure
const ImmutableCFOptions* immutable_cf_options() const {
return &immutable_cf_options_;
}
// Return the MutableCFOptions that should be used throughout the compaction
// procedure
const MutableCFOptions* mutable_cf_options() const {
return &mutable_cf_options_;
}
// Returns the size in bytes that the output file should be preallocated to.
// In level compaction, that is max_file_size_. In universal compaction, that
// is the sum of all input file sizes.
uint64_t OutputFilePreallocationSize();
uint64_t OutputFilePreallocationSize() const;
private:
friend class CompactionPicker;
friend class UniversalCompactionPicker;
friend class FIFOCompactionPicker;
friend class LevelCompactionPicker;
void SetInputVersion(Version* input_version);
Compaction(Version* input_version, int start_level, int out_level,
uint64_t target_file_size, uint64_t max_grandparent_overlap_bytes,
uint32_t output_path_id, CompressionType output_compression,
bool seek_compaction = false, bool deletion_compaction = false);
struct InputLevelSummaryBuffer {
char buffer[128];
};
const int start_level_; // the lowest level to be compacted
const int output_level_; // levels to which output files are stored
uint64_t max_output_file_size_;
uint64_t max_grandparent_overlap_bytes_;
Version* input_version_;
VersionEdit* edit_;
int number_levels_;
ColumnFamilyData* cfd_;
Arena arena_; // Arena used to allocate space for file_levels_
const char* InputLevelSummary(InputLevelSummaryBuffer* scratch) const;
uint32_t output_path_id_;
CompressionType output_compression_;
bool seek_compaction_;
// If true, then the comaction can be done by simply deleting input files.
bool deletion_compaction_;
// Compaction input files organized by level.
autovector<CompactionInputFiles> inputs_;
// A copy of inputs_, organized more closely in memory
autovector<FileLevel, 2> input_levels_;
// State used to check for number of of overlapping grandparent files
// (grandparent == "output_level_ + 1")
// This vector is updated by Version::GetOverlappingInputs().
std::vector<FileMetaData*> grandparents_;
size_t grandparent_index_; // Index in grandparent_starts_
bool seen_key_; // Some output key has been seen
uint64_t overlapped_bytes_; // Bytes of overlap between current output
// and grandparent files
int base_index_; // index of the file in files_[start_level_]
int parent_index_; // index of some file with same range in
// files_[start_level_+1]
double score_; // score that was used to pick this compaction.
// Is this compaction creating a file in the bottom most level?
bool bottommost_level_;
// Does this compaction include all sst files?
bool is_full_compaction_;
// Is this compaction requested by the client?
bool is_manual_compaction_;
// "level_ptrs_" holds indices into "input_version_->levels_", where each
// index remembers which file of an associated level we are currently used
// to check KeyNotExistsBeyondOutputLevel() for deletion operation.
// As it is for checking KeyNotExistsBeyondOutputLevel(), it only
// records indices for all levels beyond "output_level_".
std::vector<size_t> level_ptrs_;
// mark (or clear) all files that are being compacted
void MarkFilesBeingCompacted(bool mark_as_compacted);
// Initialize whether the compaction is producing files at the
// bottommost level.
//
// @see BottomMostLevel()
void SetupBottomMostLevel(bool is_manual);
uint64_t CalculateTotalInputSize() const;
// In case of compaction error, reset the nextIndex that is used
// to pick up the next file to be compacted from files_by_size_
void ResetNextCompactionIndex();
// Create a CompactionFilter from compaction_filter_factory
std::unique_ptr<CompactionFilter> CreateCompactionFilter() const;
// Is the input level corresponding to output_level_ empty?
bool IsOutputLevelEmpty() const;
// Should this compaction be broken up into smaller ones run in parallel?
bool ShouldFormSubcompactions() const;
// test function to validate the functionality of IsBottommostLevel()
// function -- determines if compaction with inputs and storage is bottommost
static bool TEST_IsBottommostLevel(
int output_level, VersionStorageInfo* vstorage,
const std::vector<CompactionInputFiles>& inputs);
TablePropertiesCollection GetOutputTableProperties() const {
return output_table_properties_;
}
void SetOutputTableProperties(TablePropertiesCollection tp) {
output_table_properties_ = std::move(tp);
}
Slice GetSmallestUserKey() const { return smallest_user_key_; }
Slice GetLargestUserKey() const { return largest_user_key_; }
CompactionReason compaction_reason() { return compaction_reason_; }
const std::vector<FileMetaData*>& grandparents() const {
return grandparents_;
}
uint64_t max_compaction_bytes() const { return max_compaction_bytes_; }
uint64_t MaxInputFileCreationTime() const;
private:
// mark (or clear) all files that are being compacted
void MarkFilesBeingCompacted(bool mark_as_compacted);
// get the smallest and largest key present in files to be compacted
static void GetBoundaryKeys(VersionStorageInfo* vstorage,
const std::vector<CompactionInputFiles>& inputs,
Slice* smallest_key, Slice* largest_key);
// helper function to determine if compaction with inputs and storage is
// bottommost
static bool IsBottommostLevel(
int output_level, VersionStorageInfo* vstorage,
const std::vector<CompactionInputFiles>& inputs);
static bool IsFullCompaction(VersionStorageInfo* vstorage,
const std::vector<CompactionInputFiles>& inputs);
VersionStorageInfo* input_vstorage_;
const int start_level_; // the lowest level to be compacted
const int output_level_; // levels to which output files are stored
uint64_t max_output_file_size_;
uint64_t max_compaction_bytes_;
const ImmutableCFOptions immutable_cf_options_;
const MutableCFOptions mutable_cf_options_;
Version* input_version_;
VersionEdit edit_;
const int number_levels_;
ColumnFamilyData* cfd_;
Arena arena_; // Arena used to allocate space for file_levels_
const uint32_t output_path_id_;
CompressionType output_compression_;
// If true, then the comaction can be done by simply deleting input files.
const bool deletion_compaction_;
// Compaction input files organized by level. Constant after construction
const std::vector<CompactionInputFiles> inputs_;
// A copy of inputs_, organized more closely in memory
autovector<LevelFilesBrief, 2> input_levels_;
// State used to check for number of overlapping grandparent files
// (grandparent == "output_level_ + 1")
std::vector<FileMetaData*> grandparents_;
const double score_; // score that was used to pick this compaction.
// Is this compaction creating a file in the bottom most level?
const bool bottommost_level_;
// Does this compaction include all sst files?
const bool is_full_compaction_;
// Is this compaction requested by the client?
const bool is_manual_compaction_;
// True if we can do trivial move in Universal multi level
// compaction
bool is_trivial_move_;
// Does input compression match the output compression?
bool InputCompressionMatchesOutput() const;
// table properties of output files
TablePropertiesCollection output_table_properties_;
// smallest user keys in compaction
Slice smallest_user_key_;
// largest user keys in compaction
Slice largest_user_key_;
// Reason for compaction
CompactionReason compaction_reason_;
};
// Utility function

35
db/compaction_iteration_stats.h Normal file
View File

@@ -0,0 +1,35 @@
// Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
struct CompactionIterationStats {
// Compaction statistics
// Doesn't include records skipped because of
// CompactionFilter::Decision::kRemoveAndSkipUntil.
int64_t num_record_drop_user = 0;
int64_t num_record_drop_hidden = 0;
int64_t num_record_drop_obsolete = 0;
int64_t num_record_drop_range_del = 0;
int64_t num_range_del_drop_obsolete = 0;
// Deletions obsoleted before bottom level due to file gap optimization.
int64_t num_optimized_del_drop_obsolete = 0;
uint64_t total_filter_time = 0;
// Input statistics
// TODO(noetzli): The stats are incomplete. They are lacking everything
// consumed by MergeHelper.
uint64_t num_input_records = 0;
uint64_t num_input_deletion_records = 0;
uint64_t num_input_corrupt_records = 0;
uint64_t total_input_raw_key_bytes = 0;
uint64_t total_input_raw_value_bytes = 0;
// Single-Delete diagnostics for exceptional situations
uint64_t num_single_del_fallthru = 0;
uint64_t num_single_del_mismatch = 0;
};

588
db/compaction_iterator.cc Normal file
View File

@@ -0,0 +1,588 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "db/compaction_iterator.h"
#include "rocksdb/listener.h"
#include "table/internal_iterator.h"
namespace rocksdb {
#ifndef ROCKSDB_LITE
CompactionEventListener::CompactionListenerValueType fromInternalValueType(
ValueType vt) {
switch (vt) {
case kTypeDeletion:
return CompactionEventListener::CompactionListenerValueType::kDelete;
case kTypeValue:
return CompactionEventListener::CompactionListenerValueType::kValue;
case kTypeMerge:
return CompactionEventListener::CompactionListenerValueType::
kMergeOperand;
case kTypeSingleDeletion:
return CompactionEventListener::CompactionListenerValueType::
kSingleDelete;
case kTypeRangeDeletion:
return CompactionEventListener::CompactionListenerValueType::kRangeDelete;
case kTypeBlobIndex:
return CompactionEventListener::CompactionListenerValueType::kBlobIndex;
default:
assert(false);
return CompactionEventListener::CompactionListenerValueType::kInvalid;
}
}
#endif // ROCKSDB_LITE
CompactionIterator::CompactionIterator(
InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
SequenceNumber earliest_write_conflict_snapshot, Env* env,
bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
const Compaction* compaction, const CompactionFilter* compaction_filter,
CompactionEventListener* compaction_listener,
const std::atomic<bool>* shutting_down)
: CompactionIterator(
input, cmp, merge_helper, last_sequence, snapshots,
earliest_write_conflict_snapshot, env, expect_valid_internal_key,
range_del_agg,
std::unique_ptr<CompactionProxy>(
compaction ? new CompactionProxy(compaction) : nullptr),
compaction_filter, compaction_listener, shutting_down) {}
CompactionIterator::CompactionIterator(
InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
SequenceNumber earliest_write_conflict_snapshot, Env* env,
bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
std::unique_ptr<CompactionProxy> compaction,
const CompactionFilter* compaction_filter,
CompactionEventListener* compaction_listener,
const std::atomic<bool>* shutting_down)
: input_(input),
cmp_(cmp),
merge_helper_(merge_helper),
snapshots_(snapshots),
earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
env_(env),
expect_valid_internal_key_(expect_valid_internal_key),
range_del_agg_(range_del_agg),
compaction_(std::move(compaction)),
compaction_filter_(compaction_filter),
#ifndef ROCKSDB_LITE
compaction_listener_(compaction_listener),
#endif // ROCKSDB_LITE
shutting_down_(shutting_down),
ignore_snapshots_(false),
merge_out_iter_(merge_helper_) {
assert(compaction_filter_ == nullptr || compaction_ != nullptr);
bottommost_level_ =
compaction_ == nullptr ? false : compaction_->bottommost_level();
if (compaction_ != nullptr) {
level_ptrs_ = std::vector<size_t>(compaction_->number_levels(), 0);
}
if (snapshots_->size() == 0) {
// optimize for fast path if there are no snapshots
visible_at_tip_ = true;
earliest_snapshot_ = kMaxSequenceNumber;
latest_snapshot_ = 0;
} else {
visible_at_tip_ = false;
earliest_snapshot_ = snapshots_->at(0);
latest_snapshot_ = snapshots_->back();
}
if (compaction_filter_ != nullptr) {
if (compaction_filter_->IgnoreSnapshots()) {
ignore_snapshots_ = true;
}
} else {
ignore_snapshots_ = false;
}
input_->SetPinnedItersMgr(&pinned_iters_mgr_);
}
CompactionIterator::~CompactionIterator() {
// input_ Iteartor lifetime is longer than pinned_iters_mgr_ lifetime
input_->SetPinnedItersMgr(nullptr);
}
void CompactionIterator::ResetRecordCounts() {
iter_stats_.num_record_drop_user = 0;
iter_stats_.num_record_drop_hidden = 0;
iter_stats_.num_record_drop_obsolete = 0;
iter_stats_.num_record_drop_range_del = 0;
iter_stats_.num_range_del_drop_obsolete = 0;
iter_stats_.num_optimized_del_drop_obsolete = 0;
}
void CompactionIterator::SeekToFirst() {
NextFromInput();
PrepareOutput();
}
void CompactionIterator::Next() {
// If there is a merge output, return it before continuing to process the
// input.
if (merge_out_iter_.Valid()) {
merge_out_iter_.Next();
// Check if we returned all records of the merge output.
if (merge_out_iter_.Valid()) {
key_ = merge_out_iter_.key();
value_ = merge_out_iter_.value();
bool valid_key __attribute__((__unused__)) =
ParseInternalKey(key_, &ikey_);
// MergeUntil stops when it encounters a corrupt key and does not
// include them in the result, so we expect the keys here to be valid.
assert(valid_key);
// Keep current_key_ in sync.
current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
key_ = current_key_.GetInternalKey();
ikey_.user_key = current_key_.GetUserKey();
valid_ = true;
} else {
// We consumed all pinned merge operands, release pinned iterators
pinned_iters_mgr_.ReleasePinnedData();
// MergeHelper moves the iterator to the first record after the merged
// records, so even though we reached the end of the merge output, we do
// not want to advance the iterator.
NextFromInput();
}
} else {
// Only advance the input iterator if there is no merge output and the
// iterator is not already at the next record.
if (!at_next_) {
input_->Next();
}
NextFromInput();
}
if (valid_) {
// Record that we've outputted a record for the current key.
has_outputted_key_ = true;
}
PrepareOutput();
}
void CompactionIterator::NextFromInput() {
at_next_ = false;
valid_ = false;
while (!valid_ && input_->Valid() && !IsShuttingDown()) {
key_ = input_->key();
value_ = input_->value();
iter_stats_.num_input_records++;
if (!ParseInternalKey(key_, &ikey_)) {
// If `expect_valid_internal_key_` is false, return the corrupted key
// and let the caller decide what to do with it.
// TODO(noetzli): We should have a more elegant solution for this.
if (expect_valid_internal_key_) {
assert(!"Corrupted internal key not expected.");
status_ = Status::Corruption("Corrupted internal key not expected.");
break;
}
key_ = current_key_.SetInternalKey(key_);
has_current_user_key_ = false;
current_user_key_sequence_ = kMaxSequenceNumber;
current_user_key_snapshot_ = 0;
iter_stats_.num_input_corrupt_records++;
valid_ = true;
break;
}
// Update input statistics
if (ikey_.type == kTypeDeletion || ikey_.type == kTypeSingleDeletion) {
iter_stats_.num_input_deletion_records++;
}
iter_stats_.total_input_raw_key_bytes += key_.size();
iter_stats_.total_input_raw_value_bytes += value_.size();
// If need_skip is true, we should seek the input iterator
// to internal key skip_until and continue from there.
bool need_skip = false;
// Points either into compaction_filter_skip_until_ or into
// merge_helper_->compaction_filter_skip_until_.
Slice skip_until;
// Check whether the user key changed. After this if statement current_key_
// is a copy of the current input key (maybe converted to a delete by the
// compaction filter). ikey_.user_key is pointing to the copy.
if (!has_current_user_key_ ||
!cmp_->Equal(ikey_.user_key, current_user_key_)) {
// First occurrence of this user key
// Copy key for output
key_ = current_key_.SetInternalKey(key_, &ikey_);
current_user_key_ = ikey_.user_key;
has_current_user_key_ = true;
has_outputted_key_ = false;
current_user_key_sequence_ = kMaxSequenceNumber;
current_user_key_snapshot_ = 0;
#ifndef ROCKSDB_LITE
if (compaction_listener_) {
compaction_listener_->OnCompaction(compaction_->level(), ikey_.user_key,
fromInternalValueType(ikey_.type),
value_, ikey_.sequence, true);
}
#endif // ROCKSDB_LITE
// apply the compaction filter to the first occurrence of the user key
if (compaction_filter_ != nullptr &&
(ikey_.type == kTypeValue || ikey_.type == kTypeBlobIndex) &&
(visible_at_tip_ || ikey_.sequence > latest_snapshot_ ||
ignore_snapshots_)) {
// If the user has specified a compaction filter and the sequence
// number is greater than any external snapshot, then invoke the
// filter. If the return value of the compaction filter is true,
// replace the entry with a deletion marker.
CompactionFilter::Decision filter;
compaction_filter_value_.clear();
compaction_filter_skip_until_.Clear();
CompactionFilter::ValueType value_type =
ikey_.type == kTypeValue ? CompactionFilter::ValueType::kValue
: CompactionFilter::ValueType::kBlobIndex;
{
StopWatchNano timer(env_, true);
filter = compaction_filter_->FilterV2(
compaction_->level(), ikey_.user_key, value_type, value_,
&compaction_filter_value_, compaction_filter_skip_until_.rep());
iter_stats_.total_filter_time +=
env_ != nullptr ? timer.ElapsedNanos() : 0;
}
if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil &&
cmp_->Compare(*compaction_filter_skip_until_.rep(),
ikey_.user_key) <= 0) {
// Can't skip to a key smaller than the current one.
// Keep the key as per FilterV2 documentation.
filter = CompactionFilter::Decision::kKeep;
}
if (filter == CompactionFilter::Decision::kRemove) {
// convert the current key to a delete; key_ is pointing into
// current_key_ at this point, so updating current_key_ updates key()
ikey_.type = kTypeDeletion;
current_key_.UpdateInternalKey(ikey_.sequence, kTypeDeletion);
// no value associated with delete
value_.clear();
iter_stats_.num_record_drop_user++;
} else if (filter == CompactionFilter::Decision::kChangeValue) {
value_ = compaction_filter_value_;
} else if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil) {
need_skip = true;
compaction_filter_skip_until_.ConvertFromUserKey(kMaxSequenceNumber,
kValueTypeForSeek);
skip_until = compaction_filter_skip_until_.Encode();
}
}
} else {
#ifndef ROCKSDB_LITE
if (compaction_listener_) {
compaction_listener_->OnCompaction(compaction_->level(), ikey_.user_key,
fromInternalValueType(ikey_.type),
value_, ikey_.sequence, false);
}
#endif // ROCKSDB_LITE
// Update the current key to reflect the new sequence number/type without
// copying the user key.
// TODO(rven): Compaction filter does not process keys in this path
// Need to have the compaction filter process multiple versions
// if we have versions on both sides of a snapshot
current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
key_ = current_key_.GetInternalKey();
ikey_.user_key = current_key_.GetUserKey();
}
// If there are no snapshots, then this kv affect visibility at tip.
// Otherwise, search though all existing snapshots to find the earliest
// snapshot that is affected by this kv.
SequenceNumber last_sequence __attribute__((__unused__)) =
current_user_key_sequence_;
current_user_key_sequence_ = ikey_.sequence;
SequenceNumber last_snapshot = current_user_key_snapshot_;
SequenceNumber prev_snapshot = 0; // 0 means no previous snapshot
current_user_key_snapshot_ =
visible_at_tip_
? earliest_snapshot_
: findEarliestVisibleSnapshot(ikey_.sequence, &prev_snapshot);
if (need_skip) {
// This case is handled below.
} else if (clear_and_output_next_key_) {
// In the previous iteration we encountered a single delete that we could
// not compact out. We will keep this Put, but can drop it's data.
// (See Optimization 3, below.)
assert(ikey_.type == kTypeValue);
assert(current_user_key_snapshot_ == last_snapshot);
value_.clear();
valid_ = true;
clear_and_output_next_key_ = false;
} else if (ikey_.type == kTypeSingleDeletion) {
// We can compact out a SingleDelete if:
// 1) We encounter the corresponding PUT -OR- we know that this key
// doesn't appear past this output level
// =AND=
// 2) We've already returned a record in this snapshot -OR-
// there are no earlier earliest_write_conflict_snapshot.
//
// Rule 1 is needed for SingleDelete correctness. Rule 2 is needed to
// allow Transactions to do write-conflict checking (if we compacted away
// all keys, then we wouldn't know that a write happened in this
// snapshot). If there is no earlier snapshot, then we know that there
// are no active transactions that need to know about any writes.
//
// Optimization 3:
// If we encounter a SingleDelete followed by a PUT and Rule 2 is NOT
// true, then we must output a SingleDelete. In this case, we will decide
// to also output the PUT. While we are compacting less by outputting the
// PUT now, hopefully this will lead to better compaction in the future
// when Rule 2 is later true (Ie, We are hoping we can later compact out
// both the SingleDelete and the Put, while we couldn't if we only
// outputted the SingleDelete now).
// In this case, we can save space by removing the PUT's value as it will
// never be read.
//
// Deletes and Merges are not supported on the same key that has a
// SingleDelete as it is not possible to correctly do any partial
// compaction of such a combination of operations. The result of mixing
// those operations for a given key is documented as being undefined. So
// we can choose how to handle such a combinations of operations. We will
// try to compact out as much as we can in these cases.
// We will report counts on these anomalous cases.
// The easiest way to process a SingleDelete during iteration is to peek
// ahead at the next key.
ParsedInternalKey next_ikey;
input_->Next();
// Check whether the next key exists, is not corrupt, and is the same key
// as the single delete.
if (input_->Valid() && ParseInternalKey(input_->key(), &next_ikey) &&
cmp_->Equal(ikey_.user_key, next_ikey.user_key)) {
// Check whether the next key belongs to the same snapshot as the
// SingleDelete.
if (prev_snapshot == 0 || next_ikey.sequence > prev_snapshot) {
if (next_ikey.type == kTypeSingleDeletion) {
// We encountered two SingleDeletes in a row. This could be due to
// unexpected user input.
// Skip the first SingleDelete and let the next iteration decide how
// to handle the second SingleDelete
// First SingleDelete has been skipped since we already called
// input_->Next().
++iter_stats_.num_record_drop_obsolete;
++iter_stats_.num_single_del_mismatch;
} else if ((ikey_.sequence <= earliest_write_conflict_snapshot_) ||
has_outputted_key_) {
// Found a matching value, we can drop the single delete and the
// value. It is safe to drop both records since we've already
// outputted a key in this snapshot, or there is no earlier
// snapshot (Rule 2 above).
// Note: it doesn't matter whether the second key is a Put or if it
// is an unexpected Merge or Delete. We will compact it out
// either way. We will maintain counts of how many mismatches
// happened
if (next_ikey.type != kTypeValue) {
++iter_stats_.num_single_del_mismatch;
}
++iter_stats_.num_record_drop_hidden;
++iter_stats_.num_record_drop_obsolete;
// Already called input_->Next() once. Call it a second time to
// skip past the second key.
input_->Next();
} else {
// Found a matching value, but we cannot drop both keys since
// there is an earlier snapshot and we need to leave behind a record
// to know that a write happened in this snapshot (Rule 2 above).
// Clear the value and output the SingleDelete. (The value will be
// outputted on the next iteration.)
// Setting valid_ to true will output the current SingleDelete
valid_ = true;
// Set up the Put to be outputted in the next iteration.
// (Optimization 3).
clear_and_output_next_key_ = true;
}
} else {
// We hit the next snapshot without hitting a put, so the iterator
// returns the single delete.
valid_ = true;
}
} else {
// We are at the end of the input, could not parse the next key, or hit
// a different key. The iterator returns the single delete if the key
// possibly exists beyond the current output level. We set
// has_current_user_key to false so that if the iterator is at the next
// key, we do not compare it again against the previous key at the next
// iteration. If the next key is corrupt, we return before the
// comparison, so the value of has_current_user_key does not matter.
has_current_user_key_ = false;
if (compaction_ != nullptr && ikey_.sequence <= earliest_snapshot_ &&
compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
&level_ptrs_)) {
// Key doesn't exist outside of this range.
// Can compact out this SingleDelete.
++iter_stats_.num_record_drop_obsolete;
++iter_stats_.num_single_del_fallthru;
if (!bottommost_level_) {
++iter_stats_.num_optimized_del_drop_obsolete;
}
} else {
// Output SingleDelete
valid_ = true;
}
}
if (valid_) {
at_next_ = true;
}
} else if (last_snapshot == current_user_key_snapshot_) {
// If the earliest snapshot is which this key is visible in
// is the same as the visibility of a previous instance of the
// same key, then this kv is not visible in any snapshot.
// Hidden by an newer entry for same user key
// TODO(noetzli): why not > ?
//
// Note: Dropping this key will not affect TransactionDB write-conflict
// checking since there has already been a record returned for this key
// in this snapshot.
assert(last_sequence >= current_user_key_sequence_);
++iter_stats_.num_record_drop_hidden; // (A)
input_->Next();
} else if (compaction_ != nullptr && ikey_.type == kTypeDeletion &&
ikey_.sequence <= earliest_snapshot_ &&
compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
&level_ptrs_)) {
// TODO(noetzli): This is the only place where we use compaction_
// (besides the constructor). We should probably get rid of this
// dependency and find a way to do similar filtering during flushes.
//
// For this user key:
// (1) there is no data in higher levels
// (2) data in lower levels will have larger sequence numbers
// (3) data in layers that are being compacted here and have
// smaller sequence numbers will be dropped in the next
// few iterations of this loop (by rule (A) above).
// Therefore this deletion marker is obsolete and can be dropped.
//
// Note: Dropping this Delete will not affect TransactionDB
// write-conflict checking since it is earlier than any snapshot.
++iter_stats_.num_record_drop_obsolete;
if (!bottommost_level_) {
++iter_stats_.num_optimized_del_drop_obsolete;
}
input_->Next();
} else if (ikey_.type == kTypeMerge) {
if (!merge_helper_->HasOperator()) {
status_ = Status::InvalidArgument(
"merge_operator is not properly initialized.");
return;
}
pinned_iters_mgr_.StartPinning();
// We know the merge type entry is not hidden, otherwise we would
// have hit (A)
// We encapsulate the merge related state machine in a different
// object to minimize change to the existing flow.
Status s = merge_helper_->MergeUntil(input_, range_del_agg_,
prev_snapshot, bottommost_level_);
merge_out_iter_.SeekToFirst();
if (!s.ok() && !s.IsMergeInProgress()) {
status_ = s;
return;
} else if (merge_out_iter_.Valid()) {
// NOTE: key, value, and ikey_ refer to old entries.
// These will be correctly set below.
key_ = merge_out_iter_.key();
value_ = merge_out_iter_.value();
bool valid_key __attribute__((__unused__)) =
ParseInternalKey(key_, &ikey_);
// MergeUntil stops when it encounters a corrupt key and does not
// include them in the result, so we expect the keys here to valid.
assert(valid_key);
// Keep current_key_ in sync.
current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
key_ = current_key_.GetInternalKey();
ikey_.user_key = current_key_.GetUserKey();
valid_ = true;
} else {
// all merge operands were filtered out. reset the user key, since the
// batch consumed by the merge operator should not shadow any keys
// coming after the merges
has_current_user_key_ = false;
pinned_iters_mgr_.ReleasePinnedData();
if (merge_helper_->FilteredUntil(&skip_until)) {
need_skip = true;
}
}
} else {
// 1. new user key -OR-
// 2. different snapshot stripe
bool should_delete = range_del_agg_->ShouldDelete(
key_, RangeDelAggregator::RangePositioningMode::kForwardTraversal);
if (should_delete) {
++iter_stats_.num_record_drop_hidden;
++iter_stats_.num_record_drop_range_del;
input_->Next();
} else {
valid_ = true;
}
}
if (need_skip) {
input_->Seek(skip_until);
}
}
if (!valid_ && IsShuttingDown()) {
status_ = Status::ShutdownInProgress();
}
}
void CompactionIterator::PrepareOutput() {
// Zeroing out the sequence number leads to better compression.
// If this is the bottommost level (no files in lower levels)
// and the earliest snapshot is larger than this seqno
// and the userkey differs from the last userkey in compaction
// then we can squash the seqno to zero.
// This is safe for TransactionDB write-conflict checking since transactions
// only care about sequence number larger than any active snapshots.
if ((compaction_ != nullptr && !compaction_->allow_ingest_behind()) &&
bottommost_level_ && valid_ && ikey_.sequence <= earliest_snapshot_ &&
ikey_.type != kTypeMerge &&
!cmp_->Equal(compaction_->GetLargestUserKey(), ikey_.user_key)) {
assert(ikey_.type != kTypeDeletion && ikey_.type != kTypeSingleDeletion);
ikey_.sequence = 0;
current_key_.UpdateInternalKey(0, ikey_.type);
}
}
inline SequenceNumber CompactionIterator::findEarliestVisibleSnapshot(
SequenceNumber in, SequenceNumber* prev_snapshot) {
assert(snapshots_->size());
SequenceNumber prev __attribute__((__unused__)) = kMaxSequenceNumber;
for (const auto cur : *snapshots_) {
assert(prev == kMaxSequenceNumber || prev <= cur);
if (cur >= in) {
*prev_snapshot = prev == kMaxSequenceNumber ? 0 : prev;
return cur;
}
prev = cur;
assert(prev < kMaxSequenceNumber);
}
*prev_snapshot = prev;
return kMaxSequenceNumber;
}
} // namespace rocksdb

197
db/compaction_iterator.h Normal file
View File

@@ -0,0 +1,197 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#include <algorithm>
#include <deque>
#include <string>
#include <vector>
#include "db/compaction.h"
#include "db/compaction_iteration_stats.h"
#include "db/merge_helper.h"
#include "db/pinned_iterators_manager.h"
#include "db/range_del_aggregator.h"
#include "options/cf_options.h"
#include "rocksdb/compaction_filter.h"
namespace rocksdb {
class CompactionEventListener;
class CompactionIterator {
public:
// A wrapper around Compaction. Has a much smaller interface, only what
// CompactionIterator uses. Tests can override it.
class CompactionProxy {
public:
explicit CompactionProxy(const Compaction* compaction)
: compaction_(compaction) {}
virtual ~CompactionProxy() = default;
virtual int level(size_t compaction_input_level = 0) const {
return compaction_->level();
}
virtual bool KeyNotExistsBeyondOutputLevel(
const Slice& user_key, std::vector<size_t>* level_ptrs) const {
return compaction_->KeyNotExistsBeyondOutputLevel(user_key, level_ptrs);
}
virtual bool bottommost_level() const {
return compaction_->bottommost_level();
}
virtual int number_levels() const { return compaction_->number_levels(); }
virtual Slice GetLargestUserKey() const {
return compaction_->GetLargestUserKey();
}
virtual bool allow_ingest_behind() const {
return compaction_->immutable_cf_options()->allow_ingest_behind;
}
protected:
CompactionProxy() = default;
private:
const Compaction* compaction_;
};
CompactionIterator(InternalIterator* input, const Comparator* cmp,
MergeHelper* merge_helper, SequenceNumber last_sequence,
std::vector<SequenceNumber>* snapshots,
SequenceNumber earliest_write_conflict_snapshot, Env* env,
bool expect_valid_internal_key,
RangeDelAggregator* range_del_agg,
const Compaction* compaction = nullptr,
const CompactionFilter* compaction_filter = nullptr,
CompactionEventListener* compaction_listener = nullptr,
const std::atomic<bool>* shutting_down = nullptr);
// Constructor with custom CompactionProxy, used for tests.
CompactionIterator(InternalIterator* input, const Comparator* cmp,
MergeHelper* merge_helper, SequenceNumber last_sequence,
std::vector<SequenceNumber>* snapshots,
SequenceNumber earliest_write_conflict_snapshot, Env* env,
bool expect_valid_internal_key,
RangeDelAggregator* range_del_agg,
std::unique_ptr<CompactionProxy> compaction,
const CompactionFilter* compaction_filter = nullptr,
CompactionEventListener* compaction_listener = nullptr,
const std::atomic<bool>* shutting_down = nullptr);
~CompactionIterator();
void ResetRecordCounts();
// Seek to the beginning of the compaction iterator output.
//
// REQUIRED: Call only once.
void SeekToFirst();
// Produces the next record in the compaction.
//
// REQUIRED: SeekToFirst() has been called.
void Next();
// Getters
const Slice& key() const { return key_; }
const Slice& value() const { return value_; }
const Status& status() const { return status_; }
const ParsedInternalKey& ikey() const { return ikey_; }
bool Valid() const { return valid_; }
const Slice& user_key() const { return current_user_key_; }
const CompactionIterationStats& iter_stats() const { return iter_stats_; }
private:
// Processes the input stream to find the next output
void NextFromInput();
// Do last preparations before presenting the output to the callee. At this
// point this only zeroes out the sequence number if possible for better
// compression.
void PrepareOutput();
// Given a sequence number, return the sequence number of the
// earliest snapshot that this sequence number is visible in.
// The snapshots themselves are arranged in ascending order of
// sequence numbers.
// Employ a sequential search because the total number of
// snapshots are typically small.
inline SequenceNumber findEarliestVisibleSnapshot(
SequenceNumber in, SequenceNumber* prev_snapshot);
InternalIterator* input_;
const Comparator* cmp_;
MergeHelper* merge_helper_;
const std::vector<SequenceNumber>* snapshots_;
const SequenceNumber earliest_write_conflict_snapshot_;
Env* env_;
bool expect_valid_internal_key_;
RangeDelAggregator* range_del_agg_;
std::unique_ptr<CompactionProxy> compaction_;
const CompactionFilter* compaction_filter_;
#ifndef ROCKSDB_LITE
CompactionEventListener* compaction_listener_;
#endif // ROCKSDB_LITE
const std::atomic<bool>* shutting_down_;
bool bottommost_level_;
bool valid_ = false;
bool visible_at_tip_;
SequenceNumber earliest_snapshot_;
SequenceNumber latest_snapshot_;
bool ignore_snapshots_;
// State
//
// Points to a copy of the current compaction iterator output (current_key_)
// if valid_.
Slice key_;
// Points to the value in the underlying iterator that corresponds to the
// current output.
Slice value_;
// The status is OK unless compaction iterator encounters a merge operand
// while not having a merge operator defined.
Status status_;
// Stores the user key, sequence number and type of the current compaction
// iterator output (or current key in the underlying iterator during
// NextFromInput()).
ParsedInternalKey ikey_;
// Stores whether ikey_.user_key is valid. If set to false, the user key is
// not compared against the current key in the underlying iterator.
bool has_current_user_key_ = false;
bool at_next_ = false; // If false, the iterator
// Holds a copy of the current compaction iterator output (or current key in
// the underlying iterator during NextFromInput()).
IterKey current_key_;
Slice current_user_key_;
SequenceNumber current_user_key_sequence_;
SequenceNumber current_user_key_snapshot_;
// True if the iterator has already returned a record for the current key.
bool has_outputted_key_ = false;
// truncated the value of the next key and output it without applying any
// compaction rules. This is used for outputting a put after a single delete.
bool clear_and_output_next_key_ = false;
MergeOutputIterator merge_out_iter_;
// PinnedIteratorsManager used to pin input_ Iterator blocks while reading
// merge operands and then releasing them after consuming them.
PinnedIteratorsManager pinned_iters_mgr_;
std::string compaction_filter_value_;
InternalKey compaction_filter_skip_until_;
// "level_ptrs" holds indices that remember which file of an associated
// level we were last checking during the last call to compaction->
// KeyNotExistsBeyondOutputLevel(). This allows future calls to the function
// to pick off where it left off since each subcompaction's key range is
// increasing so a later call to the function must be looking for a key that
// is in or beyond the last file checked during the previous call
std::vector<size_t> level_ptrs_;
CompactionIterationStats iter_stats_;
bool IsShuttingDown() {
// This is a best-effort facility, so memory_order_relaxed is sufficient.
return shutting_down_ && shutting_down_->load(std::memory_order_relaxed);
}
};
} // namespace rocksdb

568
db/compaction_iterator_test.cc Normal file
View File

@@ -0,0 +1,568 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "db/compaction_iterator.h"
#include <string>
#include <vector>
#include "port/port.h"
#include "util/testharness.h"
#include "util/testutil.h"
namespace rocksdb {
// Expects no merging attempts.
class NoMergingMergeOp : public MergeOperator {
public:
bool FullMergeV2(const MergeOperationInput& merge_in,
MergeOperationOutput* merge_out) const override {
ADD_FAILURE();
return false;
}
bool PartialMergeMulti(const Slice& key,
const std::deque<Slice>& operand_list,
std::string* new_value,
Logger* logger) const override {
ADD_FAILURE();
return false;
}
const char* Name() const override {
return "CompactionIteratorTest NoMergingMergeOp";
}
};
// Compaction filter that gets stuck when it sees a particular key,
// then gets unstuck when told to.
// Always returns Decition::kRemove.
class StallingFilter : public CompactionFilter {
public:
virtual Decision FilterV2(int level, const Slice& key, ValueType t,
const Slice& existing_value, std::string* new_value,
std::string* skip_until) const override {
int k = std::atoi(key.ToString().c_str());
last_seen.store(k);
while (k >= stall_at.load()) {
std::this_thread::yield();
}
return Decision::kRemove;
}
const char* Name() const override {
return "CompactionIteratorTest StallingFilter";
}
// Wait until the filter sees a key >= k and stalls at that key.
// If `exact`, asserts that the seen key is equal to k.
void WaitForStall(int k, bool exact = true) {
stall_at.store(k);
while (last_seen.load() < k) {
std::this_thread::yield();
}
if (exact) {
EXPECT_EQ(k, last_seen.load());
}
}
// Filter will stall on key >= stall_at. Advance stall_at to unstall.
mutable std::atomic<int> stall_at{0};
// Last key the filter was called with.
mutable std::atomic<int> last_seen{0};
};
class LoggingForwardVectorIterator : public InternalIterator {
public:
struct Action {
enum class Type {
SEEK_TO_FIRST,
SEEK,
NEXT,
};
Type type;
std::string arg;
explicit Action(Type _type, std::string _arg = "")
: type(_type), arg(_arg) {}
bool operator==(const Action& rhs) const {
return std::tie(type, arg) == std::tie(rhs.type, rhs.arg);
}
};
LoggingForwardVectorIterator(const std::vector<std::string>& keys,
const std::vector<std::string>& values)
: keys_(keys), values_(values), current_(keys.size()) {
assert(keys_.size() == values_.size());
}
virtual bool Valid() const override { return current_ < keys_.size(); }
virtual void SeekToFirst() override {
log.emplace_back(Action::Type::SEEK_TO_FIRST);
current_ = 0;
}
virtual void SeekToLast() override { assert(false); }
virtual void Seek(const Slice& target) override {
log.emplace_back(Action::Type::SEEK, target.ToString());
current_ = std::lower_bound(keys_.begin(), keys_.end(), target.ToString()) -
keys_.begin();
}
virtual void SeekForPrev(const Slice& target) override { assert(false); }
virtual void Next() override {
assert(Valid());
log.emplace_back(Action::Type::NEXT);
current_++;
}
virtual void Prev() override { assert(false); }
virtual Slice key() const override {
assert(Valid());
return Slice(keys_[current_]);
}
virtual Slice value() const override {
assert(Valid());
return Slice(values_[current_]);
}
virtual Status status() const override { return Status::OK(); }
std::vector<Action> log;
private:
std::vector<std::string> keys_;
std::vector<std::string> values_;
size_t current_;
};
class FakeCompaction : public CompactionIterator::CompactionProxy {
public:
FakeCompaction() = default;
virtual int level(size_t compaction_input_level) const { return 0; }
virtual bool KeyNotExistsBeyondOutputLevel(
const Slice& user_key, std::vector<size_t>* level_ptrs) const {
return key_not_exists_beyond_output_level;
}
virtual bool bottommost_level() const { return false; }
virtual int number_levels() const { return 1; }
virtual Slice GetLargestUserKey() const {
return "\xff\xff\xff\xff\xff\xff\xff\xff\xff";
}
virtual bool allow_ingest_behind() const { return false; }
bool key_not_exists_beyond_output_level = false;
};
class CompactionIteratorTest : public testing::Test {
public:
CompactionIteratorTest()
: cmp_(BytewiseComparator()), icmp_(cmp_), snapshots_({}) {}
void InitIterators(const std::vector<std::string>& ks,
const std::vector<std::string>& vs,
const std::vector<std::string>& range_del_ks,
const std::vector<std::string>& range_del_vs,
SequenceNumber last_sequence,
MergeOperator* merge_op = nullptr,
CompactionFilter* filter = nullptr) {
std::unique_ptr<InternalIterator> range_del_iter(
new test::VectorIterator(range_del_ks, range_del_vs));
range_del_agg_.reset(new RangeDelAggregator(icmp_, snapshots_));
ASSERT_OK(range_del_agg_->AddTombstones(std::move(range_del_iter)));
std::unique_ptr<CompactionIterator::CompactionProxy> compaction;
if (filter) {
compaction_proxy_ = new FakeCompaction();
compaction.reset(compaction_proxy_);
}
merge_helper_.reset(new MergeHelper(Env::Default(), cmp_, merge_op, filter,
nullptr, false, 0, 0, nullptr,
&shutting_down_));
iter_.reset(new LoggingForwardVectorIterator(ks, vs));
iter_->SeekToFirst();
c_iter_.reset(new CompactionIterator(
iter_.get(), cmp_, merge_helper_.get(), last_sequence, &snapshots_,
kMaxSequenceNumber, Env::Default(), false, range_del_agg_.get(),
std::move(compaction), filter, nullptr, &shutting_down_));
}
void AddSnapshot(SequenceNumber snapshot) { snapshots_.push_back(snapshot); }
const Comparator* cmp_;
const InternalKeyComparator icmp_;
std::vector<SequenceNumber> snapshots_;
std::unique_ptr<MergeHelper> merge_helper_;
std::unique_ptr<LoggingForwardVectorIterator> iter_;
std::unique_ptr<CompactionIterator> c_iter_;
std::unique_ptr<RangeDelAggregator> range_del_agg_;
std::atomic<bool> shutting_down_{false};
FakeCompaction* compaction_proxy_;
};
// It is possible that the output of the compaction iterator is empty even if
// the input is not.
TEST_F(CompactionIteratorTest, EmptyResult) {
InitIterators({test::KeyStr("a", 5, kTypeSingleDeletion),
test::KeyStr("a", 3, kTypeValue)},
{"", "val"}, {}, {}, 5);
c_iter_->SeekToFirst();
ASSERT_FALSE(c_iter_->Valid());
}
// If there is a corruption after a single deletion, the corrupted key should
// be preserved.
TEST_F(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
InitIterators({test::KeyStr("a", 5, kTypeSingleDeletion),
test::KeyStr("a", 3, kTypeValue, true),
test::KeyStr("b", 10, kTypeValue)},
{"", "val", "val2"}, {}, {}, 10);
c_iter_->SeekToFirst();
ASSERT_TRUE(c_iter_->Valid());
ASSERT_EQ(test::KeyStr("a", 5, kTypeSingleDeletion),
c_iter_->key().ToString());
c_iter_->Next();
ASSERT_TRUE(c_iter_->Valid());
ASSERT_EQ(test::KeyStr("a", 3, kTypeValue, true), c_iter_->key().ToString());
c_iter_->Next();
ASSERT_TRUE(c_iter_->Valid());
ASSERT_EQ(test::KeyStr("b", 10, kTypeValue), c_iter_->key().ToString());
c_iter_->Next();
ASSERT_FALSE(c_iter_->Valid());
}
TEST_F(CompactionIteratorTest, SimpleRangeDeletion) {
InitIterators({test::KeyStr("morning", 5, kTypeValue),
test::KeyStr("morning", 2, kTypeValue),
test::KeyStr("night", 3, kTypeValue)},
{"zao", "zao", "wan"},
{test::KeyStr("ma", 4, kTypeRangeDeletion)}, {"mz"}, 5);
c_iter_->SeekToFirst();
ASSERT_TRUE(c_iter_->Valid());
ASSERT_EQ(test::KeyStr("morning", 5, kTypeValue), c_iter_->key().ToString());
c_iter_->Next();
ASSERT_TRUE(c_iter_->Valid());
ASSERT_EQ(test::KeyStr("night", 3, kTypeValue), c_iter_->key().ToString());
c_iter_->Next();
ASSERT_FALSE(c_iter_->Valid());
}
TEST_F(CompactionIteratorTest, RangeDeletionWithSnapshots) {
AddSnapshot(10);
std::vector<std::string> ks1;
ks1.push_back(test::KeyStr("ma", 28, kTypeRangeDeletion));
std::vector<std::string> vs1{"mz"};
std::vector<std::string> ks2{test::KeyStr("morning", 15, kTypeValue),
test::KeyStr("morning", 5, kTypeValue),
test::KeyStr("night", 40, kTypeValue),
test::KeyStr("night", 20, kTypeValue)};
std::vector<std::string> vs2{"zao 15", "zao 5", "wan 40", "wan 20"};
InitIterators(ks2, vs2, ks1, vs1, 40);
c_iter_->SeekToFirst();
ASSERT_TRUE(c_iter_->Valid());
ASSERT_EQ(test::KeyStr("morning", 5, kTypeValue), c_iter_->key().ToString());
c_iter_->Next();
ASSERT_TRUE(c_iter_->Valid());
ASSERT_EQ(test::KeyStr("night", 40, kTypeValue), c_iter_->key().ToString());
c_iter_->Next();
ASSERT_FALSE(c_iter_->Valid());
}
TEST_F(CompactionIteratorTest, CompactionFilterSkipUntil) {
class Filter : public CompactionFilter {
virtual Decision FilterV2(int level, const Slice& key, ValueType t,
const Slice& existing_value,
std::string* new_value,
std::string* skip_until) const override {
std::string k = key.ToString();
std::string v = existing_value.ToString();
// See InitIterators() call below for the sequence of keys and their
// filtering decisions. Here we closely assert that compaction filter is
// called with the expected keys and only them, and with the right values.
if (k == "a") {
EXPECT_EQ(ValueType::kValue, t);
EXPECT_EQ("av50", v);
return Decision::kKeep;
}
if (k == "b") {
EXPECT_EQ(ValueType::kValue, t);
EXPECT_EQ("bv60", v);
*skip_until = "d+";
return Decision::kRemoveAndSkipUntil;
}
if (k == "e") {
EXPECT_EQ(ValueType::kMergeOperand, t);
EXPECT_EQ("em71", v);
return Decision::kKeep;
}
if (k == "f") {
if (v == "fm65") {
EXPECT_EQ(ValueType::kMergeOperand, t);
*skip_until = "f";
} else {
EXPECT_EQ("fm30", v);
EXPECT_EQ(ValueType::kMergeOperand, t);
*skip_until = "g+";
}
return Decision::kRemoveAndSkipUntil;
}
if (k == "h") {
EXPECT_EQ(ValueType::kValue, t);
EXPECT_EQ("hv91", v);
return Decision::kKeep;
}
if (k == "i") {
EXPECT_EQ(ValueType::kMergeOperand, t);
EXPECT_EQ("im95", v);
*skip_until = "z";
return Decision::kRemoveAndSkipUntil;
}
ADD_FAILURE();
return Decision::kKeep;
}
const char* Name() const override {
return "CompactionIteratorTest.CompactionFilterSkipUntil::Filter";
}
};
NoMergingMergeOp merge_op;
Filter filter;
InitIterators(
{test::KeyStr("a", 50, kTypeValue), // keep
test::KeyStr("a", 45, kTypeMerge),
test::KeyStr("b", 60, kTypeValue), // skip to "d+"
test::KeyStr("b", 40, kTypeValue), test::KeyStr("c", 35, kTypeValue),
test::KeyStr("d", 70, kTypeMerge),
test::KeyStr("e", 71, kTypeMerge), // keep
test::KeyStr("f", 65, kTypeMerge), // skip to "f", aka keep
test::KeyStr("f", 30, kTypeMerge), // skip to "g+"
test::KeyStr("f", 25, kTypeValue), test::KeyStr("g", 90, kTypeValue),
test::KeyStr("h", 91, kTypeValue), // keep
test::KeyStr("i", 95, kTypeMerge), // skip to "z"
test::KeyStr("j", 99, kTypeValue)},
{"av50", "am45", "bv60", "bv40", "cv35", "dm70", "em71", "fm65", "fm30",
"fv25", "gv90", "hv91", "im95", "jv99"},
{}, {}, kMaxSequenceNumber, &merge_op, &filter);
// Compaction should output just "a", "e" and "h" keys.
c_iter_->SeekToFirst();
ASSERT_TRUE(c_iter_->Valid());
ASSERT_EQ(test::KeyStr("a", 50, kTypeValue), c_iter_->key().ToString());
ASSERT_EQ("av50", c_iter_->value().ToString());
c_iter_->Next();
ASSERT_TRUE(c_iter_->Valid());
ASSERT_EQ(test::KeyStr("e", 71, kTypeMerge), c_iter_->key().ToString());
ASSERT_EQ("em71", c_iter_->value().ToString());
c_iter_->Next();
ASSERT_TRUE(c_iter_->Valid());
ASSERT_EQ(test::KeyStr("h", 91, kTypeValue), c_iter_->key().ToString());
ASSERT_EQ("hv91", c_iter_->value().ToString());
c_iter_->Next();
ASSERT_FALSE(c_iter_->Valid());
// Check that the compaction iterator did the correct sequence of calls on
// the underlying iterator.
using A = LoggingForwardVectorIterator::Action;
using T = A::Type;
std::vector<A> expected_actions = {
A(T::SEEK_TO_FIRST),
A(T::NEXT),
A(T::NEXT),
A(T::SEEK, test::KeyStr("d+", kMaxSequenceNumber, kValueTypeForSeek)),
A(T::NEXT),
A(T::NEXT),
A(T::SEEK, test::KeyStr("g+", kMaxSequenceNumber, kValueTypeForSeek)),
A(T::NEXT),
A(T::SEEK, test::KeyStr("z", kMaxSequenceNumber, kValueTypeForSeek))};
ASSERT_EQ(expected_actions, iter_->log);
}
TEST_F(CompactionIteratorTest, ShuttingDownInFilter) {
NoMergingMergeOp merge_op;
StallingFilter filter;
InitIterators(
{test::KeyStr("1", 1, kTypeValue), test::KeyStr("2", 2, kTypeValue),
test::KeyStr("3", 3, kTypeValue), test::KeyStr("4", 4, kTypeValue)},
{"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, &merge_op, &filter);
// Don't leave tombstones (kTypeDeletion) for filtered keys.
compaction_proxy_->key_not_exists_beyond_output_level = true;
std::atomic<bool> seek_done{false};
rocksdb::port::Thread compaction_thread([&] {
c_iter_->SeekToFirst();
EXPECT_FALSE(c_iter_->Valid());
EXPECT_TRUE(c_iter_->status().IsShutdownInProgress());
seek_done.store(true);
});
// Let key 1 through.
filter.WaitForStall(1);
// Shutdown during compaction filter call for key 2.
filter.WaitForStall(2);
shutting_down_.store(true);
EXPECT_FALSE(seek_done.load());
// Unstall filter and wait for SeekToFirst() to return.
filter.stall_at.store(3);
compaction_thread.join();
assert(seek_done.load());
// Check that filter was never called again.
EXPECT_EQ(2, filter.last_seen.load());
}
// Same as ShuttingDownInFilter, but shutdown happens during filter call for
// a merge operand, not for a value.
TEST_F(CompactionIteratorTest, ShuttingDownInMerge) {
NoMergingMergeOp merge_op;
StallingFilter filter;
InitIterators(
{test::KeyStr("1", 1, kTypeValue), test::KeyStr("2", 2, kTypeMerge),
test::KeyStr("3", 3, kTypeMerge), test::KeyStr("4", 4, kTypeValue)},
{"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, &merge_op, &filter);
compaction_proxy_->key_not_exists_beyond_output_level = true;
std::atomic<bool> seek_done{false};
rocksdb::port::Thread compaction_thread([&] {
c_iter_->SeekToFirst();
ASSERT_FALSE(c_iter_->Valid());
ASSERT_TRUE(c_iter_->status().IsShutdownInProgress());
seek_done.store(true);
});
// Let key 1 through.
filter.WaitForStall(1);
// Shutdown during compaction filter call for key 2.
filter.WaitForStall(2);
shutting_down_.store(true);
EXPECT_FALSE(seek_done.load());
// Unstall filter and wait for SeekToFirst() to return.
filter.stall_at.store(3);
compaction_thread.join();
assert(seek_done.load());
// Check that filter was never called again.
EXPECT_EQ(2, filter.last_seen.load());
}
TEST_F(CompactionIteratorTest, SingleMergeOperand) {
class Filter : public CompactionFilter {
virtual Decision FilterV2(int level, const Slice& key, ValueType t,
const Slice& existing_value,
std::string* new_value,
std::string* skip_until) const override {
std::string k = key.ToString();
std::string v = existing_value.ToString();
// See InitIterators() call below for the sequence of keys and their
// filtering decisions. Here we closely assert that compaction filter is
// called with the expected keys and only them, and with the right values.
if (k == "a") {
EXPECT_EQ(ValueType::kMergeOperand, t);
EXPECT_EQ("av1", v);
return Decision::kKeep;
} else if (k == "b") {
EXPECT_EQ(ValueType::kMergeOperand, t);
return Decision::kKeep;
} else if (k == "c") {
return Decision::kKeep;
}
ADD_FAILURE();
return Decision::kKeep;
}
const char* Name() const override {
return "CompactionIteratorTest.SingleMergeOperand::Filter";
}
};
class SingleMergeOp : public MergeOperator {
public:
bool FullMergeV2(const MergeOperationInput& merge_in,
MergeOperationOutput* merge_out) const override {
// See InitIterators() call below for why "c" is the only key for which
// FullMergeV2 should be called.
EXPECT_EQ("c", merge_in.key.ToString());
std::string temp_value;
if (merge_in.existing_value != nullptr) {
temp_value = merge_in.existing_value->ToString();
}
for (auto& operand : merge_in.operand_list) {
temp_value.append(operand.ToString());
}
merge_out->new_value = temp_value;
return true;
}
bool PartialMergeMulti(const Slice& key,
const std::deque<Slice>& operand_list,
std::string* new_value,
Logger* logger) const override {
std::string string_key = key.ToString();
EXPECT_TRUE(string_key == "a" || string_key == "b");
if (string_key == "a") {
EXPECT_EQ(1, operand_list.size());
} else if (string_key == "b") {
EXPECT_EQ(2, operand_list.size());
}
std::string temp_value;
for (auto& operand : operand_list) {
temp_value.append(operand.ToString());
}
swap(temp_value, *new_value);
return true;
}
const char* Name() const override {
return "CompactionIteratorTest SingleMergeOp";
}
bool AllowSingleOperand() const override { return true; }
};
SingleMergeOp merge_op;
Filter filter;
InitIterators(
// a should invoke PartialMergeMulti with a single merge operand.
{test::KeyStr("a", 50, kTypeMerge),
// b should invoke PartialMergeMulti with two operands.
test::KeyStr("b", 70, kTypeMerge), test::KeyStr("b", 60, kTypeMerge),
// c should invoke FullMerge due to kTypeValue at the beginning.
test::KeyStr("c", 90, kTypeMerge), test::KeyStr("c", 80, kTypeValue)},
{"av1", "bv2", "bv1", "cv2", "cv1"}, {}, {}, kMaxSequenceNumber,
&merge_op, &filter);
c_iter_->SeekToFirst();
ASSERT_TRUE(c_iter_->Valid());
ASSERT_EQ(test::KeyStr("a", 50, kTypeMerge), c_iter_->key().ToString());
ASSERT_EQ("av1", c_iter_->value().ToString());
c_iter_->Next();
ASSERT_TRUE(c_iter_->Valid());
ASSERT_EQ("bv1bv2", c_iter_->value().ToString());
c_iter_->Next();
ASSERT_EQ("cv1cv2", c_iter_->value().ToString());
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

1471
db/compaction_job.cc Normal file
View File

File diff suppressed because it is too large Load Diff

165
db/compaction_job.h Normal file
View File

@@ -0,0 +1,165 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include <atomic>
#include <deque>
#include <functional>
#include <limits>
#include <set>
#include <string>
#include <utility>
#include <vector>
#include "db/column_family.h"
#include "db/compaction_iterator.h"
#include "db/dbformat.h"
#include "db/flush_scheduler.h"
#include "db/internal_stats.h"
#include "db/job_context.h"
#include "db/log_writer.h"
#include "db/memtable_list.h"
#include "db/range_del_aggregator.h"
#include "db/version_edit.h"
#include "db/write_controller.h"
#include "db/write_thread.h"
#include "options/db_options.h"
#include "port/port.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/compaction_job_stats.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/transaction_log.h"
#include "table/scoped_arena_iterator.h"
#include "util/autovector.h"
#include "util/event_logger.h"
#include "util/stop_watch.h"
#include "util/thread_local.h"
namespace rocksdb {
class MemTable;
class TableCache;
class Version;
class VersionEdit;
class VersionSet;
class Arena;
class CompactionJob {
public:
CompactionJob(int job_id, Compaction* compaction,
const ImmutableDBOptions& db_options,
const EnvOptions& env_options, VersionSet* versions,
const std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
Directory* db_directory, Directory* output_directory,
Statistics* stats, InstrumentedMutex* db_mutex,
Status* db_bg_error,
std::vector<SequenceNumber> existing_snapshots,
SequenceNumber earliest_write_conflict_snapshot,
std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
bool paranoid_file_checks, bool measure_io_stats,
const std::string& dbname,
CompactionJobStats* compaction_job_stats);
~CompactionJob();
// no copy/move
CompactionJob(CompactionJob&& job) = delete;
CompactionJob(const CompactionJob& job) = delete;
CompactionJob& operator=(const CompactionJob& job) = delete;
// REQUIRED: mutex held
void Prepare();
// REQUIRED mutex not held
Status Run();
// REQUIRED: mutex held
Status Install(const MutableCFOptions& mutable_cf_options);
private:
struct SubcompactionState;
void AggregateStatistics();
void GenSubcompactionBoundaries();
// update the thread status for starting a compaction.
void ReportStartedCompaction(Compaction* compaction);
void AllocateCompactionOutputFileNumbers();
// Call compaction filter. Then iterate through input and compact the
// kv-pairs
void ProcessKeyValueCompaction(SubcompactionState* sub_compact);
Status FinishCompactionOutputFile(
const Status& input_status, SubcompactionState* sub_compact,
RangeDelAggregator* range_del_agg,
CompactionIterationStats* range_del_out_stats,
const Slice* next_table_min_key = nullptr);
Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options);
void RecordCompactionIOStats();
Status OpenCompactionOutputFile(SubcompactionState* sub_compact);
void CleanupCompaction();
void UpdateCompactionJobStats(
const InternalStats::CompactionStats& stats) const;
void RecordDroppedKeys(const CompactionIterationStats& c_iter_stats,
CompactionJobStats* compaction_job_stats = nullptr);
void UpdateCompactionStats();
void UpdateCompactionInputStatsHelper(
int* num_files, uint64_t* bytes_read, int input_level);
void LogCompaction();
int job_id_;
// CompactionJob state
struct CompactionState;
CompactionState* compact_;
CompactionJobStats* compaction_job_stats_;
InternalStats::CompactionStats compaction_stats_;
// DBImpl state
const std::string& dbname_;
const ImmutableDBOptions& db_options_;
const EnvOptions& env_options_;
Env* env_;
VersionSet* versions_;
const std::atomic<bool>* shutting_down_;
LogBuffer* log_buffer_;
Directory* db_directory_;
Directory* output_directory_;
Statistics* stats_;
InstrumentedMutex* db_mutex_;
Status* db_bg_error_;
// If there were two snapshots with seq numbers s1 and
// s2 and s1 < s2, and if we find two instances of a key k1 then lies
// entirely within s1 and s2, then the earlier version of k1 can be safely
// deleted because that version is not visible in any snapshot.
std::vector<SequenceNumber> existing_snapshots_;
// This is the earliest snapshot that could be used for write-conflict
// checking by a transaction. For any user-key newer than this snapshot, we
// should make sure not to remove evidence that a write occurred.
SequenceNumber earliest_write_conflict_snapshot_;
std::shared_ptr<Cache> table_cache_;
EventLogger* event_logger_;
bool bottommost_level_;
bool paranoid_file_checks_;
bool measure_io_stats_;
// Stores the Slices that designate the boundaries for each subcompaction
std::vector<Slice> boundaries_;
// Stores the approx size of keys covered in the range of each subcompaction
std::vector<uint64_t> sizes_;
};
} // namespace rocksdb

1047
db/compaction_job_stats_test.cc Normal file
View File

File diff suppressed because it is too large Load Diff

949
db/compaction_job_test.cc Normal file
View File

@@ -0,0 +1,949 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include <algorithm>
#include <map>
#include <string>
#include <tuple>
#include "db/column_family.h"
#include "db/compaction_job.h"
#include "db/version_set.h"
#include "rocksdb/cache.h"
#include "rocksdb/db.h"
#include "rocksdb/options.h"
#include "rocksdb/write_buffer_manager.h"
#include "table/mock_table.h"
#include "util/file_reader_writer.h"
#include "util/string_util.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "utilities/merge_operators.h"
namespace rocksdb {
namespace {
void VerifyInitializationOfCompactionJobStats(
const CompactionJobStats& compaction_job_stats) {
#if !defined(IOS_CROSS_COMPILE)
ASSERT_EQ(compaction_job_stats.elapsed_micros, 0U);
ASSERT_EQ(compaction_job_stats.num_input_records, 0U);
ASSERT_EQ(compaction_job_stats.num_input_files, 0U);
ASSERT_EQ(compaction_job_stats.num_input_files_at_output_level, 0U);
ASSERT_EQ(compaction_job_stats.num_output_records, 0U);
ASSERT_EQ(compaction_job_stats.num_output_files, 0U);
ASSERT_EQ(compaction_job_stats.is_manual_compaction, true);
ASSERT_EQ(compaction_job_stats.total_input_bytes, 0U);
ASSERT_EQ(compaction_job_stats.total_output_bytes, 0U);
ASSERT_EQ(compaction_job_stats.total_input_raw_key_bytes, 0U);
ASSERT_EQ(compaction_job_stats.total_input_raw_value_bytes, 0U);
ASSERT_EQ(compaction_job_stats.smallest_output_key_prefix[0], 0);
ASSERT_EQ(compaction_job_stats.largest_output_key_prefix[0], 0);
ASSERT_EQ(compaction_job_stats.num_records_replaced, 0U);
ASSERT_EQ(compaction_job_stats.num_input_deletion_records, 0U);
ASSERT_EQ(compaction_job_stats.num_expired_deletion_records, 0U);
ASSERT_EQ(compaction_job_stats.num_corrupt_keys, 0U);
#endif // !defined(IOS_CROSS_COMPILE)
}
} // namespace
// TODO(icanadi) Make it simpler once we mock out VersionSet
class CompactionJobTest : public testing::Test {
public:
CompactionJobTest()
: env_(Env::Default()),
dbname_(test::TmpDir() + "/compaction_job_test"),
db_options_(),
mutable_cf_options_(cf_options_),
table_cache_(NewLRUCache(50000, 16)),
write_buffer_manager_(db_options_.db_write_buffer_size),
versions_(new VersionSet(dbname_, &db_options_, env_options_,
table_cache_.get(), &write_buffer_manager_,
&write_controller_)),
shutting_down_(false),
mock_table_factory_(new mock::MockTableFactory()) {
EXPECT_OK(env_->CreateDirIfMissing(dbname_));
db_options_.db_paths.emplace_back(dbname_,
std::numeric_limits<uint64_t>::max());
}
std::string GenerateFileName(uint64_t file_number) {
FileMetaData meta;
std::vector<DbPath> db_paths;
db_paths.emplace_back(dbname_, std::numeric_limits<uint64_t>::max());
meta.fd = FileDescriptor(file_number, 0, 0);
return TableFileName(db_paths, meta.fd.GetNumber(), meta.fd.GetPathId());
}
std::string KeyStr(const std::string& user_key, const SequenceNumber seq_num,
const ValueType t) {
return InternalKey(user_key, seq_num, t).Encode().ToString();
}
void AddMockFile(const stl_wrappers::KVMap& contents, int level = 0) {
assert(contents.size() > 0);
bool first_key = true;
std::string smallest, largest;
InternalKey smallest_key, largest_key;
SequenceNumber smallest_seqno = kMaxSequenceNumber;
SequenceNumber largest_seqno = 0;
for (auto kv : contents) {
ParsedInternalKey key;
std::string skey;
std::string value;
std::tie(skey, value) = kv;
ParseInternalKey(skey, &key);
smallest_seqno = std::min(smallest_seqno, key.sequence);
largest_seqno = std::max(largest_seqno, key.sequence);
if (first_key ||
cfd_->user_comparator()->Compare(key.user_key, smallest) < 0) {
smallest.assign(key.user_key.data(), key.user_key.size());
smallest_key.DecodeFrom(skey);
}
if (first_key ||
cfd_->user_comparator()->Compare(key.user_key, largest) > 0) {
largest.assign(key.user_key.data(), key.user_key.size());
largest_key.DecodeFrom(skey);
}
first_key = false;
}
uint64_t file_number = versions_->NewFileNumber();
EXPECT_OK(mock_table_factory_->CreateMockTable(
env_, GenerateFileName(file_number), std::move(contents)));
VersionEdit edit;
edit.AddFile(level, file_number, 0, 10, smallest_key, largest_key,
smallest_seqno, largest_seqno, false);
mutex_.Lock();
versions_->LogAndApply(versions_->GetColumnFamilySet()->GetDefault(),
mutable_cf_options_, &edit, &mutex_);
mutex_.Unlock();
}
void SetLastSequence(const SequenceNumber sequence_number) {
versions_->SetLastToBeWrittenSequence(sequence_number + 1);
versions_->SetLastSequence(sequence_number + 1);
}
// returns expected result after compaction
stl_wrappers::KVMap CreateTwoFiles(bool gen_corrupted_keys) {
auto expected_results = mock::MakeMockFile();
const int kKeysPerFile = 10000;
const int kCorruptKeysPerFile = 200;
const int kMatchingKeys = kKeysPerFile / 2;
SequenceNumber sequence_number = 0;
auto corrupt_id = [&](int id) {
return gen_corrupted_keys && id > 0 && id <= kCorruptKeysPerFile;
};
for (int i = 0; i < 2; ++i) {
auto contents = mock::MakeMockFile();
for (int k = 0; k < kKeysPerFile; ++k) {
auto key = ToString(i * kMatchingKeys + k);
auto value = ToString(i * kKeysPerFile + k);
InternalKey internal_key(key, ++sequence_number, kTypeValue);
// This is how the key will look like once it's written in bottommost
// file
InternalKey bottommost_internal_key(
key, (key == "9999") ? sequence_number : 0, kTypeValue);
if (corrupt_id(k)) {
test::CorruptKeyType(&internal_key);
test::CorruptKeyType(&bottommost_internal_key);
}
contents.insert({ internal_key.Encode().ToString(), value });
if (i == 1 || k < kMatchingKeys || corrupt_id(k - kMatchingKeys)) {
expected_results.insert(
{ bottommost_internal_key.Encode().ToString(), value });
}
}
AddMockFile(contents);
}
SetLastSequence(sequence_number);
return expected_results;
}
void NewDB() {
VersionEdit new_db;
new_db.SetLogNumber(0);
new_db.SetNextFile(2);
new_db.SetLastSequence(0);
const std::string manifest = DescriptorFileName(dbname_, 1);
unique_ptr<WritableFile> file;
Status s = env_->NewWritableFile(
manifest, &file, env_->OptimizeForManifestWrite(env_options_));
ASSERT_OK(s);
unique_ptr<WritableFileWriter> file_writer(
new WritableFileWriter(std::move(file), env_options_));
{
log::Writer log(std::move(file_writer), 0, false);
std::string record;
new_db.EncodeTo(&record);
s = log.AddRecord(record);
}
ASSERT_OK(s);
// Make "CURRENT" file that points to the new manifest file.
s = SetCurrentFile(env_, dbname_, 1, nullptr);
std::vector<ColumnFamilyDescriptor> column_families;
cf_options_.table_factory = mock_table_factory_;
cf_options_.merge_operator = merge_op_;
cf_options_.compaction_filter = compaction_filter_.get();
column_families.emplace_back(kDefaultColumnFamilyName, cf_options_);
EXPECT_OK(versions_->Recover(column_families, false));
cfd_ = versions_->GetColumnFamilySet()->GetDefault();
}
void RunCompaction(
const std::vector<std::vector<FileMetaData*>>& input_files,
const stl_wrappers::KVMap& expected_results,
const std::vector<SequenceNumber>& snapshots = {},
SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber) {
auto cfd = versions_->GetColumnFamilySet()->GetDefault();
size_t num_input_files = 0;
std::vector<CompactionInputFiles> compaction_input_files;
for (size_t level = 0; level < input_files.size(); level++) {
auto level_files = input_files[level];
CompactionInputFiles compaction_level;
compaction_level.level = static_cast<int>(level);
compaction_level.files.insert(compaction_level.files.end(),
level_files.begin(), level_files.end());
compaction_input_files.push_back(compaction_level);
num_input_files += level_files.size();
}
Compaction compaction(cfd->current()->storage_info(), *cfd->ioptions(),
*cfd->GetLatestMutableCFOptions(),
compaction_input_files, 1, 1024 * 1024,
10 * 1024 * 1024, 0, kNoCompression, {}, true);
compaction.SetInputVersion(cfd->current());
LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
mutex_.Lock();
EventLogger event_logger(db_options_.info_log.get());
CompactionJob compaction_job(
0, &compaction, db_options_, env_options_, versions_.get(),
&shutting_down_, &log_buffer, nullptr, nullptr, nullptr, &mutex_,
&bg_error_, snapshots, earliest_write_conflict_snapshot, table_cache_,
&event_logger, false, false, dbname_, &compaction_job_stats_);
VerifyInitializationOfCompactionJobStats(compaction_job_stats_);
compaction_job.Prepare();
mutex_.Unlock();
Status s;
s = compaction_job.Run();
ASSERT_OK(s);
mutex_.Lock();
ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions()));
mutex_.Unlock();
if (expected_results.size() == 0) {
ASSERT_GE(compaction_job_stats_.elapsed_micros, 0U);
ASSERT_EQ(compaction_job_stats_.num_input_files, num_input_files);
ASSERT_EQ(compaction_job_stats_.num_output_files, 0U);
} else {
ASSERT_GE(compaction_job_stats_.elapsed_micros, 0U);
ASSERT_EQ(compaction_job_stats_.num_input_files, num_input_files);
ASSERT_EQ(compaction_job_stats_.num_output_files, 1U);
mock_table_factory_->AssertLatestFile(expected_results);
}
}
Env* env_;
std::string dbname_;
EnvOptions env_options_;
ImmutableDBOptions db_options_;
ColumnFamilyOptions cf_options_;
MutableCFOptions mutable_cf_options_;
std::shared_ptr<Cache> table_cache_;
WriteController write_controller_;
WriteBufferManager write_buffer_manager_;
std::unique_ptr<VersionSet> versions_;
InstrumentedMutex mutex_;
std::atomic<bool> shutting_down_;
std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
CompactionJobStats compaction_job_stats_;
ColumnFamilyData* cfd_;
std::unique_ptr<CompactionFilter> compaction_filter_;
std::shared_ptr<MergeOperator> merge_op_;
Status bg_error_;
};
TEST_F(CompactionJobTest, Simple) {
NewDB();
auto expected_results = CreateTwoFiles(false);
auto cfd = versions_->GetColumnFamilySet()->GetDefault();
auto files = cfd->current()->storage_info()->LevelFiles(0);
ASSERT_EQ(2U, files.size());
RunCompaction({ files }, expected_results);
}
TEST_F(CompactionJobTest, SimpleCorrupted) {
NewDB();
auto expected_results = CreateTwoFiles(true);
auto cfd = versions_->GetColumnFamilySet()->GetDefault();
auto files = cfd->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results);
ASSERT_EQ(compaction_job_stats_.num_corrupt_keys, 400U);
}
TEST_F(CompactionJobTest, SimpleDeletion) {
NewDB();
auto file1 = mock::MakeMockFile({{KeyStr("c", 4U, kTypeDeletion), ""},
{KeyStr("c", 3U, kTypeValue), "val"}});
AddMockFile(file1);
auto file2 = mock::MakeMockFile({{KeyStr("b", 2U, kTypeValue), "val"},
{KeyStr("b", 1U, kTypeValue), "val"}});
AddMockFile(file2);
auto expected_results =
mock::MakeMockFile({{KeyStr("b", 0U, kTypeValue), "val"}});
SetLastSequence(4U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results);
}
TEST_F(CompactionJobTest, OutputNothing) {
NewDB();
auto file1 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"}});
AddMockFile(file1);
auto file2 = mock::MakeMockFile({{KeyStr("a", 2U, kTypeDeletion), ""}});
AddMockFile(file2);
auto expected_results = mock::MakeMockFile();
SetLastSequence(4U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results);
}
TEST_F(CompactionJobTest, SimpleOverwrite) {
NewDB();
auto file1 = mock::MakeMockFile({
{KeyStr("a", 3U, kTypeValue), "val2"},
{KeyStr("b", 4U, kTypeValue), "val3"},
});
AddMockFile(file1);
auto file2 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"},
{KeyStr("b", 2U, kTypeValue), "val"}});
AddMockFile(file2);
auto expected_results =
mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "val2"},
{KeyStr("b", 4U, kTypeValue), "val3"}});
SetLastSequence(4U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results);
}
TEST_F(CompactionJobTest, SimpleNonLastLevel) {
NewDB();
auto file1 = mock::MakeMockFile({
{KeyStr("a", 5U, kTypeValue), "val2"},
{KeyStr("b", 6U, kTypeValue), "val3"},
});
AddMockFile(file1);
auto file2 = mock::MakeMockFile({{KeyStr("a", 3U, kTypeValue), "val"},
{KeyStr("b", 4U, kTypeValue), "val"}});
AddMockFile(file2, 1);
auto file3 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"},
{KeyStr("b", 2U, kTypeValue), "val"}});
AddMockFile(file3, 2);
// Because level 1 is not the last level, the sequence numbers of a and b
// cannot be set to 0
auto expected_results =
mock::MakeMockFile({{KeyStr("a", 5U, kTypeValue), "val2"},
{KeyStr("b", 6U, kTypeValue), "val3"}});
SetLastSequence(6U);
auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0);
auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1);
RunCompaction({lvl0_files, lvl1_files}, expected_results);
}
TEST_F(CompactionJobTest, SimpleMerge) {
merge_op_ = MergeOperators::CreateStringAppendOperator();
NewDB();
auto file1 = mock::MakeMockFile({
{KeyStr("a", 5U, kTypeMerge), "5"},
{KeyStr("a", 4U, kTypeMerge), "4"},
{KeyStr("a", 3U, kTypeValue), "3"},
});
AddMockFile(file1);
auto file2 = mock::MakeMockFile(
{{KeyStr("b", 2U, kTypeMerge), "2"}, {KeyStr("b", 1U, kTypeValue), "1"}});
AddMockFile(file2);
auto expected_results =
mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"},
{KeyStr("b", 2U, kTypeValue), "1,2"}});
SetLastSequence(5U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results);
}
TEST_F(CompactionJobTest, NonAssocMerge) {
merge_op_ = MergeOperators::CreateStringAppendTESTOperator();
NewDB();
auto file1 = mock::MakeMockFile({
{KeyStr("a", 5U, kTypeMerge), "5"},
{KeyStr("a", 4U, kTypeMerge), "4"},
{KeyStr("a", 3U, kTypeMerge), "3"},
});
AddMockFile(file1);
auto file2 = mock::MakeMockFile(
{{KeyStr("b", 2U, kTypeMerge), "2"}, {KeyStr("b", 1U, kTypeMerge), "1"}});
AddMockFile(file2);
auto expected_results =
mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"},
{KeyStr("b", 2U, kTypeMerge), "2"},
{KeyStr("b", 1U, kTypeMerge), "1"}});
SetLastSequence(5U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results);
}
// Filters merge operands with value 10.
TEST_F(CompactionJobTest, MergeOperandFilter) {
merge_op_ = MergeOperators::CreateUInt64AddOperator();
compaction_filter_.reset(new test::FilterNumber(10U));
NewDB();
auto file1 = mock::MakeMockFile(
{{KeyStr("a", 5U, kTypeMerge), test::EncodeInt(5U)},
{KeyStr("a", 4U, kTypeMerge), test::EncodeInt(10U)}, // Filtered
{KeyStr("a", 3U, kTypeMerge), test::EncodeInt(3U)}});
AddMockFile(file1);
auto file2 = mock::MakeMockFile({
{KeyStr("b", 2U, kTypeMerge), test::EncodeInt(2U)},
{KeyStr("b", 1U, kTypeMerge), test::EncodeInt(10U)} // Filtered
});
AddMockFile(file2);
auto expected_results =
mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), test::EncodeInt(8U)},
{KeyStr("b", 2U, kTypeMerge), test::EncodeInt(2U)}});
SetLastSequence(5U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results);
}
TEST_F(CompactionJobTest, FilterSomeMergeOperands) {
merge_op_ = MergeOperators::CreateUInt64AddOperator();
compaction_filter_.reset(new test::FilterNumber(10U));
NewDB();
auto file1 = mock::MakeMockFile(
{{KeyStr("a", 5U, kTypeMerge), test::EncodeInt(5U)},
{KeyStr("a", 4U, kTypeMerge), test::EncodeInt(10U)}, // Filtered
{KeyStr("a", 3U, kTypeValue), test::EncodeInt(5U)},
{KeyStr("d", 8U, kTypeMerge), test::EncodeInt(10U)}});
AddMockFile(file1);
auto file2 =
mock::MakeMockFile({{KeyStr("b", 2U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("b", 1U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("c", 2U, kTypeMerge), test::EncodeInt(3U)},
{KeyStr("c", 1U, kTypeValue), test::EncodeInt(7U)},
{KeyStr("d", 1U, kTypeValue), test::EncodeInt(6U)}});
AddMockFile(file2);
auto file3 =
mock::MakeMockFile({{KeyStr("a", 1U, kTypeMerge), test::EncodeInt(3U)}});
AddMockFile(file3, 2);
auto expected_results = mock::MakeMockFile({
{KeyStr("a", 5U, kTypeValue), test::EncodeInt(10U)},
{KeyStr("c", 2U, kTypeValue), test::EncodeInt(10U)},
{KeyStr("d", 1U, kTypeValue), test::EncodeInt(6U)}
// b does not appear because the operands are filtered
});
SetLastSequence(5U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results);
}
// Test where all operands/merge results are filtered out.
TEST_F(CompactionJobTest, FilterAllMergeOperands) {
merge_op_ = MergeOperators::CreateUInt64AddOperator();
compaction_filter_.reset(new test::FilterNumber(10U));
NewDB();
auto file1 =
mock::MakeMockFile({{KeyStr("a", 11U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("a", 10U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("a", 9U, kTypeMerge), test::EncodeInt(10U)}});
AddMockFile(file1);
auto file2 =
mock::MakeMockFile({{KeyStr("b", 8U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("b", 7U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("b", 6U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("b", 5U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("b", 4U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("b", 3U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("b", 2U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("c", 2U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("c", 1U, kTypeMerge), test::EncodeInt(10U)}});
AddMockFile(file2);
auto file3 =
mock::MakeMockFile({{KeyStr("a", 2U, kTypeMerge), test::EncodeInt(10U)},
{KeyStr("b", 1U, kTypeMerge), test::EncodeInt(10U)}});
AddMockFile(file3, 2);
SetLastSequence(11U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
stl_wrappers::KVMap empty_map;
RunCompaction({files}, empty_map);
}
TEST_F(CompactionJobTest, SimpleSingleDelete) {
NewDB();
auto file1 = mock::MakeMockFile({
{KeyStr("a", 5U, kTypeDeletion), ""},
{KeyStr("b", 6U, kTypeSingleDeletion), ""},
});
AddMockFile(file1);
auto file2 = mock::MakeMockFile({{KeyStr("a", 3U, kTypeValue), "val"},
{KeyStr("b", 4U, kTypeValue), "val"}});
AddMockFile(file2);
auto file3 = mock::MakeMockFile({
{KeyStr("a", 1U, kTypeValue), "val"},
});
AddMockFile(file3, 2);
auto expected_results =
mock::MakeMockFile({{KeyStr("a", 5U, kTypeDeletion), ""}});
SetLastSequence(6U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results);
}
TEST_F(CompactionJobTest, SingleDeleteSnapshots) {
NewDB();
auto file1 = mock::MakeMockFile({
{KeyStr("A", 12U, kTypeSingleDeletion), ""},
{KeyStr("a", 12U, kTypeSingleDeletion), ""},
{KeyStr("b", 21U, kTypeSingleDeletion), ""},
{KeyStr("c", 22U, kTypeSingleDeletion), ""},
{KeyStr("d", 9U, kTypeSingleDeletion), ""},
{KeyStr("f", 21U, kTypeSingleDeletion), ""},
{KeyStr("j", 11U, kTypeSingleDeletion), ""},
{KeyStr("j", 9U, kTypeSingleDeletion), ""},
{KeyStr("k", 12U, kTypeSingleDeletion), ""},
{KeyStr("k", 11U, kTypeSingleDeletion), ""},
{KeyStr("l", 3U, kTypeSingleDeletion), ""},
{KeyStr("l", 2U, kTypeSingleDeletion), ""},
});
AddMockFile(file1);
auto file2 = mock::MakeMockFile({
{KeyStr("0", 2U, kTypeSingleDeletion), ""},
{KeyStr("a", 11U, kTypeValue), "val1"},
{KeyStr("b", 11U, kTypeValue), "val2"},
{KeyStr("c", 21U, kTypeValue), "val3"},
{KeyStr("d", 8U, kTypeValue), "val4"},
{KeyStr("e", 2U, kTypeSingleDeletion), ""},
{KeyStr("f", 1U, kTypeValue), "val1"},
{KeyStr("g", 11U, kTypeSingleDeletion), ""},
{KeyStr("h", 2U, kTypeSingleDeletion), ""},
{KeyStr("m", 12U, kTypeValue), "val1"},
{KeyStr("m", 11U, kTypeSingleDeletion), ""},
{KeyStr("m", 8U, kTypeValue), "val2"},
});
AddMockFile(file2);
auto file3 = mock::MakeMockFile({
{KeyStr("A", 1U, kTypeValue), "val"},
{KeyStr("e", 1U, kTypeValue), "val"},
});
AddMockFile(file3, 2);
auto expected_results = mock::MakeMockFile({
{KeyStr("A", 12U, kTypeSingleDeletion), ""},
{KeyStr("a", 12U, kTypeSingleDeletion), ""},
{KeyStr("a", 11U, kTypeValue), ""},
{KeyStr("b", 21U, kTypeSingleDeletion), ""},
{KeyStr("b", 11U, kTypeValue), "val2"},
{KeyStr("c", 22U, kTypeSingleDeletion), ""},
{KeyStr("c", 21U, kTypeValue), ""},
{KeyStr("e", 2U, kTypeSingleDeletion), ""},
{KeyStr("f", 21U, kTypeSingleDeletion), ""},
{KeyStr("f", 1U, kTypeValue), "val1"},
{KeyStr("g", 11U, kTypeSingleDeletion), ""},
{KeyStr("j", 11U, kTypeSingleDeletion), ""},
{KeyStr("k", 11U, kTypeSingleDeletion), ""},
{KeyStr("m", 12U, kTypeValue), "val1"},
{KeyStr("m", 11U, kTypeSingleDeletion), ""},
{KeyStr("m", 8U, kTypeValue), "val2"},
});
SetLastSequence(22U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results, {10U, 20U}, 10U);
}
TEST_F(CompactionJobTest, EarliestWriteConflictSnapshot) {
NewDB();
// Test multiple snapshots where the earliest snapshot is not a
// write-conflic-snapshot.
auto file1 = mock::MakeMockFile({
{KeyStr("A", 24U, kTypeSingleDeletion), ""},
{KeyStr("A", 23U, kTypeValue), "val"},
{KeyStr("B", 24U, kTypeSingleDeletion), ""},
{KeyStr("B", 23U, kTypeValue), "val"},
{KeyStr("D", 24U, kTypeSingleDeletion), ""},
{KeyStr("G", 32U, kTypeSingleDeletion), ""},
{KeyStr("G", 31U, kTypeValue), "val"},
{KeyStr("G", 24U, kTypeSingleDeletion), ""},
{KeyStr("G", 23U, kTypeValue), "val2"},
{KeyStr("H", 31U, kTypeValue), "val"},
{KeyStr("H", 24U, kTypeSingleDeletion), ""},
{KeyStr("H", 23U, kTypeValue), "val"},
{KeyStr("I", 35U, kTypeSingleDeletion), ""},
{KeyStr("I", 34U, kTypeValue), "val2"},
{KeyStr("I", 33U, kTypeSingleDeletion), ""},
{KeyStr("I", 32U, kTypeValue), "val3"},
{KeyStr("I", 31U, kTypeSingleDeletion), ""},
{KeyStr("J", 34U, kTypeValue), "val"},
{KeyStr("J", 33U, kTypeSingleDeletion), ""},
{KeyStr("J", 25U, kTypeValue), "val2"},
{KeyStr("J", 24U, kTypeSingleDeletion), ""},
});
AddMockFile(file1);
auto file2 = mock::MakeMockFile({
{KeyStr("A", 14U, kTypeSingleDeletion), ""},
{KeyStr("A", 13U, kTypeValue), "val2"},
{KeyStr("C", 14U, kTypeSingleDeletion), ""},
{KeyStr("C", 13U, kTypeValue), "val"},
{KeyStr("E", 12U, kTypeSingleDeletion), ""},
{KeyStr("F", 4U, kTypeSingleDeletion), ""},
{KeyStr("F", 3U, kTypeValue), "val"},
{KeyStr("G", 14U, kTypeSingleDeletion), ""},
{KeyStr("G", 13U, kTypeValue), "val3"},
{KeyStr("H", 14U, kTypeSingleDeletion), ""},
{KeyStr("H", 13U, kTypeValue), "val2"},
{KeyStr("I", 13U, kTypeValue), "val4"},
{KeyStr("I", 12U, kTypeSingleDeletion), ""},
{KeyStr("I", 11U, kTypeValue), "val5"},
{KeyStr("J", 15U, kTypeValue), "val3"},
{KeyStr("J", 14U, kTypeSingleDeletion), ""},
});
AddMockFile(file2);
auto expected_results = mock::MakeMockFile({
{KeyStr("A", 24U, kTypeSingleDeletion), ""},
{KeyStr("A", 23U, kTypeValue), ""},
{KeyStr("B", 24U, kTypeSingleDeletion), ""},
{KeyStr("B", 23U, kTypeValue), ""},
{KeyStr("D", 24U, kTypeSingleDeletion), ""},
{KeyStr("E", 12U, kTypeSingleDeletion), ""},
{KeyStr("G", 32U, kTypeSingleDeletion), ""},
{KeyStr("G", 31U, kTypeValue), ""},
{KeyStr("H", 31U, kTypeValue), "val"},
{KeyStr("I", 35U, kTypeSingleDeletion), ""},
{KeyStr("I", 34U, kTypeValue), ""},
{KeyStr("I", 31U, kTypeSingleDeletion), ""},
{KeyStr("I", 13U, kTypeValue), "val4"},
{KeyStr("J", 34U, kTypeValue), "val"},
{KeyStr("J", 33U, kTypeSingleDeletion), ""},
{KeyStr("J", 25U, kTypeValue), "val2"},
{KeyStr("J", 24U, kTypeSingleDeletion), ""},
{KeyStr("J", 15U, kTypeValue), "val3"},
{KeyStr("J", 14U, kTypeSingleDeletion), ""},
});
SetLastSequence(24U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results, {10U, 20U, 30U}, 20U);
}
TEST_F(CompactionJobTest, SingleDeleteZeroSeq) {
NewDB();
auto file1 = mock::MakeMockFile({
{KeyStr("A", 10U, kTypeSingleDeletion), ""},
{KeyStr("dummy", 5U, kTypeValue), "val2"},
});
AddMockFile(file1);
auto file2 = mock::MakeMockFile({
{KeyStr("A", 0U, kTypeValue), "val"},
});
AddMockFile(file2);
auto expected_results = mock::MakeMockFile({
{KeyStr("dummy", 5U, kTypeValue), "val2"},
});
SetLastSequence(22U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results, {});
}
TEST_F(CompactionJobTest, MultiSingleDelete) {
// Tests three scenarios involving multiple single delete/put pairs:
//
// A: Put Snapshot SDel Put SDel -> Put Snapshot SDel
// B: Snapshot Put SDel Put SDel Snapshot -> Snapshot SDel Snapshot
// C: SDel Put SDel Snapshot Put -> Snapshot Put
// D: (Put) SDel Snapshot Put SDel -> (Put) SDel Snapshot SDel
// E: Put SDel Snapshot Put SDel -> Snapshot SDel
// F: Put SDel Put Sdel Snapshot -> removed
// G: Snapshot SDel Put SDel Put -> Snapshot Put SDel
// H: (Put) Put SDel Put Sdel Snapshot -> Removed
// I: (Put) Snapshot Put SDel Put SDel -> SDel
// J: Put Put SDel Put SDel SDel Snapshot Put Put SDel SDel Put
// -> Snapshot Put
// K: SDel SDel Put SDel Put Put Snapshot SDel Put SDel SDel Put SDel
// -> Snapshot Put Snapshot SDel
// L: SDel Put Del Put SDel Snapshot Del Put Del SDel Put SDel
// -> Snapshot SDel
// M: (Put) SDel Put Del Put SDel Snapshot Put Del SDel Put SDel Del
// -> SDel Snapshot Del
NewDB();
auto file1 = mock::MakeMockFile({
{KeyStr("A", 14U, kTypeSingleDeletion), ""},
{KeyStr("A", 13U, kTypeValue), "val5"},
{KeyStr("A", 12U, kTypeSingleDeletion), ""},
{KeyStr("B", 14U, kTypeSingleDeletion), ""},
{KeyStr("B", 13U, kTypeValue), "val2"},
{KeyStr("C", 14U, kTypeValue), "val3"},
{KeyStr("D", 12U, kTypeSingleDeletion), ""},
{KeyStr("D", 11U, kTypeValue), "val4"},
{KeyStr("G", 15U, kTypeValue), "val"},
{KeyStr("G", 14U, kTypeSingleDeletion), ""},
{KeyStr("G", 13U, kTypeValue), "val"},
{KeyStr("I", 14U, kTypeSingleDeletion), ""},
{KeyStr("I", 13U, kTypeValue), "val"},
{KeyStr("J", 15U, kTypeValue), "val"},
{KeyStr("J", 14U, kTypeSingleDeletion), ""},
{KeyStr("J", 13U, kTypeSingleDeletion), ""},
{KeyStr("J", 12U, kTypeValue), "val"},
{KeyStr("J", 11U, kTypeValue), "val"},
{KeyStr("K", 16U, kTypeSingleDeletion), ""},
{KeyStr("K", 15U, kTypeValue), "val1"},
{KeyStr("K", 14U, kTypeSingleDeletion), ""},
{KeyStr("K", 13U, kTypeSingleDeletion), ""},
{KeyStr("K", 12U, kTypeValue), "val2"},
{KeyStr("K", 11U, kTypeSingleDeletion), ""},
{KeyStr("L", 16U, kTypeSingleDeletion), ""},
{KeyStr("L", 15U, kTypeValue), "val"},
{KeyStr("L", 14U, kTypeSingleDeletion), ""},
{KeyStr("L", 13U, kTypeDeletion), ""},
{KeyStr("L", 12U, kTypeValue), "val"},
{KeyStr("L", 11U, kTypeDeletion), ""},
{KeyStr("M", 16U, kTypeDeletion), ""},
{KeyStr("M", 15U, kTypeSingleDeletion), ""},
{KeyStr("M", 14U, kTypeValue), "val"},
{KeyStr("M", 13U, kTypeSingleDeletion), ""},
{KeyStr("M", 12U, kTypeDeletion), ""},
{KeyStr("M", 11U, kTypeValue), "val"},
});
AddMockFile(file1);
auto file2 = mock::MakeMockFile({
{KeyStr("A", 10U, kTypeValue), "val"},
{KeyStr("B", 12U, kTypeSingleDeletion), ""},
{KeyStr("B", 11U, kTypeValue), "val2"},
{KeyStr("C", 10U, kTypeSingleDeletion), ""},
{KeyStr("C", 9U, kTypeValue), "val6"},
{KeyStr("C", 8U, kTypeSingleDeletion), ""},
{KeyStr("D", 10U, kTypeSingleDeletion), ""},
{KeyStr("E", 12U, kTypeSingleDeletion), ""},
{KeyStr("E", 11U, kTypeValue), "val"},
{KeyStr("E", 5U, kTypeSingleDeletion), ""},
{KeyStr("E", 4U, kTypeValue), "val"},
{KeyStr("F", 6U, kTypeSingleDeletion), ""},
{KeyStr("F", 5U, kTypeValue), "val"},
{KeyStr("F", 4U, kTypeSingleDeletion), ""},
{KeyStr("F", 3U, kTypeValue), "val"},
{KeyStr("G", 12U, kTypeSingleDeletion), ""},
{KeyStr("H", 6U, kTypeSingleDeletion), ""},
{KeyStr("H", 5U, kTypeValue), "val"},
{KeyStr("H", 4U, kTypeSingleDeletion), ""},
{KeyStr("H", 3U, kTypeValue), "val"},
{KeyStr("I", 12U, kTypeSingleDeletion), ""},
{KeyStr("I", 11U, kTypeValue), "val"},
{KeyStr("J", 6U, kTypeSingleDeletion), ""},
{KeyStr("J", 5U, kTypeSingleDeletion), ""},
{KeyStr("J", 4U, kTypeValue), "val"},
{KeyStr("J", 3U, kTypeSingleDeletion), ""},
{KeyStr("J", 2U, kTypeValue), "val"},
{KeyStr("K", 8U, kTypeValue), "val3"},
{KeyStr("K", 7U, kTypeValue), "val4"},
{KeyStr("K", 6U, kTypeSingleDeletion), ""},
{KeyStr("K", 5U, kTypeValue), "val5"},
{KeyStr("K", 2U, kTypeSingleDeletion), ""},
{KeyStr("K", 1U, kTypeSingleDeletion), ""},
{KeyStr("L", 5U, kTypeSingleDeletion), ""},
{KeyStr("L", 4U, kTypeValue), "val"},
{KeyStr("L", 3U, kTypeDeletion), ""},
{KeyStr("L", 2U, kTypeValue), "val"},
{KeyStr("L", 1U, kTypeSingleDeletion), ""},
{KeyStr("M", 10U, kTypeSingleDeletion), ""},
{KeyStr("M", 7U, kTypeValue), "val"},
{KeyStr("M", 5U, kTypeDeletion), ""},
{KeyStr("M", 4U, kTypeValue), "val"},
{KeyStr("M", 3U, kTypeSingleDeletion), ""},
});
AddMockFile(file2);
auto file3 = mock::MakeMockFile({
{KeyStr("D", 1U, kTypeValue), "val"},
{KeyStr("H", 1U, kTypeValue), "val"},
{KeyStr("I", 2U, kTypeValue), "val"},
});
AddMockFile(file3, 2);
auto file4 = mock::MakeMockFile({
{KeyStr("M", 1U, kTypeValue), "val"},
});
AddMockFile(file4, 2);
auto expected_results =
mock::MakeMockFile({{KeyStr("A", 14U, kTypeSingleDeletion), ""},
{KeyStr("A", 13U, kTypeValue), ""},
{KeyStr("A", 12U, kTypeSingleDeletion), ""},
{KeyStr("A", 10U, kTypeValue), "val"},
{KeyStr("B", 14U, kTypeSingleDeletion), ""},
{KeyStr("B", 13U, kTypeValue), ""},
{KeyStr("C", 14U, kTypeValue), "val3"},
{KeyStr("D", 12U, kTypeSingleDeletion), ""},
{KeyStr("D", 11U, kTypeValue), ""},
{KeyStr("D", 10U, kTypeSingleDeletion), ""},
{KeyStr("E", 12U, kTypeSingleDeletion), ""},
{KeyStr("E", 11U, kTypeValue), ""},
{KeyStr("G", 15U, kTypeValue), "val"},
{KeyStr("G", 12U, kTypeSingleDeletion), ""},
{KeyStr("I", 14U, kTypeSingleDeletion), ""},
{KeyStr("I", 13U, kTypeValue), ""},
{KeyStr("J", 15U, kTypeValue), "val"},
{KeyStr("K", 16U, kTypeSingleDeletion), ""},
{KeyStr("K", 15U, kTypeValue), ""},
{KeyStr("K", 11U, kTypeSingleDeletion), ""},
{KeyStr("K", 8U, kTypeValue), "val3"},
{KeyStr("L", 16U, kTypeSingleDeletion), ""},
{KeyStr("L", 15U, kTypeValue), ""},
{KeyStr("M", 16U, kTypeDeletion), ""},
{KeyStr("M", 3U, kTypeSingleDeletion), ""}});
SetLastSequence(22U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results, {10U}, 10U);
}
// This test documents the behavior where a corrupt key follows a deletion or a
// single deletion and the (single) deletion gets removed while the corrupt key
// gets written out. TODO(noetzli): We probably want a better way to treat
// corrupt keys.
TEST_F(CompactionJobTest, CorruptionAfterDeletion) {
NewDB();
auto file1 =
mock::MakeMockFile({{test::KeyStr("A", 6U, kTypeValue), "val3"},
{test::KeyStr("a", 5U, kTypeDeletion), ""},
{test::KeyStr("a", 4U, kTypeValue, true), "val"}});
AddMockFile(file1);
auto file2 =
mock::MakeMockFile({{test::KeyStr("b", 3U, kTypeSingleDeletion), ""},
{test::KeyStr("b", 2U, kTypeValue, true), "val"},
{test::KeyStr("c", 1U, kTypeValue), "val2"}});
AddMockFile(file2);
auto expected_results =
mock::MakeMockFile({{test::KeyStr("A", 0U, kTypeValue), "val3"},
{test::KeyStr("a", 0U, kTypeValue, true), "val"},
{test::KeyStr("b", 0U, kTypeValue, true), "val"},
{test::KeyStr("c", 1U, kTypeValue), "val2"}});
SetLastSequence(6U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
RunCompaction({files}, expected_results);
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
#include <stdio.h>
int main(int argc, char** argv) {
fprintf(stderr,
"SKIPPED as CompactionJobStats is not supported in ROCKSDB_LITE\n");
return 0;
}
#endif // ROCKSDB_LITE

2090
db/compaction_picker.cc
View File

File diff suppressed because it is too large Load Diff

320
db/compaction_picker.h
View File

@@ -1,39 +1,47 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include "db/version_set.h"
#include "db/compaction.h"
#include "rocksdb/status.h"
#include "rocksdb/options.h"
#include "rocksdb/env.h"
#include <vector>
#include <memory>
#include <set>
#include <string>
#include <unordered_set>
#include <vector>
#include "db/compaction.h"
#include "db/version_set.h"
#include "options/cf_options.h"
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "rocksdb/status.h"
namespace rocksdb {
class LogBuffer;
class Compaction;
class Version;
class VersionStorageInfo;
struct CompactionInputFiles;
class CompactionPicker {
public:
CompactionPicker(const Options* options, const InternalKeyComparator* icmp);
CompactionPicker(const ImmutableCFOptions& ioptions,
const InternalKeyComparator* icmp);
virtual ~CompactionPicker();
// Pick level and inputs for a new compaction.
// Returns nullptr if there is no compaction to be done.
// Otherwise returns a pointer to a heap-allocated object that
// describes the compaction. Caller should delete the result.
virtual Compaction* PickCompaction(Version* version,
virtual Compaction* PickCompaction(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage,
LogBuffer* log_buffer) = 0;
// Return a compaction object for compacting the range [begin,end] in
@@ -47,49 +55,84 @@ class CompactionPicker {
// compaction_end will be set to nullptr.
// Client is responsible for compaction_end storage -- when called,
// *compaction_end should point to valid InternalKey!
virtual Compaction* CompactRange(Version* version, int input_level,
int output_level, uint32_t output_path_id,
const InternalKey* begin,
const InternalKey* end,
InternalKey** compaction_end);
virtual Compaction* CompactRange(
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, int input_level, int output_level,
uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
InternalKey** compaction_end, bool* manual_conflict);
// Given the current number of levels, returns the lowest allowed level
// for compaction input.
virtual int MaxInputLevel(int current_num_levels) const = 0;
// The maximum allowed output level. Default value is NumberLevels() - 1.
virtual int MaxOutputLevel() const { return NumberLevels() - 1; }
virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const = 0;
// Sanitize the input set of compaction input files.
// When the input parameters do not describe a valid compaction, the
// function will try to fix the input_files by adding necessary
// files. If it's not possible to conver an invalid input_files
// into a valid one by adding more files, the function will return a
// non-ok status with specific reason.
#ifndef ROCKSDB_LITE
Status SanitizeCompactionInputFiles(std::unordered_set<uint64_t>* input_files,
const ColumnFamilyMetaData& cf_meta,
const int output_level) const;
#endif // ROCKSDB_LITE
// Free up the files that participated in a compaction
//
// Requirement: DB mutex held
void ReleaseCompactionFiles(Compaction* c, Status status);
// Return the total amount of data that is undergoing
// compactions per level
void SizeBeingCompacted(std::vector<uint64_t>& sizes);
// Returns true if any one of the specified files are being compacted
bool AreFilesInCompaction(const std::vector<FileMetaData*>& files);
// Returns maximum total overlap bytes with grandparent
// level (i.e., level+2) before we stop building a single
// file in level->level+1 compaction.
uint64_t MaxGrandParentOverlapBytes(int level);
// Takes a list of CompactionInputFiles and returns a (manual) Compaction
// object.
Compaction* CompactFiles(const CompactionOptions& compact_options,
const std::vector<CompactionInputFiles>& input_files,
int output_level, VersionStorageInfo* vstorage,
const MutableCFOptions& mutable_cf_options,
uint32_t output_path_id);
// Returns maximum total bytes of data on a given level.
double MaxBytesForLevel(int level);
// Converts a set of compaction input file numbers into
// a list of CompactionInputFiles.
Status GetCompactionInputsFromFileNumbers(
std::vector<CompactionInputFiles>* input_files,
std::unordered_set<uint64_t>* input_set,
const VersionStorageInfo* vstorage,
const CompactionOptions& compact_options) const;
// Get the max file size in a given level.
uint64_t MaxFileSizeForLevel(int level) const;
// Is there currently a compaction involving level 0 taking place
bool IsLevel0CompactionInProgress() const {
return !level0_compactions_in_progress_.empty();
}
protected:
int NumberLevels() const { return num_levels_; }
// Return true if the passed key range overlap with a compaction output
// that is currently running.
bool RangeOverlapWithCompaction(const Slice& smallest_user_key,
const Slice& largest_user_key,
int level) const;
// Stores the minimal range that covers all entries in inputs in
// *smallest, *largest.
// REQUIRES: inputs is not empty
void GetRange(const std::vector<FileMetaData*>& inputs, InternalKey* smallest,
InternalKey* largest);
void GetRange(const CompactionInputFiles& inputs, InternalKey* smallest,
InternalKey* largest) const;
// Stores the minimal range that covers all entries in inputs1 and inputs2
// in *smallest, *largest.
// REQUIRES: inputs is not empty
void GetRange(const std::vector<FileMetaData*>& inputs1,
const std::vector<FileMetaData*>& inputs2,
InternalKey* smallest, InternalKey* largest);
void GetRange(const CompactionInputFiles& inputs1,
const CompactionInputFiles& inputs2, InternalKey* smallest,
InternalKey* largest) const;
// Stores the minimal range that covers all entries in inputs
// in *smallest, *largest.
// REQUIRES: inputs is not empty (at least on entry have one file)
void GetRange(const std::vector<CompactionInputFiles>& inputs,
InternalKey* smallest, InternalKey* largest) const;
int NumberLevels() const { return ioptions_.num_levels; }
// Add more files to the inputs on "level" to make sure that
// no newer version of a key is compacted to "level+1" while leaving an older
@@ -101,110 +144,155 @@ class CompactionPicker {
// populated.
//
// Will return false if it is impossible to apply this compaction.
bool ExpandWhileOverlapping(Compaction* c);
uint64_t ExpandedCompactionByteSizeLimit(int level);
// Returns true if any one of the specified files are being compacted
bool FilesInCompaction(std::vector<FileMetaData*>& files);
bool ExpandInputsToCleanCut(const std::string& cf_name,
VersionStorageInfo* vstorage,
CompactionInputFiles* inputs);
// Returns true if any one of the parent files are being compacted
bool ParentRangeInCompaction(Version* version, const InternalKey* smallest,
const InternalKey* largest, int level,
int* index);
bool IsRangeInCompaction(VersionStorageInfo* vstorage,
const InternalKey* smallest,
const InternalKey* largest, int level, int* index);
void SetupOtherInputs(Compaction* c);
// Returns true if the key range that `inputs` files cover overlap with the
// key range of a currently running compaction.
bool FilesRangeOverlapWithCompaction(
const std::vector<CompactionInputFiles>& inputs, int level) const;
// record all the ongoing compactions for all levels
std::vector<std::set<Compaction*>> compactions_in_progress_;
bool SetupOtherInputs(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage,
CompactionInputFiles* inputs,
CompactionInputFiles* output_level_inputs,
int* parent_index, int base_index);
// Per-level target file size.
std::unique_ptr<uint64_t[]> max_file_size_;
void GetGrandparents(VersionStorageInfo* vstorage,
const CompactionInputFiles& inputs,
const CompactionInputFiles& output_level_inputs,
std::vector<FileMetaData*>* grandparents);
// Per-level max bytes
std::unique_ptr<uint64_t[]> level_max_bytes_;
// Register this compaction in the set of running compactions
void RegisterCompaction(Compaction* c);
const Options* const options_;
// Remove this compaction from the set of running compactions
void UnregisterCompaction(Compaction* c);
private:
int num_levels_;
std::set<Compaction*>* level0_compactions_in_progress() {
return &level0_compactions_in_progress_;
}
std::unordered_set<Compaction*>* compactions_in_progress() {
return &compactions_in_progress_;
}
protected:
const ImmutableCFOptions& ioptions_;
// A helper function to SanitizeCompactionInputFiles() that
// sanitizes "input_files" by adding necessary files.
#ifndef ROCKSDB_LITE
virtual Status SanitizeCompactionInputFilesForAllLevels(
std::unordered_set<uint64_t>* input_files,
const ColumnFamilyMetaData& cf_meta, const int output_level) const;
#endif // ROCKSDB_LITE
// Keeps track of all compactions that are running on Level0.
// Protected by DB mutex
std::set<Compaction*> level0_compactions_in_progress_;
// Keeps track of all compactions that are running.
// Protected by DB mutex
std::unordered_set<Compaction*> compactions_in_progress_;
const InternalKeyComparator* const icmp_;
};
class UniversalCompactionPicker : public CompactionPicker {
public:
UniversalCompactionPicker(const Options* options,
const InternalKeyComparator* icmp)
: CompactionPicker(options, icmp) {}
virtual Compaction* PickCompaction(Version* version,
LogBuffer* log_buffer) override;
// The maxinum allowed input level. Always return 0.
virtual int MaxInputLevel(int current_num_levels) const override {
return 0;
}
private:
// Pick Universal compaction to limit read amplification
Compaction* PickCompactionUniversalReadAmp(Version* version, double score,
unsigned int ratio,
unsigned int num_files,
LogBuffer* log_buffer);
// Pick Universal compaction to limit space amplification.
Compaction* PickCompactionUniversalSizeAmp(Version* version, double score,
LogBuffer* log_buffer);
// Pick a path ID to place a newly generated file, with its estimated file
// size.
static uint32_t GetPathId(const Options& options, uint64_t file_size);
};
class LevelCompactionPicker : public CompactionPicker {
public:
LevelCompactionPicker(const Options* options,
LevelCompactionPicker(const ImmutableCFOptions& ioptions,
const InternalKeyComparator* icmp)
: CompactionPicker(options, icmp) {}
virtual Compaction* PickCompaction(Version* version,
: CompactionPicker(ioptions, icmp) {}
virtual Compaction* PickCompaction(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage,
LogBuffer* log_buffer) override;
// Returns current_num_levels - 2, meaning the last level cannot be
// compaction input level.
virtual int MaxInputLevel(int current_num_levels) const override {
return current_num_levels - 2;
}
private:
// For the specfied level, pick a compaction.
// Returns nullptr if there is no compaction to be done.
// If level is 0 and there is already a compaction on that level, this
// function will return nullptr.
Compaction* PickCompactionBySize(Version* version, int level, double score);
virtual bool NeedsCompaction(
const VersionStorageInfo* vstorage) const override;
};
#ifndef ROCKSDB_LITE
class FIFOCompactionPicker : public CompactionPicker {
public:
FIFOCompactionPicker(const Options* options,
FIFOCompactionPicker(const ImmutableCFOptions& ioptions,
const InternalKeyComparator* icmp)
: CompactionPicker(options, icmp) {}
: CompactionPicker(ioptions, icmp) {}
virtual Compaction* PickCompaction(Version* version,
virtual Compaction* PickCompaction(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* version,
LogBuffer* log_buffer) override;
virtual Compaction* CompactRange(Version* version, int input_level,
int output_level, uint32_t output_path_id,
const InternalKey* begin,
const InternalKey* end,
InternalKey** compaction_end) override;
virtual Compaction* CompactRange(
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, int input_level, int output_level,
uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
InternalKey** compaction_end, bool* manual_conflict) override;
// The maxinum allowed input level. Always return 0.
virtual int MaxInputLevel(int current_num_levels) const override {
return 0;
}
// The maximum allowed output level. Always returns 0.
virtual int MaxOutputLevel() const override { return 0; }
virtual bool NeedsCompaction(
const VersionStorageInfo* vstorage) const override;
private:
Compaction* PickTTLCompaction(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* version,
LogBuffer* log_buffer);
Compaction* PickSizeCompaction(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* version,
LogBuffer* log_buffer);
};
// Utility function
extern uint64_t TotalCompensatedFileSize(const std::vector<FileMetaData*>& files);
class NullCompactionPicker : public CompactionPicker {
public:
NullCompactionPicker(const ImmutableCFOptions& ioptions,
const InternalKeyComparator* icmp)
: CompactionPicker(ioptions, icmp) {}
virtual ~NullCompactionPicker() {}
// Always return "nullptr"
Compaction* PickCompaction(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage,
LogBuffer* log_buffer) override {
return nullptr;
}
// Always return "nullptr"
Compaction* CompactRange(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, int input_level,
int output_level, uint32_t output_path_id,
const InternalKey* begin, const InternalKey* end,
InternalKey** compaction_end,
bool* manual_conflict) override {
return nullptr;
}
// Always returns false.
virtual bool NeedsCompaction(
const VersionStorageInfo* vstorage) const override {
return false;
}
};
#endif // !ROCKSDB_LITE
CompressionType GetCompressionType(const ImmutableCFOptions& ioptions,
const VersionStorageInfo* vstorage,
const MutableCFOptions& mutable_cf_options,
int level, int base_level,
const bool enable_compression = true);
} // namespace rocksdb

1441
db/compaction_picker_test.cc Normal file
View File

File diff suppressed because it is too large Load Diff

748
db/compaction_picker_universal.cc Normal file
View File

@@ -0,0 +1,748 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/compaction_picker_universal.h"
#ifndef ROCKSDB_LITE
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include <inttypes.h>
#include <limits>
#include <queue>
#include <string>
#include <utility>
#include "db/column_family.h"
#include "monitoring/statistics.h"
#include "util/filename.h"
#include "util/log_buffer.h"
#include "util/random.h"
#include "util/string_util.h"
#include "util/sync_point.h"
namespace rocksdb {
namespace {
// Used in universal compaction when trivial move is enabled.
// This structure is used for the construction of min heap
// that contains the file meta data, the level of the file
// and the index of the file in that level
struct InputFileInfo {
InputFileInfo() : f(nullptr) {}
FileMetaData* f;
size_t level;
size_t index;
};
// Used in universal compaction when trivial move is enabled.
// This comparator is used for the construction of min heap
// based on the smallest key of the file.
struct SmallestKeyHeapComparator {
explicit SmallestKeyHeapComparator(const Comparator* ucmp) { ucmp_ = ucmp; }
bool operator()(InputFileInfo i1, InputFileInfo i2) const {
return (ucmp_->Compare(i1.f->smallest.user_key(),
i2.f->smallest.user_key()) > 0);
}
private:
const Comparator* ucmp_;
};
typedef std::priority_queue<InputFileInfo, std::vector<InputFileInfo>,
SmallestKeyHeapComparator>
SmallestKeyHeap;
// This function creates the heap that is used to find if the files are
// overlapping during universal compaction when the allow_trivial_move
// is set.
SmallestKeyHeap create_level_heap(Compaction* c, const Comparator* ucmp) {
SmallestKeyHeap smallest_key_priority_q =
SmallestKeyHeap(SmallestKeyHeapComparator(ucmp));
InputFileInfo input_file;
for (size_t l = 0; l < c->num_input_levels(); l++) {
if (c->num_input_files(l) != 0) {
if (l == 0 && c->start_level() == 0) {
for (size_t i = 0; i < c->num_input_files(0); i++) {
input_file.f = c->input(0, i);
input_file.level = 0;
input_file.index = i;
smallest_key_priority_q.push(std::move(input_file));
}
} else {
input_file.f = c->input(l, 0);
input_file.level = l;
input_file.index = 0;
smallest_key_priority_q.push(std::move(input_file));
}
}
}
return smallest_key_priority_q;
}
#ifndef NDEBUG
// smallest_seqno and largest_seqno are set iff. `files` is not empty.
void GetSmallestLargestSeqno(const std::vector<FileMetaData*>& files,
SequenceNumber* smallest_seqno,
SequenceNumber* largest_seqno) {
bool is_first = true;
for (FileMetaData* f : files) {
assert(f->smallest_seqno <= f->largest_seqno);
if (is_first) {
is_first = false;
*smallest_seqno = f->smallest_seqno;
*largest_seqno = f->largest_seqno;
} else {
if (f->smallest_seqno < *smallest_seqno) {
*smallest_seqno = f->smallest_seqno;
}
if (f->largest_seqno > *largest_seqno) {
*largest_seqno = f->largest_seqno;
}
}
}
}
#endif
} // namespace
// Algorithm that checks to see if there are any overlapping
// files in the input
bool UniversalCompactionPicker::IsInputFilesNonOverlapping(Compaction* c) {
auto comparator = icmp_->user_comparator();
int first_iter = 1;
InputFileInfo prev, curr, next;
SmallestKeyHeap smallest_key_priority_q =
create_level_heap(c, icmp_->user_comparator());
while (!smallest_key_priority_q.empty()) {
curr = smallest_key_priority_q.top();
smallest_key_priority_q.pop();
if (first_iter) {
prev = curr;
first_iter = 0;
} else {
if (comparator->Compare(prev.f->largest.user_key(),
curr.f->smallest.user_key()) >= 0) {
// found overlapping files, return false
return false;
}
assert(comparator->Compare(curr.f->largest.user_key(),
prev.f->largest.user_key()) > 0);
prev = curr;
}
next.f = nullptr;
if (curr.level != 0 && curr.index < c->num_input_files(curr.level) - 1) {
next.f = c->input(curr.level, curr.index + 1);
next.level = curr.level;
next.index = curr.index + 1;
}
if (next.f) {
smallest_key_priority_q.push(std::move(next));
}
}
return true;
}
bool UniversalCompactionPicker::NeedsCompaction(
const VersionStorageInfo* vstorage) const {
const int kLevel0 = 0;
return vstorage->CompactionScore(kLevel0) >= 1;
}
void UniversalCompactionPicker::SortedRun::Dump(char* out_buf,
size_t out_buf_size,
bool print_path) const {
if (level == 0) {
assert(file != nullptr);
if (file->fd.GetPathId() == 0 || !print_path) {
snprintf(out_buf, out_buf_size, "file %" PRIu64, file->fd.GetNumber());
} else {
snprintf(out_buf, out_buf_size, "file %" PRIu64
"(path "
"%" PRIu32 ")",
file->fd.GetNumber(), file->fd.GetPathId());
}
} else {
snprintf(out_buf, out_buf_size, "level %d", level);
}
}
void UniversalCompactionPicker::SortedRun::DumpSizeInfo(
char* out_buf, size_t out_buf_size, size_t sorted_run_count) const {
if (level == 0) {
assert(file != nullptr);
snprintf(out_buf, out_buf_size,
"file %" PRIu64 "[%" ROCKSDB_PRIszt
"] "
"with size %" PRIu64 " (compensated size %" PRIu64 ")",
file->fd.GetNumber(), sorted_run_count, file->fd.GetFileSize(),
file->compensated_file_size);
} else {
snprintf(out_buf, out_buf_size,
"level %d[%" ROCKSDB_PRIszt
"] "
"with size %" PRIu64 " (compensated size %" PRIu64 ")",
level, sorted_run_count, size, compensated_file_size);
}
}
std::vector<UniversalCompactionPicker::SortedRun>
UniversalCompactionPicker::CalculateSortedRuns(
const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions) {
std::vector<UniversalCompactionPicker::SortedRun> ret;
for (FileMetaData* f : vstorage.LevelFiles(0)) {
ret.emplace_back(0, f, f->fd.GetFileSize(), f->compensated_file_size,
f->being_compacted);
}
for (int level = 1; level < vstorage.num_levels(); level++) {
uint64_t total_compensated_size = 0U;
uint64_t total_size = 0U;
bool being_compacted = false;
bool is_first = true;
for (FileMetaData* f : vstorage.LevelFiles(level)) {
total_compensated_size += f->compensated_file_size;
total_size += f->fd.GetFileSize();
if (ioptions.compaction_options_universal.allow_trivial_move == true) {
if (f->being_compacted) {
being_compacted = f->being_compacted;
}
} else {
// Compaction always includes all files for a non-zero level, so for a
// non-zero level, all the files should share the same being_compacted
// value.
// This assumption is only valid when
// ioptions.compaction_options_universal.allow_trivial_move is false
assert(is_first || f->being_compacted == being_compacted);
}
if (is_first) {
being_compacted = f->being_compacted;
is_first = false;
}
}
if (total_compensated_size > 0) {
ret.emplace_back(level, nullptr, total_size, total_compensated_size,
being_compacted);
}
}
return ret;
}
// Universal style of compaction. Pick files that are contiguous in
// time-range to compact.
//
Compaction* UniversalCompactionPicker::PickCompaction(
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
const int kLevel0 = 0;
double score = vstorage->CompactionScore(kLevel0);
std::vector<SortedRun> sorted_runs =
CalculateSortedRuns(*vstorage, ioptions_);
if (sorted_runs.size() == 0 ||
sorted_runs.size() <
(unsigned int)mutable_cf_options.level0_file_num_compaction_trigger) {
ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: nothing to do\n",
cf_name.c_str());
TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
nullptr);
return nullptr;
}
VersionStorageInfo::LevelSummaryStorage tmp;
ROCKS_LOG_BUFFER_MAX_SZ(
log_buffer, 3072,
"[%s] Universal: sorted runs files(%" ROCKSDB_PRIszt "): %s\n",
cf_name.c_str(), sorted_runs.size(), vstorage->LevelSummary(&tmp));
// Check for size amplification first.
Compaction* c;
if ((c = PickCompactionToReduceSizeAmp(cf_name, mutable_cf_options, vstorage,
score, sorted_runs, log_buffer)) !=
nullptr) {
ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: compacting for size amp\n",
cf_name.c_str());
} else {
// Size amplification is within limits. Try reducing read
// amplification while maintaining file size ratios.
unsigned int ratio = ioptions_.compaction_options_universal.size_ratio;
if ((c = PickCompactionToReduceSortedRuns(
cf_name, mutable_cf_options, vstorage, score, ratio, UINT_MAX,
sorted_runs, log_buffer)) != nullptr) {
ROCKS_LOG_BUFFER(log_buffer,
"[%s] Universal: compacting for size ratio\n",
cf_name.c_str());
} else {
// Size amplification and file size ratios are within configured limits.
// If max read amplification is exceeding configured limits, then force
// compaction without looking at filesize ratios and try to reduce
// the number of files to fewer than level0_file_num_compaction_trigger.
// This is guaranteed by NeedsCompaction()
assert(sorted_runs.size() >=
static_cast<size_t>(
mutable_cf_options.level0_file_num_compaction_trigger));
// Get the total number of sorted runs that are not being compacted
int num_sr_not_compacted = 0;
for (size_t i = 0; i < sorted_runs.size(); i++) {
if (sorted_runs[i].being_compacted == false) {
num_sr_not_compacted++;
}
}
// The number of sorted runs that are not being compacted is greater than
// the maximum allowed number of sorted runs
if (num_sr_not_compacted >
mutable_cf_options.level0_file_num_compaction_trigger) {
unsigned int num_files =
num_sr_not_compacted -
mutable_cf_options.level0_file_num_compaction_trigger + 1;
if ((c = PickCompactionToReduceSortedRuns(
cf_name, mutable_cf_options, vstorage, score, UINT_MAX,
num_files, sorted_runs, log_buffer)) != nullptr) {
ROCKS_LOG_BUFFER(log_buffer,
"[%s] Universal: compacting for file num -- %u\n",
cf_name.c_str(), num_files);
}
}
}
}
if (c == nullptr) {
TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
nullptr);
return nullptr;
}
if (ioptions_.compaction_options_universal.allow_trivial_move == true) {
c->set_is_trivial_move(IsInputFilesNonOverlapping(c));
}
// validate that all the chosen files of L0 are non overlapping in time
#ifndef NDEBUG
SequenceNumber prev_smallest_seqno = 0U;
bool is_first = true;
size_t level_index = 0U;
if (c->start_level() == 0) {
for (auto f : *c->inputs(0)) {
assert(f->smallest_seqno <= f->largest_seqno);
if (is_first) {
is_first = false;
}
prev_smallest_seqno = f->smallest_seqno;
}
level_index = 1U;
}
for (; level_index < c->num_input_levels(); level_index++) {
if (c->num_input_files(level_index) != 0) {
SequenceNumber smallest_seqno = 0U;
SequenceNumber largest_seqno = 0U;
GetSmallestLargestSeqno(*(c->inputs(level_index)), &smallest_seqno,
&largest_seqno);
if (is_first) {
is_first = false;
} else if (prev_smallest_seqno > 0) {
// A level is considered as the bottommost level if there are
// no files in higher levels or if files in higher levels do
// not overlap with the files being compacted. Sequence numbers
// of files in bottommost level can be set to 0 to help
// compression. As a result, the following assert may not hold
// if the prev_smallest_seqno is 0.
assert(prev_smallest_seqno > largest_seqno);
}
prev_smallest_seqno = smallest_seqno;
}
}
#endif
// update statistics
MeasureTime(ioptions_.statistics, NUM_FILES_IN_SINGLE_COMPACTION,
c->inputs(0)->size());
RegisterCompaction(c);
vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options);
TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
c);
return c;
}
uint32_t UniversalCompactionPicker::GetPathId(
const ImmutableCFOptions& ioptions, uint64_t file_size) {
// Two conditions need to be satisfied:
// (1) the target path needs to be able to hold the file's size
// (2) Total size left in this and previous paths need to be not
// smaller than expected future file size before this new file is
// compacted, which is estimated based on size_ratio.
// For example, if now we are compacting files of size (1, 1, 2, 4, 8),
// we will make sure the target file, probably with size of 16, will be
// placed in a path so that eventually when new files are generated and
// compacted to (1, 1, 2, 4, 8, 16), all those files can be stored in or
// before the path we chose.
//
// TODO(sdong): now the case of multiple column families is not
// considered in this algorithm. So the target size can be violated in
// that case. We need to improve it.
uint64_t accumulated_size = 0;
uint64_t future_size =
file_size * (100 - ioptions.compaction_options_universal.size_ratio) /
100;
uint32_t p = 0;
assert(!ioptions.db_paths.empty());
for (; p < ioptions.db_paths.size() - 1; p++) {
uint64_t target_size = ioptions.db_paths[p].target_size;
if (target_size > file_size &&
accumulated_size + (target_size - file_size) > future_size) {
return p;
}
accumulated_size += target_size;
}
return p;
}
//
// Consider compaction files based on their size differences with
// the next file in time order.
//
Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, double score, unsigned int ratio,
unsigned int max_number_of_files_to_compact,
const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
unsigned int min_merge_width =
ioptions_.compaction_options_universal.min_merge_width;
unsigned int max_merge_width =
ioptions_.compaction_options_universal.max_merge_width;
const SortedRun* sr = nullptr;
bool done = false;
size_t start_index = 0;
unsigned int candidate_count = 0;
unsigned int max_files_to_compact =
std::min(max_merge_width, max_number_of_files_to_compact);
min_merge_width = std::max(min_merge_width, 2U);
// Caller checks the size before executing this function. This invariant is
// important because otherwise we may have a possible integer underflow when
// dealing with unsigned types.
assert(sorted_runs.size() > 0);
// Considers a candidate file only if it is smaller than the
// total size accumulated so far.
for (size_t loop = 0; loop < sorted_runs.size(); loop++) {
candidate_count = 0;
// Skip files that are already being compacted
for (sr = nullptr; loop < sorted_runs.size(); loop++) {
sr = &sorted_runs[loop];
if (!sr->being_compacted) {
candidate_count = 1;
break;
}
char file_num_buf[kFormatFileNumberBufSize];
sr->Dump(file_num_buf, sizeof(file_num_buf));
ROCKS_LOG_BUFFER(log_buffer,
"[%s] Universal: %s"
"[%d] being compacted, skipping",
cf_name.c_str(), file_num_buf, loop);
sr = nullptr;
}
// This file is not being compacted. Consider it as the
// first candidate to be compacted.
uint64_t candidate_size = sr != nullptr ? sr->compensated_file_size : 0;
if (sr != nullptr) {
char file_num_buf[kFormatFileNumberBufSize];
sr->Dump(file_num_buf, sizeof(file_num_buf), true);
ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: Possible candidate %s[%d].",
cf_name.c_str(), file_num_buf, loop);
}
// Check if the succeeding files need compaction.
for (size_t i = loop + 1;
candidate_count < max_files_to_compact && i < sorted_runs.size();
i++) {
const SortedRun* succeeding_sr = &sorted_runs[i];
if (succeeding_sr->being_compacted) {
break;
}
// Pick files if the total/last candidate file size (increased by the
// specified ratio) is still larger than the next candidate file.
// candidate_size is the total size of files picked so far with the
// default kCompactionStopStyleTotalSize; with
// kCompactionStopStyleSimilarSize, it's simply the size of the last
// picked file.
double sz = candidate_size * (100.0 + ratio) / 100.0;
if (sz < static_cast<double>(succeeding_sr->size)) {
break;
}
if (ioptions_.compaction_options_universal.stop_style ==
kCompactionStopStyleSimilarSize) {
// Similar-size stopping rule: also check the last picked file isn't
// far larger than the next candidate file.
sz = (succeeding_sr->size * (100.0 + ratio)) / 100.0;
if (sz < static_cast<double>(candidate_size)) {
// If the small file we've encountered begins a run of similar-size
// files, we'll pick them up on a future iteration of the outer
// loop. If it's some lonely straggler, it'll eventually get picked
// by the last-resort read amp strategy which disregards size ratios.
break;
}
candidate_size = succeeding_sr->compensated_file_size;
} else { // default kCompactionStopStyleTotalSize
candidate_size += succeeding_sr->compensated_file_size;
}
candidate_count++;
}
// Found a series of consecutive files that need compaction.
if (candidate_count >= (unsigned int)min_merge_width) {
start_index = loop;
done = true;
break;
} else {
for (size_t i = loop;
i < loop + candidate_count && i < sorted_runs.size(); i++) {
const SortedRun* skipping_sr = &sorted_runs[i];
char file_num_buf[256];
skipping_sr->DumpSizeInfo(file_num_buf, sizeof(file_num_buf), loop);
ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: Skipping %s",
cf_name.c_str(), file_num_buf);
}
}
}
if (!done || candidate_count <= 1) {
return nullptr;
}
size_t first_index_after = start_index + candidate_count;
// Compression is enabled if files compacted earlier already reached
// size ratio of compression.
bool enable_compression = true;
int ratio_to_compress =
ioptions_.compaction_options_universal.compression_size_percent;
if (ratio_to_compress >= 0) {
uint64_t total_size = 0;
for (auto& sorted_run : sorted_runs) {
total_size += sorted_run.compensated_file_size;
}
uint64_t older_file_size = 0;
for (size_t i = sorted_runs.size() - 1; i >= first_index_after; i--) {
older_file_size += sorted_runs[i].size;
if (older_file_size * 100L >= total_size * (long)ratio_to_compress) {
enable_compression = false;
break;
}
}
}
uint64_t estimated_total_size = 0;
for (unsigned int i = 0; i < first_index_after; i++) {
estimated_total_size += sorted_runs[i].size;
}
uint32_t path_id = GetPathId(ioptions_, estimated_total_size);
int start_level = sorted_runs[start_index].level;
int output_level;
if (first_index_after == sorted_runs.size()) {
output_level = vstorage->num_levels() - 1;
} else if (sorted_runs[first_index_after].level == 0) {
output_level = 0;
} else {
output_level = sorted_runs[first_index_after].level - 1;
}
// last level is reserved for the files ingested behind
if (ioptions_.allow_ingest_behind &&
(output_level == vstorage->num_levels() - 1)) {
assert(output_level > 1);
output_level--;
}
std::vector<CompactionInputFiles> inputs(vstorage->num_levels());
for (size_t i = 0; i < inputs.size(); ++i) {
inputs[i].level = start_level + static_cast<int>(i);
}
for (size_t i = start_index; i < first_index_after; i++) {
auto& picking_sr = sorted_runs[i];
if (picking_sr.level == 0) {
FileMetaData* picking_file = picking_sr.file;
inputs[0].files.push_back(picking_file);
} else {
auto& files = inputs[picking_sr.level - start_level].files;
for (auto* f : vstorage->LevelFiles(picking_sr.level)) {
files.push_back(f);
}
}
char file_num_buf[256];
picking_sr.DumpSizeInfo(file_num_buf, sizeof(file_num_buf), i);
ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: Picking %s", cf_name.c_str(),
file_num_buf);
}
CompactionReason compaction_reason;
if (max_number_of_files_to_compact == UINT_MAX) {
compaction_reason = CompactionReason::kUniversalSortedRunNum;
} else {
compaction_reason = CompactionReason::kUniversalSizeRatio;
}
return new Compaction(
vstorage, ioptions_, mutable_cf_options, std::move(inputs), output_level,
mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id,
GetCompressionType(ioptions_, vstorage, mutable_cf_options, start_level,
1, enable_compression),
/* grandparents */ {}, /* is manual */ false, score,
false /* deletion_compaction */, compaction_reason);
}
// Look at overall size amplification. If size amplification
// exceeeds the configured value, then do a compaction
// of the candidate files all the way upto the earliest
// base file (overrides configured values of file-size ratios,
// min_merge_width and max_merge_width).
//
Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, double score,
const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
// percentage flexibility while reducing size amplification
uint64_t ratio =
ioptions_.compaction_options_universal.max_size_amplification_percent;
unsigned int candidate_count = 0;
uint64_t candidate_size = 0;
size_t start_index = 0;
const SortedRun* sr = nullptr;
// Skip files that are already being compacted
for (size_t loop = 0; loop < sorted_runs.size() - 1; loop++) {
sr = &sorted_runs[loop];
if (!sr->being_compacted) {
start_index = loop; // Consider this as the first candidate.
break;
}
char file_num_buf[kFormatFileNumberBufSize];
sr->Dump(file_num_buf, sizeof(file_num_buf), true);
ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: skipping %s[%d] compacted %s",
cf_name.c_str(), file_num_buf, loop,
" cannot be a candidate to reduce size amp.\n");
sr = nullptr;
}
if (sr == nullptr) {
return nullptr; // no candidate files
}
{
char file_num_buf[kFormatFileNumberBufSize];
sr->Dump(file_num_buf, sizeof(file_num_buf), true);
ROCKS_LOG_BUFFER(
log_buffer,
"[%s] Universal: First candidate %s[%" ROCKSDB_PRIszt "] %s",
cf_name.c_str(), file_num_buf, start_index, " to reduce size amp.\n");
}
// keep adding up all the remaining files
for (size_t loop = start_index; loop < sorted_runs.size() - 1; loop++) {
sr = &sorted_runs[loop];
if (sr->being_compacted) {
char file_num_buf[kFormatFileNumberBufSize];
sr->Dump(file_num_buf, sizeof(file_num_buf), true);
ROCKS_LOG_BUFFER(
log_buffer, "[%s] Universal: Possible candidate %s[%d] %s",
cf_name.c_str(), file_num_buf, start_index,
" is already being compacted. No size amp reduction possible.\n");
return nullptr;
}
candidate_size += sr->compensated_file_size;
candidate_count++;
}
if (candidate_count == 0) {
return nullptr;
}
// size of earliest file
uint64_t earliest_file_size = sorted_runs.back().size;
// size amplification = percentage of additional size
if (candidate_size * 100 < ratio * earliest_file_size) {
ROCKS_LOG_BUFFER(
log_buffer,
"[%s] Universal: size amp not needed. newer-files-total-size %" PRIu64
" earliest-file-size %" PRIu64,
cf_name.c_str(), candidate_size, earliest_file_size);
return nullptr;
} else {
ROCKS_LOG_BUFFER(
log_buffer,
"[%s] Universal: size amp needed. newer-files-total-size %" PRIu64
" earliest-file-size %" PRIu64,
cf_name.c_str(), candidate_size, earliest_file_size);
}
assert(start_index < sorted_runs.size() - 1);
// Estimate total file size
uint64_t estimated_total_size = 0;
for (size_t loop = start_index; loop < sorted_runs.size(); loop++) {
estimated_total_size += sorted_runs[loop].size;
}
uint32_t path_id = GetPathId(ioptions_, estimated_total_size);
int start_level = sorted_runs[start_index].level;
std::vector<CompactionInputFiles> inputs(vstorage->num_levels());
for (size_t i = 0; i < inputs.size(); ++i) {
inputs[i].level = start_level + static_cast<int>(i);
}
// We always compact all the files, so always compress.
for (size_t loop = start_index; loop < sorted_runs.size(); loop++) {
auto& picking_sr = sorted_runs[loop];
if (picking_sr.level == 0) {
FileMetaData* f = picking_sr.file;
inputs[0].files.push_back(f);
} else {
auto& files = inputs[picking_sr.level - start_level].files;
for (auto* f : vstorage->LevelFiles(picking_sr.level)) {
files.push_back(f);
}
}
char file_num_buf[256];
picking_sr.DumpSizeInfo(file_num_buf, sizeof(file_num_buf), loop);
ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: size amp picking %s",
cf_name.c_str(), file_num_buf);
}
// output files at the bottom most level, unless it's reserved
int output_level = vstorage->num_levels() - 1;
// last level is reserved for the files ingested behind
if (ioptions_.allow_ingest_behind) {
assert(output_level > 1);
output_level--;
}
return new Compaction(
vstorage, ioptions_, mutable_cf_options, std::move(inputs),
output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
/* max_grandparent_overlap_bytes */ LLONG_MAX, path_id,
GetCompressionType(ioptions_, vstorage, mutable_cf_options,
output_level, 1),
/* grandparents */ {}, /* is manual */ false, score,
false /* deletion_compaction */,
CompactionReason::kUniversalSizeAmplification);
}
} // namespace rocksdb
#endif // !ROCKSDB_LITE

91
db/compaction_picker_universal.h Normal file
View File

@@ -0,0 +1,91 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#ifndef ROCKSDB_LITE
#include "db/compaction_picker.h"
namespace rocksdb {
class UniversalCompactionPicker : public CompactionPicker {
public:
UniversalCompactionPicker(const ImmutableCFOptions& ioptions,
const InternalKeyComparator* icmp)
: CompactionPicker(ioptions, icmp) {}
virtual Compaction* PickCompaction(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage,
LogBuffer* log_buffer) override;
virtual int MaxOutputLevel() const override { return NumberLevels() - 1; }
virtual bool NeedsCompaction(
const VersionStorageInfo* vstorage) const override;
private:
struct SortedRun {
SortedRun(int _level, FileMetaData* _file, uint64_t _size,
uint64_t _compensated_file_size, bool _being_compacted)
: level(_level),
file(_file),
size(_size),
compensated_file_size(_compensated_file_size),
being_compacted(_being_compacted) {
assert(compensated_file_size > 0);
assert(level != 0 || file != nullptr);
}
void Dump(char* out_buf, size_t out_buf_size,
bool print_path = false) const;
// sorted_run_count is added into the string to print
void DumpSizeInfo(char* out_buf, size_t out_buf_size,
size_t sorted_run_count) const;
int level;
// `file` Will be null for level > 0. For level = 0, the sorted run is
// for this file.
FileMetaData* file;
// For level > 0, `size` and `compensated_file_size` are sum of sizes all
// files in the level. `being_compacted` should be the same for all files
// in a non-zero level. Use the value here.
uint64_t size;
uint64_t compensated_file_size;
bool being_compacted;
};
// Pick Universal compaction to limit read amplification
Compaction* PickCompactionToReduceSortedRuns(
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, double score, unsigned int ratio,
unsigned int num_files, const std::vector<SortedRun>& sorted_runs,
LogBuffer* log_buffer);
// Pick Universal compaction to limit space amplification.
Compaction* PickCompactionToReduceSizeAmp(
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, double score,
const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer);
// Used in universal compaction when the enabled_trivial_move
// option is set. Checks whether there are any overlapping files
// in the input. Returns true if the input files are non
// overlapping.
bool IsInputFilesNonOverlapping(Compaction* c);
static std::vector<SortedRun> CalculateSortedRuns(
const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions);
// Pick a path ID to place a newly generated file, with its estimated file
// size.
static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
uint64_t file_size);
};
} // namespace rocksdb
#endif // !ROCKSDB_LITE

441
db/comparator_db_test.cc Normal file
View File

@@ -0,0 +1,441 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include <map>
#include <string>
#include "memtable/stl_wrappers.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "util/hash.h"
#include "util/kv_map.h"
#include "util/string_util.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "utilities/merge_operators.h"
using std::unique_ptr;
namespace rocksdb {
namespace {
static const Comparator* comparator;
class KVIter : public Iterator {
public:
explicit KVIter(const stl_wrappers::KVMap* map)
: map_(map), iter_(map_->end()) {}
virtual bool Valid() const override { return iter_ != map_->end(); }
virtual void SeekToFirst() override { iter_ = map_->begin(); }
virtual void SeekToLast() override {
if (map_->empty()) {
iter_ = map_->end();
} else {
iter_ = map_->find(map_->rbegin()->first);
}
}
virtual void Seek(const Slice& k) override {
iter_ = map_->lower_bound(k.ToString());
}
virtual void SeekForPrev(const Slice& k) override {
iter_ = map_->upper_bound(k.ToString());
Prev();
}
virtual void Next() override { ++iter_; }
virtual void Prev() override {
if (iter_ == map_->begin()) {
iter_ = map_->end();
return;
}
--iter_;
}
virtual Slice key() const override { return iter_->first; }
virtual Slice value() const override { return iter_->second; }
virtual Status status() const override { return Status::OK(); }
private:
const stl_wrappers::KVMap* const map_;
stl_wrappers::KVMap::const_iterator iter_;
};
void AssertItersEqual(Iterator* iter1, Iterator* iter2) {
ASSERT_EQ(iter1->Valid(), iter2->Valid());
if (iter1->Valid()) {
ASSERT_EQ(iter1->key().ToString(), iter2->key().ToString());
ASSERT_EQ(iter1->value().ToString(), iter2->value().ToString());
}
}
// Measuring operations on DB (expect to be empty).
// source_strings are candidate keys
void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
Random* rnd, int num_writes, int num_iter_ops,
int num_trigger_flush) {
stl_wrappers::KVMap map((stl_wrappers::LessOfComparator(comparator)));
for (int i = 0; i < num_writes; i++) {
if (num_trigger_flush > 0 && i != 0 && i % num_trigger_flush == 0) {
db->Flush(FlushOptions());
}
int type = rnd->Uniform(2);
int index = rnd->Uniform(static_cast<int>(source_strings.size()));
auto& key = source_strings[index];
switch (type) {
case 0:
// put
map[key] = key;
ASSERT_OK(db->Put(WriteOptions(), key, key));
break;
case 1:
// delete
if (map.find(key) != map.end()) {
map.erase(key);
}
ASSERT_OK(db->Delete(WriteOptions(), key));
break;
default:
assert(false);
}
}
std::unique_ptr<Iterator> iter(db->NewIterator(ReadOptions()));
std::unique_ptr<Iterator> result_iter(new KVIter(&map));
bool is_valid = false;
for (int i = 0; i < num_iter_ops; i++) {
// Random walk and make sure iter and result_iter returns the
// same key and value
int type = rnd->Uniform(6);
ASSERT_OK(iter->status());
switch (type) {
case 0:
// Seek to First
iter->SeekToFirst();
result_iter->SeekToFirst();
break;
case 1:
// Seek to last
iter->SeekToLast();
result_iter->SeekToLast();
break;
case 2: {
// Seek to random key
auto key_idx = rnd->Uniform(static_cast<int>(source_strings.size()));
auto key = source_strings[key_idx];
iter->Seek(key);
result_iter->Seek(key);
break;
}
case 3:
// Next
if (is_valid) {
iter->Next();
result_iter->Next();
} else {
continue;
}
break;
case 4:
// Prev
if (is_valid) {
iter->Prev();
result_iter->Prev();
} else {
continue;
}
break;
default: {
assert(type == 5);
auto key_idx = rnd->Uniform(static_cast<int>(source_strings.size()));
auto key = source_strings[key_idx];
std::string result;
auto status = db->Get(ReadOptions(), key, &result);
if (map.find(key) == map.end()) {
ASSERT_TRUE(status.IsNotFound());
} else {
ASSERT_EQ(map[key], result);
}
break;
}
}
AssertItersEqual(iter.get(), result_iter.get());
is_valid = iter->Valid();
}
}
class DoubleComparator : public Comparator {
public:
DoubleComparator() {}
virtual const char* Name() const override { return "DoubleComparator"; }
virtual int Compare(const Slice& a, const Slice& b) const override {
#ifndef CYGWIN
double da = std::stod(a.ToString());
double db = std::stod(b.ToString());
#else
double da = std::strtod(a.ToString().c_str(), 0 /* endptr */);
double db = std::strtod(a.ToString().c_str(), 0 /* endptr */);
#endif
if (da == db) {
return a.compare(b);
} else if (da > db) {
return 1;
} else {
return -1;
}
}
virtual void FindShortestSeparator(std::string* start,
const Slice& limit) const override {}
virtual void FindShortSuccessor(std::string* key) const override {}
};
class HashComparator : public Comparator {
public:
HashComparator() {}
virtual const char* Name() const override { return "HashComparator"; }
virtual int Compare(const Slice& a, const Slice& b) const override {
uint32_t ha = Hash(a.data(), a.size(), 66);
uint32_t hb = Hash(b.data(), b.size(), 66);
if (ha == hb) {
return a.compare(b);
} else if (ha > hb) {
return 1;
} else {
return -1;
}
}
virtual void FindShortestSeparator(std::string* start,
const Slice& limit) const override {}
virtual void FindShortSuccessor(std::string* key) const override {}
};
class TwoStrComparator : public Comparator {
public:
TwoStrComparator() {}
virtual const char* Name() const override { return "TwoStrComparator"; }
virtual int Compare(const Slice& a, const Slice& b) const override {
assert(a.size() >= 2);
assert(b.size() >= 2);
size_t size_a1 = static_cast<size_t>(a[0]);
size_t size_b1 = static_cast<size_t>(b[0]);
size_t size_a2 = static_cast<size_t>(a[1]);
size_t size_b2 = static_cast<size_t>(b[1]);
assert(size_a1 + size_a2 + 2 == a.size());
assert(size_b1 + size_b2 + 2 == b.size());
Slice a1 = Slice(a.data() + 2, size_a1);
Slice b1 = Slice(b.data() + 2, size_b1);
Slice a2 = Slice(a.data() + 2 + size_a1, size_a2);
Slice b2 = Slice(b.data() + 2 + size_b1, size_b2);
if (a1 != b1) {
return a1.compare(b1);
}
return a2.compare(b2);
}
virtual void FindShortestSeparator(std::string* start,
const Slice& limit) const override {}
virtual void FindShortSuccessor(std::string* key) const override {}
};
} // namespace
class ComparatorDBTest : public testing::Test {
private:
std::string dbname_;
Env* env_;
DB* db_;
Options last_options_;
std::unique_ptr<const Comparator> comparator_guard;
public:
ComparatorDBTest() : env_(Env::Default()), db_(nullptr) {
comparator = BytewiseComparator();
dbname_ = test::TmpDir() + "/comparator_db_test";
EXPECT_OK(DestroyDB(dbname_, last_options_));
}
~ComparatorDBTest() {
delete db_;
EXPECT_OK(DestroyDB(dbname_, last_options_));
comparator = BytewiseComparator();
}
DB* GetDB() { return db_; }
void SetOwnedComparator(const Comparator* cmp) {
comparator_guard.reset(cmp);
comparator = cmp;
last_options_.comparator = cmp;
}
// Return the current option configuration.
Options* GetOptions() { return &last_options_; }
void DestroyAndReopen() {
// Destroy using last options
Destroy();
ASSERT_OK(TryReopen());
}
void Destroy() {
delete db_;
db_ = nullptr;
ASSERT_OK(DestroyDB(dbname_, last_options_));
}
Status TryReopen() {
delete db_;
db_ = nullptr;
last_options_.create_if_missing = true;
return DB::Open(last_options_, dbname_, &db_);
}
};
TEST_F(ComparatorDBTest, Bytewise) {
for (int rand_seed = 301; rand_seed < 306; rand_seed++) {
DestroyAndReopen();
Random rnd(rand_seed);
DoRandomIteraratorTest(GetDB(),
{"a", "b", "c", "d", "e", "f", "g", "h", "i"}, &rnd,
8, 100, 3);
}
}
TEST_F(ComparatorDBTest, SimpleSuffixReverseComparator) {
SetOwnedComparator(new test::SimpleSuffixReverseComparator());
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
Options* opt = GetOptions();
opt->comparator = comparator;
DestroyAndReopen();
Random rnd(rnd_seed);
std::vector<std::string> source_strings;
std::vector<std::string> source_prefixes;
// Randomly generate 5 prefixes
for (int i = 0; i < 5; i++) {
source_prefixes.push_back(test::RandomHumanReadableString(&rnd, 8));
}
for (int j = 0; j < 20; j++) {
int prefix_index = rnd.Uniform(static_cast<int>(source_prefixes.size()));
std::string key = source_prefixes[prefix_index] +
test::RandomHumanReadableString(&rnd, rnd.Uniform(8));
source_strings.push_back(key);
}
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 30, 600, 66);
}
}
TEST_F(ComparatorDBTest, Uint64Comparator) {
SetOwnedComparator(test::Uint64Comparator());
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
Options* opt = GetOptions();
opt->comparator = comparator;
DestroyAndReopen();
Random rnd(rnd_seed);
Random64 rnd64(rnd_seed);
std::vector<std::string> source_strings;
// Randomly generate source keys
for (int i = 0; i < 100; i++) {
uint64_t r = rnd64.Next();
std::string str;
str.resize(8);
memcpy(&str[0], static_cast<void*>(&r), 8);
source_strings.push_back(str);
}
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
}
}
TEST_F(ComparatorDBTest, DoubleComparator) {
SetOwnedComparator(new DoubleComparator());
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
Options* opt = GetOptions();
opt->comparator = comparator;
DestroyAndReopen();
Random rnd(rnd_seed);
std::vector<std::string> source_strings;
// Randomly generate source keys
for (int i = 0; i < 100; i++) {
uint32_t r = rnd.Next();
uint32_t divide_order = rnd.Uniform(8);
double to_divide = 1.0;
for (uint32_t j = 0; j < divide_order; j++) {
to_divide *= 10.0;
}
source_strings.push_back(ToString(r / to_divide));
}
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
}
}
TEST_F(ComparatorDBTest, HashComparator) {
SetOwnedComparator(new HashComparator());
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
Options* opt = GetOptions();
opt->comparator = comparator;
DestroyAndReopen();
Random rnd(rnd_seed);
std::vector<std::string> source_strings;
// Randomly generate source keys
for (int i = 0; i < 100; i++) {
source_strings.push_back(test::RandomKey(&rnd, 8));
}
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
}
}
TEST_F(ComparatorDBTest, TwoStrComparator) {
SetOwnedComparator(new TwoStrComparator());
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
Options* opt = GetOptions();
opt->comparator = comparator;
DestroyAndReopen();
Random rnd(rnd_seed);
std::vector<std::string> source_strings;
// Randomly generate source keys
for (int i = 0; i < 100; i++) {
std::string str;
uint32_t size1 = rnd.Uniform(8);
uint32_t size2 = rnd.Uniform(8);
str.append(1, static_cast<char>(size1));
str.append(1, static_cast<char>(size2));
str.append(test::RandomKey(&rnd, size1));
str.append(test::RandomKey(&rnd, size2));
source_strings.push_back(str);
}
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
}
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

58
db/convenience.cc Normal file
View File

@@ -0,0 +1,58 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
#ifndef ROCKSDB_LITE
#include "rocksdb/convenience.h"
#include "db/db_impl.h"
#include "util/cast_util.h"
namespace rocksdb {
void CancelAllBackgroundWork(DB* db, bool wait) {
(static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
->CancelAllBackgroundWork(wait);
}
Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
const Slice* begin, const Slice* end) {
return (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
->DeleteFilesInRange(column_family, begin, end);
}
Status VerifySstFileChecksum(const Options& options,
const EnvOptions& env_options,
const std::string& file_path) {
unique_ptr<RandomAccessFile> file;
uint64_t file_size;
InternalKeyComparator internal_comparator(options.comparator);
ImmutableCFOptions ioptions(options);
Status s = ioptions.env->NewRandomAccessFile(file_path, &file, env_options);
if (s.ok()) {
s = ioptions.env->GetFileSize(file_path, &file_size);
} else {
return s;
}
unique_ptr<TableReader> table_reader;
std::unique_ptr<RandomAccessFileReader> file_reader(
new RandomAccessFileReader(std::move(file), file_path));
s = ioptions.table_factory->NewTableReader(
TableReaderOptions(ioptions, env_options, internal_comparator,
false /* skip_filters */, -1 /* level */),
std::move(file_reader), file_size, &table_reader,
false /* prefetch_index_and_filter_in_cache */);
if (!s.ok()) {
return s;
}
s = table_reader->VerifyChecksum();
return s;
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

Some files were not shown because too many files have changed in this diff Show More