From 2a2506b62980ff74c54dad6f8073f84d6df31f6d Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Fri, 13 Dec 2013 23:58:18 -0800 Subject: [PATCH 1/3] C bindings: add a bunch of the newer options --- db/c.cc | 151 +++++++++++++++++++++++++++++++++++++++++++- include/rocksdb/c.h | 62 +++++++++++++++++- 2 files changed, 209 insertions(+), 4 deletions(-) diff --git a/db/c.cc b/db/c.cc index 021122301b..36ee2d486b 100644 --- a/db/c.cc +++ b/db/c.cc @@ -20,6 +20,8 @@ #include "rocksdb/options.h" #include "rocksdb/status.h" #include "rocksdb/write_batch.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/universal_compaction.h" using rocksdb::Cache; using rocksdb::Comparator; @@ -134,6 +136,11 @@ struct rocksdb_env_t { bool is_default; }; +struct rocksdb_universal_compaction_options_t { + rocksdb::CompactionOptionsUniversal *rep; +}; + + static bool SaveError(char** errptr, const Status& s) { assert(errptr != NULL); if (s.ok()) { @@ -531,12 +538,12 @@ void rocksdb_options_set_compression_options( } void rocksdb_options_set_disable_data_sync( - rocksdb_options_t* opt, bool disable_data_sync) { + rocksdb_options_t* opt, int disable_data_sync) { opt->rep.disableDataSync = disable_data_sync; } void rocksdb_options_set_use_fsync( - rocksdb_options_t* opt, bool use_fsync) { + rocksdb_options_t* opt, int use_fsync) { opt->rep.use_fsync = use_fsync; } @@ -559,6 +566,95 @@ void rocksdb_options_set_WAL_size_limit_MB( opt->rep.WAL_size_limit_MB = limit; } +void rocksdb_options_set_max_write_buffer_number(rocksdb_options_t* opt, int n) { + opt->rep.max_write_buffer_number = n; +} + +void rocksdb_options_set_min_write_buffer_number_to_merge(rocksdb_options_t* opt, int n) { + opt->rep.min_write_buffer_number_to_merge = n; +} + +void rocksdb_options_set_max_background_compactions(rocksdb_options_t* opt, int n) { + opt->rep.max_background_compactions = n; +} + +void rocksdb_options_set_max_background_flushes(rocksdb_options_t* opt, int n) { + opt->rep.max_background_flushes = n; +} + +void rocksdb_options_set_disable_auto_compactions(rocksdb_options_t* opt, int disable) { + opt->rep.disable_auto_compactions = disable; +} + +void rocksdb_options_set_disable_seek_compaction(rocksdb_options_t* opt, int disable) { + opt->rep.disable_seek_compaction = disable; +} + +void rocksdb_options_set_source_compaction_factor( + rocksdb_options_t* opt, int n) { + opt->rep.expanded_compaction_factor = n; +} + +void rocksdb_options_prepare_for_bulk_load(rocksdb_options_t* opt) { + opt->rep.PrepareForBulkLoad(); +} + +void rocksdb_options_set_memtable_vector_rep(rocksdb_options_t *opt) { + static rocksdb::VectorRepFactory* factory = 0; + if (!factory) { + factory = new rocksdb::VectorRepFactory; + } + opt->rep.memtable_factory.reset(factory); +} + +void rocksdb_options_set_compaction_style(rocksdb_options_t *opt, int style) { + opt->rep.compaction_style = static_cast(style); +} + +void rocksdb_options_set_universal_compaction_options(rocksdb_options_t *opt, rocksdb_universal_compaction_options_t *uco) { + opt->rep.compaction_options_universal = *(uco->rep); +} + +/* +TODO: +merge_operator +compaction_filter +prefix_extractor +whole_key_filtering +max_bytes_for_level_multiplier_additional +delete_obsolete_files_period_micros +max_log_file_size +log_file_time_to_roll +keep_log_file_num +soft_rate_limit +hard_rate_limit +rate_limit_delay_max_milliseconds +max_manifest_file_size +no_block_cache +table_cache_numshardbits +table_cache_remove_scan_count_limit +arena_block_size +manifest_preallocation_size +purge_redundant_kvs_while_flush +allow_os_buffer +allow_mmap_reads +allow_mmap_writes +is_fd_close_on_exec +skip_log_error_on_recovery +stats_dump_period_sec +block_size_deviation +advise_random_on_open +access_hint_on_compaction_start +use_adaptive_mutex +bytes_per_sync +filter_deletes +max_sequential_skip_in_iterations +table_factory +table_properties_collectors +inplace_update_support +inplace_update_num_locks +*/ + rocksdb_comparator_t* rocksdb_comparator_create( void* state, void (*destructor)(void*), @@ -666,6 +762,11 @@ void rocksdb_writeoptions_set_sync( opt->rep.sync = v; } +void rocksdb_writeoptions_disable_WAL(rocksdb_writeoptions_t* opt, int disable) { + opt->rep.disableWAL = disable; +} + + rocksdb_cache_t* rocksdb_cache_create_lru(size_t capacity) { rocksdb_cache_t* c = new rocksdb_cache_t; c->rep = NewLRUCache(capacity); @@ -683,9 +784,55 @@ rocksdb_env_t* rocksdb_create_default_env() { return result; } +void rocksdb_env_set_background_threads(rocksdb_env_t* env, int n) { + env->rep->SetBackgroundThreads(n); +} + void rocksdb_env_destroy(rocksdb_env_t* env) { if (!env->is_default) delete env->rep; delete env; } +rocksdb_universal_compaction_options_t* rocksdb_universal_compaction_options_create() { + rocksdb_universal_compaction_options_t* result = new rocksdb_universal_compaction_options_t; + result->rep = new rocksdb::CompactionOptionsUniversal; + return result; +} + +void rocksdb_universal_compaction_options_set_size_ratio( + rocksdb_universal_compaction_options_t* uco, int ratio) { + uco->rep->size_ratio = ratio; +} + +void rocksdb_universal_compaction_options_set_min_merge_width( + rocksdb_universal_compaction_options_t* uco, int w) { + uco->rep->min_merge_width = w; +} + +void rocksdb_universal_compaction_options_set_max_merge_width( + rocksdb_universal_compaction_options_t* uco, int w) { + uco->rep->max_merge_width = w; +} + +void rocksdb_universal_compaction_options_set_max_size_amplification_percent( + rocksdb_universal_compaction_options_t* uco, int p) { + uco->rep->max_size_amplification_percent = p; +} + +void rocksdb_universal_compaction_options_set_compression_size_percent( + rocksdb_universal_compaction_options_t* uco, int p) { + uco->rep->compression_size_percent = p; +} + +void rocksdb_universal_compaction_options_set_stop_style( + rocksdb_universal_compaction_options_t* uco, int style) { + uco->rep->stop_style = static_cast(style); +} + +void rocksdb_universal_compaction_options_destroy( + rocksdb_universal_compaction_options_t* uco) { + delete uco->rep; + delete uco; +} + } // end extern "C" diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index e093e9448d..a3b18084a8 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -70,6 +70,7 @@ typedef struct rocksdb_snapshot_t rocksdb_snapshot_t; typedef struct rocksdb_writablefile_t rocksdb_writablefile_t; typedef struct rocksdb_writebatch_t rocksdb_writebatch_t; typedef struct rocksdb_writeoptions_t rocksdb_writeoptions_t; +typedef struct rocksdb_universal_compaction_options_t rocksdb_universal_compaction_options_t; /* DB operations */ @@ -208,14 +209,46 @@ extern void rocksdb_options_set_cache(rocksdb_options_t*, rocksdb_cache_t*); extern void rocksdb_options_set_block_size(rocksdb_options_t*, size_t); extern void rocksdb_options_set_block_restart_interval(rocksdb_options_t*, int); extern void rocksdb_options_set_compression_options( - rocksdb_options_t* opt, int w_bits, int level, int strategy); + rocksdb_options_t*, int, int, int); +extern void rocksdb_options_set_num_levels(rocksdb_options_t*, int); +extern void rocksdb_options_set_level0_file_num_compaction_trigger( + rocksdb_options_t*, int); +extern void rocksdb_options_set_level0_slowdown_writes_trigger( + rocksdb_options_t*, int); +extern void rocksdb_options_set_level0_stop_writes_trigger( + rocksdb_options_t*, int); +extern void rocksdb_options_set_target_file_size_base( + rocksdb_options_t*, uint64_t); +extern void rocksdb_options_set_target_file_size_multiplier( + rocksdb_options_t*, int); +extern void rocksdb_options_set_max_write_buffer_number(rocksdb_options_t*, int); +extern void rocksdb_options_set_min_write_buffer_number_to_merge(rocksdb_options_t*, int); +extern void rocksdb_options_set_max_background_compactions(rocksdb_options_t*, int); +extern void rocksdb_options_set_max_background_flushes(rocksdb_options_t*, int); +extern void rocksdb_options_set_use_fsync( + rocksdb_options_t*, int); +extern void rocksdb_options_set_disable_data_sync(rocksdb_options_t*, int); +extern void rocksdb_options_set_disable_auto_compactions(rocksdb_options_t*, int); +extern void rocksdb_options_set_disable_seek_compaction(rocksdb_options_t*, int); +extern void rocksdb_options_set_source_compaction_factor(rocksdb_options_t*, int); +extern void rocksdb_options_prepare_for_bulk_load(rocksdb_options_t*); +extern void rocksdb_options_set_memtable_vector_rep(rocksdb_options_t*); + enum { rocksdb_no_compression = 0, - rocksdb_snappy_compression = 1 + rocksdb_snappy_compression = 1, + rocksdb_zlib_compression = 1, + rocksdb_bz2_compression = 1 }; extern void rocksdb_options_set_compression(rocksdb_options_t*, int); +enum { + rocksdb_level_compaction = 0, + rocksdb_universal_compaction = 1 +}; +extern void rocksdb_options_set_compaction_style(rocksdb_options_t*, int); +extern void rocksdb_options_set_universal_compaction_options(rocksdb_options_t*, rocksdb_universal_compaction_options_t*); /* Comparator */ extern rocksdb_comparator_t* rocksdb_comparator_create( @@ -267,6 +300,7 @@ extern rocksdb_writeoptions_t* rocksdb_writeoptions_create(); extern void rocksdb_writeoptions_destroy(rocksdb_writeoptions_t*); extern void rocksdb_writeoptions_set_sync( rocksdb_writeoptions_t*, unsigned char); +extern void rocksdb_writeoptions_disable_WAL(rocksdb_writeoptions_t* opt, int disable); /* Cache */ @@ -276,8 +310,32 @@ extern void rocksdb_cache_destroy(rocksdb_cache_t* cache); /* Env */ extern rocksdb_env_t* rocksdb_create_default_env(); +extern void rocksdb_env_set_background_threads(rocksdb_env_t* env, int n); extern void rocksdb_env_destroy(rocksdb_env_t*); +/* Universal Compaction options */ + +enum { + rocksdb_similar_size_compaction_stop_style = 0, + rocksdb_total_size_compaction_stop_style = 1 +}; + +extern rocksdb_universal_compaction_options_t* rocksdb_universal_compaction_options_create() ; +extern void rocksdb_universal_compaction_options_set_size_ratio( + rocksdb_universal_compaction_options_t*, int); +extern void rocksdb_universal_compaction_options_set_min_merge_width( + rocksdb_universal_compaction_options_t*, int); +extern void rocksdb_universal_compaction_options_set_max_merge_width( + rocksdb_universal_compaction_options_t*, int); +extern void rocksdb_universal_compaction_options_set_max_size_amplification_percent( + rocksdb_universal_compaction_options_t*, int); +extern void rocksdb_universal_compaction_options_set_compression_size_percent( + rocksdb_universal_compaction_options_t*, int); +extern void rocksdb_universal_compaction_options_set_stop_style( + rocksdb_universal_compaction_options_t*, int); +extern void rocksdb_universal_compaction_options_destroy( + rocksdb_universal_compaction_options_t*); + #ifdef __cplusplus } /* end extern "C" */ #endif From 8c34189f0c59fbf0a32882219af3b33d4b21c278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20=C5=A0im=C3=A1nek?= Date: Sat, 14 Dec 2013 04:11:32 +0100 Subject: [PATCH 2/3] Remove .DS_Store files. --- utilities/.DS_Store | Bin 6148 -> 0 bytes utilities/merge_operators/.DS_Store | Bin 6148 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 utilities/.DS_Store delete mode 100644 utilities/merge_operators/.DS_Store diff --git a/utilities/.DS_Store b/utilities/.DS_Store deleted file mode 100644 index daeccc094b20ae294ebecc3d166c64a4b84cbe49..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKyG{c^3>-s>NED=`++W}iR#EtZd;lPU(m;X^LVXqA#iucT20^|Y0#ZNoaB2iHaNC7GEt$=?Y8r`uMj*0Q< zV2BZbxL`Vr>zE~o%@f34I3_Yfv!oJ}YBge5(wT2n*9*tQq{C|Xu)5i5Lb14==eH<_ z^+ZJ}AO)rhoac7s{r`sk!~8!bX(t7wz`s(!7Mty6%~z`4I(s?qwT*s9_nJ?-8`nW$ mh;~ejcFc{p?}cMx(3uZ9Q9lE&i%bgqwE|z3TNl3o diff --git a/utilities/merge_operators/.DS_Store b/utilities/merge_operators/.DS_Store deleted file mode 100644 index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Wed, 18 Dec 2013 13:37:06 -0800 Subject: [PATCH 3/3] Reorder tests Summary: db_test should be the first to execute because it finds the most bugs. Also, when third parties report issues, we don't want ldb error message, we prefer to have db_test error message. For example, see thread: https://github.com/facebook/rocksdb/issues/25 Test Plan: make check Reviewers: dhruba, haobo, kailiu Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D14715 --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 6200144c1e..0b113c1b59 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,7 @@ VALGRIND_VER := $(join $(VALGRIND_VER),valgrind) VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full TESTS = \ + db_test \ table_properties_collector_test \ arena_test \ auto_roll_logger_test \ @@ -81,8 +82,7 @@ TESTS = \ version_set_test \ write_batch_test\ deletefile_test \ - table_test \ - db_test + table_test TOOLS = \ sst_dump \ @@ -147,8 +147,9 @@ coverage: # Delete intermediate files find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \; -check: all $(PROGRAMS) $(TESTS) $(TOOLS) ldb_tests +check: all $(PROGRAMS) $(TESTS) $(TOOLS) for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done + python tools/ldb_test.py ldb_tests: all $(PROGRAMS) $(TOOLS) python tools/ldb_test.py