Ability to configure bufferedio-reads, filesystem-readaheads and mmap-read-write per database.

Summary:
This patch allows an application to specify whether to use bufferedio,
reads-via-mmaps and writes-via-mmaps per database. Earlier, there
was a global static variable that was used to configure this functionality.

The default setting remains the same (and is backward compatible):
 1. use bufferedio
 2. do not use mmaps for reads
 3. use mmap for writes
 4. use readaheads for reads needed for compaction

I also added a parameter to db_bench to be able to explicitly specify
whether to do readaheads for compactions or not.

Test Plan: make check

Reviewers: sheki, heyongqiang, MarkCallaghan

Reviewed By: sheki

CC: leveldb

Differential Revision: https://reviews.facebook.net/D9429

table/table.cc

@@ -29,8 +29,12 @@ struct Table::Rep {
     delete [] filter_data;
     delete index_block;
   }
+  Rep(const EnvOptions& storage_options) :
+    soptions(storage_options) {
+  }
 
   Options options;
+  const EnvOptions& soptions;
   Status status;
   unique_ptr<RandomAccessFile> file;
   char cache_key_prefix[kMaxCacheKeyPrefixSize];
@@ -62,6 +66,7 @@ void Table::SetupCacheKeyPrefix(Rep* rep) {
 }
 
 Status Table::Open(const Options& options,
+                   const EnvOptions& soptions,
                    unique_ptr<RandomAccessFile>&& file,
                    uint64_t size,
                    unique_ptr<Table>* table) {
@@ -100,7 +105,7 @@ Status Table::Open(const Options& options,
   if (s.ok()) {
     // We've successfully read the footer and the index block: we're
     // ready to serve requests.
-    Rep* rep = new Table::Rep;
+    Rep* rep = new Table::Rep(soptions);
     rep->options = options;
     rep->file = std::move(file);
     rep->metaindex_handle = footer.metaindex_handle();
@@ -260,6 +265,7 @@ Iterator* Table::BlockReader(void* arg,
 
 Iterator* Table::BlockReader(void* arg,
                              const ReadOptions& options,
+                             const EnvOptions& soptions,
                              const Slice& index_value) {
   return BlockReader(arg, options, index_value, nullptr);
 }
@@ -267,7 +273,7 @@ Iterator* Table::BlockReader(void* arg,
 Iterator* Table::NewIterator(const ReadOptions& options) const {
   return NewTwoLevelIterator(
       rep_->index_block->NewIterator(rep_->options.comparator),
-      &Table::BlockReader, const_cast<Table*>(this), options);
+      &Table::BlockReader, const_cast<Table*>(this), options, rep_->soptions);
 }
 
 Status Table::InternalGet(const ReadOptions& options, const Slice& k,

table/table_test.cc

@@ -257,7 +257,7 @@ class TableConstructor: public Constructor {
     // Open the table
     uniq_id_ = cur_uniq_id_++;
     source_.reset(new StringSource(sink_->contents(), uniq_id_));
-    return Table::Open(options, std::move(source_),
+    return Table::Open(options, soptions, std::move(source_),
                        sink_->contents().size(), &table_);
   }
 
@@ -271,7 +271,7 @@ class TableConstructor: public Constructor {
 
   virtual Status Reopen(const Options& options) {
     source_.reset(new StringSource(sink_->contents(), uniq_id_));
-    return Table::Open(options, std::move(source_),
+    return Table::Open(options, soptions, std::move(source_),
                        sink_->contents().size(), &table_);
   }
 
@@ -295,6 +295,7 @@ class TableConstructor: public Constructor {
   TableConstructor();
 
   static uint64_t cur_uniq_id_;
+  const StorageOptions soptions;
 };
 uint64_t TableConstructor::cur_uniq_id_ = 1;
 

table/two_level_iterator.cc

@@ -13,7 +13,8 @@ namespace leveldb {
 
 namespace {
 
-typedef Iterator* (*BlockFunction)(void*, const ReadOptions&, const Slice&);
+typedef Iterator* (*BlockFunction)(void*, const ReadOptions&,
+                                   const EnvOptions& soptions, const Slice&);
 
 class TwoLevelIterator: public Iterator {
  public:
@@ -21,7 +22,8 @@ class TwoLevelIterator: public Iterator {
     Iterator* index_iter,
     BlockFunction block_function,
     void* arg,
-    const ReadOptions& options);
+    const ReadOptions& options,
+    const EnvOptions& soptions);
 
   virtual ~TwoLevelIterator();
 
@@ -65,6 +67,7 @@ class TwoLevelIterator: public Iterator {
   BlockFunction block_function_;
   void* arg_;
   const ReadOptions options_;
+  const EnvOptions& soptions_;
   Status status_;
   IteratorWrapper index_iter_;
   IteratorWrapper data_iter_; // May be nullptr
@@ -77,10 +80,12 @@ TwoLevelIterator::TwoLevelIterator(
     Iterator* index_iter,
     BlockFunction block_function,
     void* arg,
-    const ReadOptions& options)
+    const ReadOptions& options,
+    const EnvOptions& soptions)
     : block_function_(block_function),
       arg_(arg),
       options_(options),
+      soptions_(soptions),
       index_iter_(index_iter),
       data_iter_(nullptr) {
 }
@@ -163,7 +168,7 @@ void TwoLevelIterator::InitDataBlock() {
       // data_iter_ is already constructed with this iterator, so
       // no need to change anything
     } else {
-      Iterator* iter = (*block_function_)(arg_, options_, handle);
+      Iterator* iter = (*block_function_)(arg_, options_, soptions_, handle);
       data_block_handle_.assign(handle.data(), handle.size());
       SetDataIterator(iter);
     }
@@ -176,8 +181,10 @@ Iterator* NewTwoLevelIterator(
     Iterator* index_iter,
     BlockFunction block_function,
     void* arg,
-    const ReadOptions& options) {
-  return new TwoLevelIterator(index_iter, block_function, arg, options);
+    const ReadOptions& options,
+    const EnvOptions& soptions) {
+  return new TwoLevelIterator(index_iter, block_function, arg,
+                              options, soptions);
 }
 
 }  // namespace leveldb

table/two_level_iterator.h

@@ -6,6 +6,7 @@
 #define STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_
 
 #include "leveldb/iterator.h"
+#include "leveldb/env.h"
 
 namespace leveldb {
 
@@ -25,9 +26,11 @@ extern Iterator* NewTwoLevelIterator(
     Iterator* (*block_function)(
         void* arg,
         const ReadOptions& options,
+        const EnvOptions& soptions,
         const Slice& index_value),
     void* arg,
-    const ReadOptions& options);
+    const ReadOptions& options,
+    const EnvOptions& soptions);
 
 }  // namespace leveldb