Introduced a new flag non_blocking_io in ReadOptions.

Summary:
If ReadOptions.non_blocking_io is set to true, then KeyMayExists
and Iterators will return data that is cached in RAM.
If the Iterator needs to do IO from storage to serve the data,
then the Iterator.status() will return Status::IsRetry().

Test Plan:
Enhanced unit test DBTest.KeyMayExist to detect if there were are IOs
issues from storage. Added DBTest.NonBlockingIteration to verify
nonblocking Iterations.

Reviewers: emayanke, haobo

Reviewed By: haobo

CC: leveldb

Maniphest Tasks: T63

Differential Revision: https://reviews.facebook.net/D12531

include/rocksdb/iterator.h

@@ -65,6 +65,8 @@ class Iterator {
   virtual Slice value() const = 0;
 
   // If an error has occurred, return it.  Else return an ok status.
+  // If non-blocking IO is requested and this operation cannot be
+  // satisfied without doing some IO, then this returns Status::Incomplete().
   virtual Status status() const = 0;
 
   // Clients are allowed to register function/arg1/arg2 triples that

include/rocksdb/options.h

@@ -543,6 +543,18 @@ struct Options {
   std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;
 };
 
+//
+// An application can issue a read request (via Get/Iterators) and specify
+// if that read should process data that ALREADY resides on a specified cache
+// level. For example, if an application specifies kBlockCacheTier then the
+// Get call will process data that is already processed in the memtable or
+// the block cache. It will not page in data from the OS cache or data that
+// resides in storage.
+enum ReadTier {
+  kReadAllTier    = 0x0, // data in memtable, block cache, OS cache or storage
+  kBlockCacheTier = 0x1  // data in memtable or block cache
+};
+
 // Options that control read operations
 struct ReadOptions {
   // If true, all data read from underlying storage will be
@@ -575,15 +587,23 @@ struct ReadOptions {
   // Default: nullptr
   const Slice* prefix;
 
+  // Specify if this read request should process data that ALREADY
+  // resides on a particular cache. If the required data is not
+  // found at the specified cache, then Status::WouldBlock is returned.
+  // Default: kReadAllTier
+  ReadTier read_tier;
+
   ReadOptions()
       : verify_checksums(false),
         fill_cache(true),
         snapshot(nullptr),
-        prefix(nullptr) {
+        prefix(nullptr),
+        read_tier(kReadAllTier) {
   }
   ReadOptions(bool cksum, bool cache) :
               verify_checksums(cksum), fill_cache(cache),
-              snapshot(nullptr), prefix(nullptr) {
+              snapshot(nullptr), prefix(nullptr),
+              read_tier(kReadAllTier) {
   }
 };
 

include/rocksdb/status.h

@@ -50,6 +50,9 @@ class Status {
   static Status MergeInProgress(const Slice& msg, const Slice& msg2 = Slice()) {
     return Status(kMergeInProgress, msg, msg2);
   }
+  static Status Incomplete(const Slice& msg, const Slice& msg2 = Slice()) {
+    return Status(kIncomplete, msg, msg2);
+  }
 
   // Returns true iff the status indicates success.
   bool ok() const { return (state_ == nullptr); }
@@ -72,6 +75,9 @@ class Status {
   // Returns true iff the status indicates an MergeInProgress.
   bool IsMergeInProgress() const { return code() == kMergeInProgress; }
 
+  // Returns true iff the status indicates Incomplete
+  bool IsIncomplete() const { return code() == kIncomplete; }
+
   // Return a string representation of this status suitable for printing.
   // Returns the string "OK" for success.
   std::string ToString() const;
@@ -92,6 +98,7 @@ class Status {
     kInvalidArgument = 4,
     kIOError = 5,
     kMergeInProgress = 6,
+    kIncomplete = 7
   };
 
   Code code() const {