Do not spin in a tight loop attempting compactions if there is a compaction error

Summary: as subject. ported the change from google code leveldb 1.5

Test Plan: run db_test

Reviewers: dhruba

Differential Revision: https://reviews.facebook.net/D4839

db/db_impl.cc

@@ -700,8 +700,21 @@ void DBImpl::BackgroundCall() {
   MutexLock l(&mutex_);
   assert(bg_compaction_scheduled_);
   if (!shutting_down_.Acquire_Load()) {
-    BackgroundCompaction();
+    Status s = BackgroundCompaction();
+    if (!s.ok()) {
+      // Wait a little bit before retrying background compaction in
+      // case this is an environmental problem and we do not want to
+      // chew up resources for failed compactions for the duration of
+      // the problem.
+      bg_cv_.SignalAll();  // In case a waiter can proceed despite the error
+      Log(options_.info_log, "Waiting after background compaction error: %s",
+          s.ToString().c_str());
+      mutex_.Unlock();
+      env_->SleepForMicroseconds(1000000);
+      mutex_.Lock();
+    }
   }
+
   bg_compaction_scheduled_ = false;
 
   MaybeScheduleLogDBDeployStats();
@@ -712,12 +725,11 @@ void DBImpl::BackgroundCall() {
   bg_cv_.SignalAll();
 }
 
-void DBImpl::BackgroundCompaction() {
+Status DBImpl::BackgroundCompaction() {
   mutex_.AssertHeld();
 
   if (imm_ != NULL) {
-    CompactMemTable();
-    return;
+    return CompactMemTable();
   }
 
   Compaction* c;
@@ -792,6 +804,7 @@ void DBImpl::BackgroundCompaction() {
     }
     manual_compaction_ = NULL;
   }
+  return status;
 }
 
 void DBImpl::CleanupCompaction(CompactionState* compact) {
@@ -1368,6 +1381,7 @@ Status DBImpl::MakeRoomForWrite(bool force) {
     } else if (imm_ != NULL) {
       // We have filled up the current memtable, but the previous
       // one is still being compacted, so we wait.
+      Log(options_.info_log, "wait for memtable compaction...\n");
       bg_cv_.Wait();
     } else if (versions_->NumLevelFiles(0) >=
 		options_.level0_stop_writes_trigger) {