Squashed 'src/rocksdb/' content from commit 457bae6

git-subtree-dir: src/rocksdb
git-subtree-split: 457bae6911

utilities/backupable/backupable_db_test.cc

@@ -0,0 +1,974 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include <string>
+#include <algorithm>
+#include <iostream>
+
+#include "port/port.h"
+#include "rocksdb/types.h"
+#include "rocksdb/transaction_log.h"
+#include "utilities/utility_db.h"
+#include "utilities/backupable_db.h"
+#include "util/testharness.h"
+#include "util/random.h"
+#include "util/mutexlock.h"
+#include "util/testutil.h"
+#include "util/auto_roll_logger.h"
+
+namespace rocksdb {
+
+namespace {
+
+using std::unique_ptr;
+
+class DummyDB : public StackableDB {
+ public:
+  /* implicit */
+  DummyDB(const Options& options, const std::string& dbname)
+     : StackableDB(nullptr), options_(options), dbname_(dbname),
+       deletions_enabled_(true), sequence_number_(0) {}
+
+  virtual SequenceNumber GetLatestSequenceNumber() const {
+    return ++sequence_number_;
+  }
+
+  virtual const std::string& GetName() const override {
+    return dbname_;
+  }
+
+  virtual Env* GetEnv() const override {
+    return options_.env;
+  }
+
+  using DB::GetOptions;
+  virtual const Options& GetOptions(ColumnFamilyHandle* column_family) const
+      override {
+    return options_;
+  }
+
+  virtual Status EnableFileDeletions(bool force) override {
+    ASSERT_TRUE(!deletions_enabled_);
+    deletions_enabled_ = true;
+    return Status::OK();
+  }
+
+  virtual Status DisableFileDeletions() override {
+    ASSERT_TRUE(deletions_enabled_);
+    deletions_enabled_ = false;
+    return Status::OK();
+  }
+
+  virtual Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs,
+                              bool flush_memtable = true) override {
+    ASSERT_TRUE(!deletions_enabled_);
+    vec = live_files_;
+    *mfs = 100;
+    return Status::OK();
+  }
+
+  virtual ColumnFamilyHandle* DefaultColumnFamily() const override {
+    return nullptr;
+  }
+
+  class DummyLogFile : public LogFile {
+   public:
+    /* implicit */
+     DummyLogFile(const std::string& path, bool alive = true)
+         : path_(path), alive_(alive) {}
+
+    virtual std::string PathName() const override {
+      return path_;
+    }
+
+    virtual uint64_t LogNumber() const {
+      // what business do you have calling this method?
+      ASSERT_TRUE(false);
+      return 0;
+    }
+
+    virtual WalFileType Type() const override {
+      return alive_ ? kAliveLogFile : kArchivedLogFile;
+    }
+
+    virtual SequenceNumber StartSequence() const {
+      // backupabledb should not need this method
+      ASSERT_TRUE(false);
+      return 0;
+    }
+
+    virtual uint64_t SizeFileBytes() const {
+      // backupabledb should not need this method
+      ASSERT_TRUE(false);
+      return 0;
+    }
+
+   private:
+    std::string path_;
+    bool alive_;
+  }; // DummyLogFile
+
+  virtual Status GetSortedWalFiles(VectorLogPtr& files) override {
+    ASSERT_TRUE(!deletions_enabled_);
+    files.resize(wal_files_.size());
+    for (size_t i = 0; i < files.size(); ++i) {
+      files[i].reset(
+          new DummyLogFile(wal_files_[i].first, wal_files_[i].second));
+    }
+    return Status::OK();
+  }
+
+  std::vector<std::string> live_files_;
+  // pair<filename, alive?>
+  std::vector<std::pair<std::string, bool>> wal_files_;
+ private:
+  Options options_;
+  std::string dbname_;
+  bool deletions_enabled_;
+  mutable SequenceNumber sequence_number_;
+}; // DummyDB
+
+class TestEnv : public EnvWrapper {
+ public:
+  explicit TestEnv(Env* t) : EnvWrapper(t) {}
+
+  class DummySequentialFile : public SequentialFile {
+   public:
+    DummySequentialFile() : SequentialFile(), rnd_(5) {}
+    virtual Status Read(size_t n, Slice* result, char* scratch) {
+      size_t read_size = (n > size_left) ? size_left : n;
+      for (size_t i = 0; i < read_size; ++i) {
+        scratch[i] = rnd_.Next() & 255;
+      }
+      *result = Slice(scratch, read_size);
+      size_left -= read_size;
+      return Status::OK();
+    }
+
+    virtual Status Skip(uint64_t n) {
+      size_left = (n > size_left) ? size_left - n : 0;
+      return Status::OK();
+    }
+   private:
+    size_t size_left = 200;
+    Random rnd_;
+  };
+
+  Status NewSequentialFile(const std::string& f,
+                           unique_ptr<SequentialFile>* r,
+                           const EnvOptions& options) {
+    MutexLock l(&mutex_);
+    if (dummy_sequential_file_) {
+      r->reset(new TestEnv::DummySequentialFile());
+      return Status::OK();
+    } else {
+      return EnvWrapper::NewSequentialFile(f, r, options);
+    }
+  }
+
+  Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
+                         const EnvOptions& options) {
+    MutexLock l(&mutex_);
+    written_files_.push_back(f);
+    if (limit_written_files_ <= 0) {
+      return Status::NotSupported("Sorry, can't do this");
+    }
+    limit_written_files_--;
+    return EnvWrapper::NewWritableFile(f, r, options);
+  }
+
+  virtual Status DeleteFile(const std::string& fname) override {
+    MutexLock l(&mutex_);
+    ASSERT_GT(limit_delete_files_, 0U);
+    limit_delete_files_--;
+    return EnvWrapper::DeleteFile(fname);
+  }
+
+  void AssertWrittenFiles(std::vector<std::string>& should_have_written) {
+    MutexLock l(&mutex_);
+    sort(should_have_written.begin(), should_have_written.end());
+    sort(written_files_.begin(), written_files_.end());
+    ASSERT_TRUE(written_files_ == should_have_written);
+  }
+
+  void ClearWrittenFiles() {
+    MutexLock l(&mutex_);
+    written_files_.clear();
+  }
+
+  void SetLimitWrittenFiles(uint64_t limit) {
+    MutexLock l(&mutex_);
+    limit_written_files_ = limit;
+  }
+
+  void SetLimitDeleteFiles(uint64_t limit) {
+    MutexLock l(&mutex_);
+    limit_delete_files_ = limit;
+  }
+
+  void SetDummySequentialFile(bool dummy_sequential_file) {
+    MutexLock l(&mutex_);
+    dummy_sequential_file_ = dummy_sequential_file;
+  }
+
+ private:
+  port::Mutex mutex_;
+  bool dummy_sequential_file_ = false;
+  std::vector<std::string> written_files_;
+  uint64_t limit_written_files_ = 1000000;
+  uint64_t limit_delete_files_ = 1000000;
+};  // TestEnv
+
+class FileManager : public EnvWrapper {
+ public:
+  explicit FileManager(Env* t) : EnvWrapper(t), rnd_(5) {}
+
+  Status DeleteRandomFileInDir(const std::string dir) {
+    std::vector<std::string> children;
+    GetChildren(dir, &children);
+    if (children.size() <= 2) { // . and ..
+      return Status::NotFound("");
+    }
+    while (true) {
+      int i = rnd_.Next() % children.size();
+      if (children[i] != "." && children[i] != "..") {
+        return DeleteFile(dir + "/" + children[i]);
+      }
+    }
+    // should never get here
+    assert(false);
+    return Status::NotFound("");
+  }
+
+  Status CorruptFile(const std::string& fname, uint64_t bytes_to_corrupt) {
+    uint64_t size;
+    Status s = GetFileSize(fname, &size);
+    if (!s.ok()) {
+      return s;
+    }
+    unique_ptr<RandomRWFile> file;
+    EnvOptions env_options;
+    env_options.use_mmap_writes = false;
+    s = NewRandomRWFile(fname, &file, env_options);
+    if (!s.ok()) {
+      return s;
+    }
+
+    for (uint64_t i = 0; s.ok() && i < bytes_to_corrupt; ++i) {
+      std::string tmp;
+      // write one random byte to a random position
+      s = file->Write(rnd_.Next() % size, test::RandomString(&rnd_, 1, &tmp));
+    }
+    return s;
+  }
+
+  Status CorruptChecksum(const std::string& fname, bool appear_valid) {
+    std::string metadata;
+    Status s = ReadFileToString(this, fname, &metadata);
+    if (!s.ok()) {
+      return s;
+    }
+    s = DeleteFile(fname);
+    if (!s.ok()) {
+      return s;
+    }
+
+    auto pos = metadata.find("private");
+    if (pos == std::string::npos) {
+      return Status::Corruption("private file is expected");
+    }
+    pos = metadata.find(" crc32 ", pos + 6);
+    if (pos == std::string::npos) {
+      return Status::Corruption("checksum not found");
+    }
+
+    if (metadata.size() < pos + 7) {
+      return Status::Corruption("bad CRC32 checksum value");
+    }
+
+    if (appear_valid) {
+      if (metadata[pos + 8] == '\n') {
+        // single digit value, safe to insert one more digit
+        metadata.insert(pos + 8, 1, '0');
+      } else {
+        metadata.erase(pos + 8, 1);
+      }
+    } else {
+      metadata[pos + 7] = 'a';
+    }
+
+    return WriteToFile(fname, metadata);
+  }
+
+  Status WriteToFile(const std::string& fname, const std::string& data) {
+    unique_ptr<WritableFile> file;
+    EnvOptions env_options;
+    env_options.use_mmap_writes = false;
+    Status s = EnvWrapper::NewWritableFile(fname, &file, env_options);
+    if (!s.ok()) {
+      return s;
+    }
+    return file->Append(Slice(data));
+  }
+
+ private:
+  Random rnd_;
+}; // FileManager
+
+// utility functions
+static size_t FillDB(DB* db, int from, int to) {
+  size_t bytes_written = 0;
+  for (int i = from; i < to; ++i) {
+    std::string key = "testkey" + std::to_string(i);
+    std::string value = "testvalue" + std::to_string(i);
+    bytes_written += key.size() + value.size();
+
+    ASSERT_OK(db->Put(WriteOptions(), Slice(key), Slice(value)));
+  }
+  return bytes_written;
+}
+
+static void AssertExists(DB* db, int from, int to) {
+  for (int i = from; i < to; ++i) {
+    std::string key = "testkey" + std::to_string(i);
+    std::string value;
+    Status s = db->Get(ReadOptions(), Slice(key), &value);
+    ASSERT_EQ(value, "testvalue" + std::to_string(i));
+  }
+}
+
+static void AssertEmpty(DB* db, int from, int to) {
+  for (int i = from; i < to; ++i) {
+    std::string key = "testkey" + std::to_string(i);
+    std::string value = "testvalue" + std::to_string(i);
+
+    Status s = db->Get(ReadOptions(), Slice(key), &value);
+    ASSERT_TRUE(s.IsNotFound());
+  }
+}
+
+class BackupableDBTest {
+ public:
+  BackupableDBTest() {
+    // set up files
+    dbname_ = test::TmpDir() + "/backupable_db";
+    backupdir_ = test::TmpDir() + "/backupable_db_backup";
+
+    // set up envs
+    env_ = Env::Default();
+    test_db_env_.reset(new TestEnv(env_));
+    test_backup_env_.reset(new TestEnv(env_));
+    file_manager_.reset(new FileManager(env_));
+
+    // set up db options
+    options_.create_if_missing = true;
+    options_.paranoid_checks = true;
+    options_.write_buffer_size = 1 << 17; // 128KB
+    options_.env = test_db_env_.get();
+    options_.wal_dir = dbname_;
+    // set up backup db options
+    CreateLoggerFromOptions(dbname_, backupdir_, env_,
+                            DBOptions(), &logger_);
+    backupable_options_.reset(new BackupableDBOptions(
+        backupdir_, test_backup_env_.get(), true, logger_.get(), true));
+
+    // delete old files in db
+    DestroyDB(dbname_, Options());
+  }
+
+  DB* OpenDB() {
+    DB* db;
+    ASSERT_OK(DB::Open(options_, dbname_, &db));
+    return db;
+  }
+
+  void OpenBackupableDB(bool destroy_old_data = false, bool dummy = false,
+                        bool share_table_files = true,
+                        bool share_with_checksums = false) {
+    // reset all the defaults
+    test_backup_env_->SetLimitWrittenFiles(1000000);
+    test_db_env_->SetLimitWrittenFiles(1000000);
+    test_db_env_->SetDummySequentialFile(dummy);
+
+    DB* db;
+    if (dummy) {
+      dummy_db_ = new DummyDB(options_, dbname_);
+      db = dummy_db_;
+    } else {
+      ASSERT_OK(DB::Open(options_, dbname_, &db));
+    }
+    backupable_options_->destroy_old_data = destroy_old_data;
+    backupable_options_->share_table_files = share_table_files;
+    backupable_options_->share_files_with_checksum = share_with_checksums;
+    db_.reset(new BackupableDB(db, *backupable_options_));
+  }
+
+  void CloseBackupableDB() {
+    db_.reset(nullptr);
+  }
+
+  void OpenRestoreDB() {
+    backupable_options_->destroy_old_data = false;
+    restore_db_.reset(
+        new RestoreBackupableDB(test_db_env_.get(), *backupable_options_));
+  }
+
+  void CloseRestoreDB() {
+    restore_db_.reset(nullptr);
+  }
+
+  // restores backup backup_id and asserts the existence of
+  // [start_exist, end_exist> and not-existence of
+  // [end_exist, end>
+  //
+  // if backup_id == 0, it means restore from latest
+  // if end == 0, don't check AssertEmpty
+  void AssertBackupConsistency(BackupID backup_id, uint32_t start_exist,
+                               uint32_t end_exist, uint32_t end = 0,
+                               bool keep_log_files = false) {
+    RestoreOptions restore_options(keep_log_files);
+    bool opened_restore = false;
+    if (restore_db_.get() == nullptr) {
+      opened_restore = true;
+      OpenRestoreDB();
+    }
+    if (backup_id > 0) {
+      ASSERT_OK(restore_db_->RestoreDBFromBackup(backup_id, dbname_, dbname_,
+                                                 restore_options));
+    } else {
+      ASSERT_OK(restore_db_->RestoreDBFromLatestBackup(dbname_, dbname_,
+                                                       restore_options));
+    }
+    DB* db = OpenDB();
+    AssertExists(db, start_exist, end_exist);
+    if (end != 0) {
+      AssertEmpty(db, end_exist, end);
+    }
+    delete db;
+    if (opened_restore) {
+      CloseRestoreDB();
+    }
+  }
+
+  void DeleteLogFiles() {
+    std::vector<std::string> delete_logs;
+    env_->GetChildren(dbname_, &delete_logs);
+    for (auto f : delete_logs) {
+      uint64_t number;
+      FileType type;
+      bool ok = ParseFileName(f, &number, &type);
+      if (ok && type == kLogFile) {
+        env_->DeleteFile(dbname_ + "/" + f);
+      }
+    }
+  }
+
+  // files
+  std::string dbname_;
+  std::string backupdir_;
+
+  // envs
+  Env* env_;
+  unique_ptr<TestEnv> test_db_env_;
+  unique_ptr<TestEnv> test_backup_env_;
+  unique_ptr<FileManager> file_manager_;
+
+  // all the dbs!
+  DummyDB* dummy_db_; // BackupableDB owns dummy_db_
+  unique_ptr<BackupableDB> db_;
+  unique_ptr<RestoreBackupableDB> restore_db_;
+
+  // options
+  Options options_;
+  unique_ptr<BackupableDBOptions> backupable_options_;
+  std::shared_ptr<Logger> logger_;
+}; // BackupableDBTest
+
+void AppendPath(const std::string& path, std::vector<std::string>& v) {
+  for (auto& f : v) {
+    f = path + f;
+  }
+}
+
+// this will make sure that backup does not copy the same file twice
+TEST(BackupableDBTest, NoDoubleCopy) {
+  OpenBackupableDB(true, true);
+
+  // should write 5 DB files + LATEST_BACKUP + one meta file
+  test_backup_env_->SetLimitWrittenFiles(7);
+  test_backup_env_->ClearWrittenFiles();
+  test_db_env_->SetLimitWrittenFiles(0);
+  dummy_db_->live_files_ = { "/00010.sst", "/00011.sst",
+                             "/CURRENT",   "/MANIFEST-01" };
+  dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
+  ASSERT_OK(db_->CreateNewBackup(false));
+  std::vector<std::string> should_have_written = {
+    "/shared/00010.sst.tmp",
+    "/shared/00011.sst.tmp",
+    "/private/1.tmp/CURRENT",
+    "/private/1.tmp/MANIFEST-01",
+    "/private/1.tmp/00011.log",
+    "/meta/1.tmp",
+    "/LATEST_BACKUP.tmp"
+  };
+  AppendPath(dbname_ + "_backup", should_have_written);
+  test_backup_env_->AssertWrittenFiles(should_have_written);
+
+  // should write 4 new DB files + LATEST_BACKUP + one meta file
+  // should not write/copy 00010.sst, since it's already there!
+  test_backup_env_->SetLimitWrittenFiles(6);
+  test_backup_env_->ClearWrittenFiles();
+  dummy_db_->live_files_ = { "/00010.sst", "/00015.sst",
+                             "/CURRENT",   "/MANIFEST-01" };
+  dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
+  ASSERT_OK(db_->CreateNewBackup(false));
+  // should not open 00010.sst - it's already there
+  should_have_written = {
+    "/shared/00015.sst.tmp",
+    "/private/2.tmp/CURRENT",
+    "/private/2.tmp/MANIFEST-01",
+    "/private/2.tmp/00011.log",
+    "/meta/2.tmp",
+    "/LATEST_BACKUP.tmp"
+  };
+  AppendPath(dbname_ + "_backup", should_have_written);
+  test_backup_env_->AssertWrittenFiles(should_have_written);
+
+  ASSERT_OK(db_->DeleteBackup(1));
+  ASSERT_EQ(true,
+            test_backup_env_->FileExists(backupdir_ + "/shared/00010.sst"));
+  // 00011.sst was only in backup 1, should be deleted
+  ASSERT_EQ(false,
+            test_backup_env_->FileExists(backupdir_ + "/shared/00011.sst"));
+  ASSERT_EQ(true,
+            test_backup_env_->FileExists(backupdir_ + "/shared/00015.sst"));
+
+  // MANIFEST file size should be only 100
+  uint64_t size;
+  test_backup_env_->GetFileSize(backupdir_ + "/private/2/MANIFEST-01", &size);
+  ASSERT_EQ(100UL, size);
+  test_backup_env_->GetFileSize(backupdir_ + "/shared/00015.sst", &size);
+  ASSERT_EQ(200UL, size);
+
+  CloseBackupableDB();
+}
+
+// test various kind of corruptions that may happen:
+// 1. Not able to write a file for backup - that backup should fail,
+//      everything else should work
+// 2. Corrupted/deleted LATEST_BACKUP - everything should work fine
+// 3. Corrupted backup meta file or missing backuped file - we should
+//      not be able to open that backup, but all other backups should be
+//      fine
+// 4. Corrupted checksum value - if the checksum is not a valid uint32_t,
+//      db open should fail, otherwise, it aborts during the restore process.
+TEST(BackupableDBTest, CorruptionsTest) {
+  const int keys_iteration = 5000;
+  Random rnd(6);
+  Status s;
+
+  OpenBackupableDB(true);
+  // create five backups
+  for (int i = 0; i < 5; ++i) {
+    FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
+    ASSERT_OK(db_->CreateNewBackup(!!(rnd.Next() % 2)));
+  }
+
+  // ---------- case 1. - fail a write -----------
+  // try creating backup 6, but fail a write
+  FillDB(db_.get(), keys_iteration * 5, keys_iteration * 6);
+  test_backup_env_->SetLimitWrittenFiles(2);
+  // should fail
+  s = db_->CreateNewBackup(!!(rnd.Next() % 2));
+  ASSERT_TRUE(!s.ok());
+  test_backup_env_->SetLimitWrittenFiles(1000000);
+  // latest backup should have all the keys
+  CloseBackupableDB();
+  AssertBackupConsistency(0, 0, keys_iteration * 5, keys_iteration * 6);
+
+  // ---------- case 2. - corrupt/delete latest backup -----------
+  ASSERT_OK(file_manager_->CorruptFile(backupdir_ + "/LATEST_BACKUP", 2));
+  AssertBackupConsistency(0, 0, keys_iteration * 5);
+  ASSERT_OK(file_manager_->DeleteFile(backupdir_ + "/LATEST_BACKUP"));
+  AssertBackupConsistency(0, 0, keys_iteration * 5);
+  // create backup 6, point LATEST_BACKUP to 5
+  OpenBackupableDB();
+  FillDB(db_.get(), keys_iteration * 5, keys_iteration * 6);
+  ASSERT_OK(db_->CreateNewBackup(false));
+  CloseBackupableDB();
+  ASSERT_OK(file_manager_->WriteToFile(backupdir_ + "/LATEST_BACKUP", "5"));
+  AssertBackupConsistency(0, 0, keys_iteration * 5, keys_iteration * 6);
+  // assert that all 6 data is gone!
+  ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/6") == false);
+  ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/6") == false);
+
+  // --------- case 3. corrupted backup meta or missing backuped file ----
+  ASSERT_OK(file_manager_->CorruptFile(backupdir_ + "/meta/5", 3));
+  // since 5 meta is now corrupted, latest backup should be 4
+  AssertBackupConsistency(0, 0, keys_iteration * 4, keys_iteration * 5);
+  OpenRestoreDB();
+  s = restore_db_->RestoreDBFromBackup(5, dbname_, dbname_);
+  ASSERT_TRUE(!s.ok());
+  CloseRestoreDB();
+  ASSERT_OK(file_manager_->DeleteRandomFileInDir(backupdir_ + "/private/4"));
+  // 4 is corrupted, 3 is the latest backup now
+  AssertBackupConsistency(0, 0, keys_iteration * 3, keys_iteration * 5);
+  OpenRestoreDB();
+  s = restore_db_->RestoreDBFromBackup(4, dbname_, dbname_);
+  CloseRestoreDB();
+  ASSERT_TRUE(!s.ok());
+
+  // --------- case 4. corrupted checksum value ----
+  ASSERT_OK(file_manager_->CorruptChecksum(backupdir_ + "/meta/3", false));
+  // checksum of backup 3 is an invalid value, this can be detected at
+  // db open time, and it reverts to the previous backup automatically
+  AssertBackupConsistency(0, 0, keys_iteration * 2, keys_iteration * 5);
+  // checksum of the backup 2 appears to be valid, this can cause checksum
+  // mismatch and abort restore process
+  ASSERT_OK(file_manager_->CorruptChecksum(backupdir_ + "/meta/2", true));
+  ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/2"));
+  OpenRestoreDB();
+  ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/2"));
+  s = restore_db_->RestoreDBFromBackup(2, dbname_, dbname_);
+  ASSERT_TRUE(!s.ok());
+  ASSERT_OK(restore_db_->DeleteBackup(2));
+  CloseRestoreDB();
+  AssertBackupConsistency(0, 0, keys_iteration * 1, keys_iteration * 5);
+
+  // new backup should be 2!
+  OpenBackupableDB();
+  FillDB(db_.get(), keys_iteration * 1, keys_iteration * 2);
+  ASSERT_OK(db_->CreateNewBackup(!!(rnd.Next() % 2)));
+  CloseBackupableDB();
+  AssertBackupConsistency(2, 0, keys_iteration * 2, keys_iteration * 5);
+}
+
+// open DB, write, close DB, backup, restore, repeat
+TEST(BackupableDBTest, OfflineIntegrationTest) {
+  // has to be a big number, so that it triggers the memtable flush
+  const int keys_iteration = 5000;
+  const int max_key = keys_iteration * 4 + 10;
+  // first iter -- flush before backup
+  // second iter -- don't flush before backup
+  for (int iter = 0; iter < 2; ++iter) {
+    // delete old data
+    DestroyDB(dbname_, Options());
+    bool destroy_data = true;
+
+    // every iteration --
+    // 1. insert new data in the DB
+    // 2. backup the DB
+    // 3. destroy the db
+    // 4. restore the db, check everything is still there
+    for (int i = 0; i < 5; ++i) {
+      // in last iteration, put smaller amount of data,
+      int fill_up_to = std::min(keys_iteration * (i + 1), max_key);
+      // ---- insert new data and back up ----
+      OpenBackupableDB(destroy_data);
+      destroy_data = false;
+      FillDB(db_.get(), keys_iteration * i, fill_up_to);
+      ASSERT_OK(db_->CreateNewBackup(iter == 0));
+      CloseBackupableDB();
+      DestroyDB(dbname_, Options());
+
+      // ---- make sure it's empty ----
+      DB* db = OpenDB();
+      AssertEmpty(db, 0, fill_up_to);
+      delete db;
+
+      // ---- restore the DB ----
+      OpenRestoreDB();
+      if (i >= 3) { // test purge old backups
+        // when i == 4, purge to only 1 backup
+        // when i == 3, purge to 2 backups
+        ASSERT_OK(restore_db_->PurgeOldBackups(5 - i));
+      }
+      // ---- make sure the data is there ---
+      AssertBackupConsistency(0, 0, fill_up_to, max_key);
+      CloseRestoreDB();
+    }
+  }
+}
+
+// open DB, write, backup, write, backup, close, restore
+TEST(BackupableDBTest, OnlineIntegrationTest) {
+  // has to be a big number, so that it triggers the memtable flush
+  const int keys_iteration = 5000;
+  const int max_key = keys_iteration * 4 + 10;
+  Random rnd(7);
+  // delete old data
+  DestroyDB(dbname_, Options());
+
+  OpenBackupableDB(true);
+  // write some data, backup, repeat
+  for (int i = 0; i < 5; ++i) {
+    if (i == 4) {
+      // delete backup number 2, online delete!
+      OpenRestoreDB();
+      ASSERT_OK(restore_db_->DeleteBackup(2));
+      CloseRestoreDB();
+    }
+    // in last iteration, put smaller amount of data,
+    // so that backups can share sst files
+    int fill_up_to = std::min(keys_iteration * (i + 1), max_key);
+    FillDB(db_.get(), keys_iteration * i, fill_up_to);
+    // we should get consistent results with flush_before_backup
+    // set to both true and false
+    ASSERT_OK(db_->CreateNewBackup(!!(rnd.Next() % 2)));
+  }
+  // close and destroy
+  CloseBackupableDB();
+  DestroyDB(dbname_, Options());
+
+  // ---- make sure it's empty ----
+  DB* db = OpenDB();
+  AssertEmpty(db, 0, max_key);
+  delete db;
+
+  // ---- restore every backup and verify all the data is there ----
+  OpenRestoreDB();
+  for (int i = 1; i <= 5; ++i) {
+    if (i == 2) {
+      // we deleted backup 2
+      Status s = restore_db_->RestoreDBFromBackup(2, dbname_, dbname_);
+      ASSERT_TRUE(!s.ok());
+    } else {
+      int fill_up_to = std::min(keys_iteration * i, max_key);
+      AssertBackupConsistency(i, 0, fill_up_to, max_key);
+    }
+  }
+
+  // delete some backups -- this should leave only backups 3 and 5 alive
+  ASSERT_OK(restore_db_->DeleteBackup(4));
+  ASSERT_OK(restore_db_->PurgeOldBackups(2));
+
+  std::vector<BackupInfo> backup_info;
+  restore_db_->GetBackupInfo(&backup_info);
+  ASSERT_EQ(2UL, backup_info.size());
+
+  // check backup 3
+  AssertBackupConsistency(3, 0, 3 * keys_iteration, max_key);
+  // check backup 5
+  AssertBackupConsistency(5, 0, max_key);
+
+  CloseRestoreDB();
+}
+
+TEST(BackupableDBTest, FailOverwritingBackups) {
+  options_.write_buffer_size = 1024 * 1024 * 1024;  // 1GB
+  // create backups 1, 2, 3, 4, 5
+  OpenBackupableDB(true);
+  for (int i = 0; i < 5; ++i) {
+    CloseBackupableDB();
+    DeleteLogFiles();
+    OpenBackupableDB(false);
+    FillDB(db_.get(), 100 * i, 100 * (i + 1));
+    ASSERT_OK(db_->CreateNewBackup(true));
+  }
+  CloseBackupableDB();
+
+  // restore 3
+  OpenRestoreDB();
+  ASSERT_OK(restore_db_->RestoreDBFromBackup(3, dbname_, dbname_));
+  CloseRestoreDB();
+
+  OpenBackupableDB(false);
+  FillDB(db_.get(), 0, 300);
+  Status s = db_->CreateNewBackup(true);
+  // the new backup fails because new table files
+  // clash with old table files from backups 4 and 5
+  // (since write_buffer_size is huge, we can be sure that
+  // each backup will generate only one sst file and that
+  // a file generated by a new backup is the same as
+  // sst file generated by backup 4)
+  ASSERT_TRUE(s.IsCorruption());
+  ASSERT_OK(db_->DeleteBackup(4));
+  ASSERT_OK(db_->DeleteBackup(5));
+  // now, the backup can succeed
+  ASSERT_OK(db_->CreateNewBackup(true));
+  CloseBackupableDB();
+}
+
+TEST(BackupableDBTest, NoShareTableFiles) {
+  const int keys_iteration = 5000;
+  OpenBackupableDB(true, false, false);
+  for (int i = 0; i < 5; ++i) {
+    FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
+    ASSERT_OK(db_->CreateNewBackup(!!(i % 2)));
+  }
+  CloseBackupableDB();
+
+  for (int i = 0; i < 5; ++i) {
+    AssertBackupConsistency(i + 1, 0, keys_iteration * (i + 1),
+                            keys_iteration * 6);
+  }
+}
+
+// Verify that you can backup and restore with share_files_with_checksum on
+TEST(BackupableDBTest, ShareTableFilesWithChecksums) {
+  const int keys_iteration = 5000;
+  OpenBackupableDB(true, false, true, true);
+  for (int i = 0; i < 5; ++i) {
+    FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
+    ASSERT_OK(db_->CreateNewBackup(!!(i % 2)));
+  }
+  CloseBackupableDB();
+
+  for (int i = 0; i < 5; ++i) {
+    AssertBackupConsistency(i + 1, 0, keys_iteration * (i + 1),
+                            keys_iteration * 6);
+  }
+}
+
+// Verify that you can backup and restore using share_files_with_checksum set to
+// false and then transition this option to true
+TEST(BackupableDBTest, ShareTableFilesWithChecksumsTransition) {
+  const int keys_iteration = 5000;
+  // set share_files_with_checksum to false
+  OpenBackupableDB(true, false, true, false);
+  for (int i = 0; i < 5; ++i) {
+    FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
+    ASSERT_OK(db_->CreateNewBackup(true));
+  }
+  CloseBackupableDB();
+
+  for (int i = 0; i < 5; ++i) {
+    AssertBackupConsistency(i + 1, 0, keys_iteration * (i + 1),
+                            keys_iteration * 6);
+  }
+
+  // set share_files_with_checksum to true and do some more backups
+  OpenBackupableDB(true, false, true, true);
+  for (int i = 5; i < 10; ++i) {
+    FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
+    ASSERT_OK(db_->CreateNewBackup(true));
+  }
+  CloseBackupableDB();
+
+  for (int i = 0; i < 5; ++i) {
+    AssertBackupConsistency(i + 1, 0, keys_iteration * (i + 5 + 1),
+                            keys_iteration * 11);
+  }
+}
+
+TEST(BackupableDBTest, DeleteTmpFiles) {
+  OpenBackupableDB();
+  CloseBackupableDB();
+  std::string shared_tmp = backupdir_ + "/shared/00006.sst.tmp";
+  std::string private_tmp_dir = backupdir_ + "/private/10.tmp";
+  std::string private_tmp_file = private_tmp_dir + "/00003.sst";
+  file_manager_->WriteToFile(shared_tmp, "tmp");
+  file_manager_->CreateDir(private_tmp_dir);
+  file_manager_->WriteToFile(private_tmp_file, "tmp");
+  ASSERT_EQ(true, file_manager_->FileExists(private_tmp_dir));
+  OpenBackupableDB();
+  CloseBackupableDB();
+  ASSERT_EQ(false, file_manager_->FileExists(shared_tmp));
+  ASSERT_EQ(false, file_manager_->FileExists(private_tmp_file));
+  ASSERT_EQ(false, file_manager_->FileExists(private_tmp_dir));
+}
+
+TEST(BackupableDBTest, KeepLogFiles) {
+  backupable_options_->backup_log_files = false;
+  // basically infinite
+  options_.WAL_ttl_seconds = 24 * 60 * 60;
+  OpenBackupableDB(true);
+  FillDB(db_.get(), 0, 100);
+  ASSERT_OK(db_->Flush(FlushOptions()));
+  FillDB(db_.get(), 100, 200);
+  ASSERT_OK(db_->CreateNewBackup(false));
+  FillDB(db_.get(), 200, 300);
+  ASSERT_OK(db_->Flush(FlushOptions()));
+  FillDB(db_.get(), 300, 400);
+  ASSERT_OK(db_->Flush(FlushOptions()));
+  FillDB(db_.get(), 400, 500);
+  ASSERT_OK(db_->Flush(FlushOptions()));
+  CloseBackupableDB();
+
+  // all data should be there if we call with keep_log_files = true
+  AssertBackupConsistency(0, 0, 500, 600, true);
+}
+
+TEST(BackupableDBTest, RateLimiting) {
+  uint64_t const KB = 1024 * 1024;
+  size_t const kMicrosPerSec = 1000 * 1000LL;
+
+  std::vector<std::pair<uint64_t, uint64_t>> limits(
+      {{KB, 5 * KB}, {2 * KB, 3 * KB}});
+
+  for (const auto& limit : limits) {
+    // destroy old data
+    DestroyDB(dbname_, Options());
+
+    backupable_options_->backup_rate_limit = limit.first;
+    backupable_options_->restore_rate_limit = limit.second;
+    options_.compression = kNoCompression;
+    OpenBackupableDB(true);
+    size_t bytes_written = FillDB(db_.get(), 0, 100000);
+
+    auto start_backup = env_->NowMicros();
+    ASSERT_OK(db_->CreateNewBackup(false));
+    auto backup_time = env_->NowMicros() - start_backup;
+    auto rate_limited_backup_time = (bytes_written * kMicrosPerSec) /
+                                    backupable_options_->backup_rate_limit;
+    ASSERT_GT(backup_time, 0.9 * rate_limited_backup_time);
+
+    CloseBackupableDB();
+
+    OpenRestoreDB();
+    auto start_restore = env_->NowMicros();
+    ASSERT_OK(restore_db_->RestoreDBFromLatestBackup(dbname_, dbname_));
+    auto restore_time = env_->NowMicros() - start_restore;
+    CloseRestoreDB();
+    auto rate_limited_restore_time = (bytes_written * kMicrosPerSec) /
+                                     backupable_options_->restore_rate_limit;
+    ASSERT_GT(restore_time, 0.9 * rate_limited_restore_time);
+
+    AssertBackupConsistency(0, 0, 100000, 100010);
+  }
+}
+
+TEST(BackupableDBTest, ReadOnlyBackupEngine) {
+  DestroyDB(dbname_, Options());
+  OpenBackupableDB(true);
+  FillDB(db_.get(), 0, 100);
+  ASSERT_OK(db_->CreateNewBackup(true));
+  FillDB(db_.get(), 100, 200);
+  ASSERT_OK(db_->CreateNewBackup(true));
+  CloseBackupableDB();
+  DestroyDB(dbname_, Options());
+
+  backupable_options_->destroy_old_data = false;
+  test_backup_env_->ClearWrittenFiles();
+  test_backup_env_->SetLimitDeleteFiles(0);
+  auto read_only_backup_engine =
+      BackupEngineReadOnly::NewReadOnlyBackupEngine(env_, *backupable_options_);
+  std::vector<BackupInfo> backup_info;
+  read_only_backup_engine->GetBackupInfo(&backup_info);
+  ASSERT_EQ(backup_info.size(), 2U);
+
+  RestoreOptions restore_options(false);
+  ASSERT_OK(read_only_backup_engine->RestoreDBFromLatestBackup(
+      dbname_, dbname_, restore_options));
+  delete read_only_backup_engine;
+  std::vector<std::string> should_have_written;
+  test_backup_env_->AssertWrittenFiles(should_have_written);
+
+  DB* db = OpenDB();
+  AssertExists(db, 0, 200);
+  delete db;
+}
+
+}  // anon namespace
+
+} //  namespace rocksdb
+
+int main(int argc, char** argv) {
+  return rocksdb::test::RunAllTests();
+}

utilities/geodb/geodb_impl.cc

@@ -0,0 +1,431 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+#ifndef ROCKSDB_LITE
+
+#include "utilities/geodb/geodb_impl.h"
+
+#define __STDC_FORMAT_MACROS
+
+#include <vector>
+#include <map>
+#include <string>
+#include <limits>
+#include "db/filename.h"
+#include "util/coding.h"
+
+//
+// There are two types of keys. The first type of key-values
+// maps a geo location to the set of object ids and their values.
+// Table 1
+//   key     : p + : + $quadkey + : + $id +
+//             : + $latitude + : + $longitude
+//   value  :  value of the object
+// This table can be used to find all objects that reside near
+// a specified geolocation.
+//
+// Table 2
+//   key  : 'k' + : + $id
+//   value:  $quadkey
+
+namespace rocksdb {
+
+GeoDBImpl::GeoDBImpl(DB* db, const GeoDBOptions& options) :
+  GeoDB(db, options), db_(db), options_(options) {
+}
+
+GeoDBImpl::~GeoDBImpl() {
+}
+
+Status GeoDBImpl::Insert(const GeoObject& obj) {
+  WriteBatch batch;
+
+  // It is possible that this id is already associated with
+  // with a different position. We first have to remove that
+  // association before we can insert the new one.
+
+  // remove existing object, if it exists
+  GeoObject old;
+  Status status = GetById(obj.id, &old);
+  if (status.ok()) {
+    assert(obj.id.compare(old.id) == 0);
+    std::string quadkey = PositionToQuad(old.position, Detail);
+    std::string key1 = MakeKey1(old.position, old.id, quadkey);
+    std::string key2 = MakeKey2(old.id);
+    batch.Delete(Slice(key1));
+    batch.Delete(Slice(key2));
+  } else if (status.IsNotFound()) {
+    // What if another thread is trying to insert the same ID concurrently?
+  } else {
+    return status;
+  }
+
+  // insert new object
+  std::string quadkey = PositionToQuad(obj.position, Detail);
+  std::string key1 = MakeKey1(obj.position, obj.id, quadkey);
+  std::string key2 = MakeKey2(obj.id);
+  batch.Put(Slice(key1), Slice(obj.value));
+  batch.Put(Slice(key2), Slice(quadkey));
+  return db_->Write(woptions_, &batch);
+}
+
+Status GeoDBImpl::GetByPosition(const GeoPosition& pos,
+                                const Slice& id,
+                                std::string* value) {
+  std::string quadkey = PositionToQuad(pos, Detail);
+  std::string key1 = MakeKey1(pos, id, quadkey);
+  return db_->Get(roptions_, Slice(key1), value);
+}
+
+Status GeoDBImpl::GetById(const Slice& id, GeoObject* object) {
+  Status status;
+  Slice quadkey;
+
+  // create an iterator so that we can get a consistent picture
+  // of the database.
+  Iterator* iter = db_->NewIterator(roptions_);
+
+  // create key for table2
+  std::string kt = MakeKey2(id);
+  Slice key2(kt);
+
+  iter->Seek(key2);
+  if (iter->Valid() && iter->status().ok()) {
+    if (iter->key().compare(key2) == 0) {
+      quadkey = iter->value();
+    }
+  }
+  if (quadkey.size() == 0) {
+    delete iter;
+    return Status::NotFound(key2);
+  }
+
+  //
+  // Seek to the quadkey + id prefix
+  //
+  std::string prefix = MakeKey1Prefix(quadkey.ToString(), id);
+  iter->Seek(Slice(prefix));
+  assert(iter->Valid());
+  if (!iter->Valid() || !iter->status().ok()) {
+    delete iter;
+    return Status::NotFound();
+  }
+
+  // split the key into p + quadkey + id + lat + lon
+  std::vector<std::string> parts;
+  Slice key = iter->key();
+  StringSplit(&parts, key.ToString(), ':');
+  assert(parts.size() == 5);
+  assert(parts[0] == "p");
+  assert(parts[1] == quadkey);
+  assert(parts[2] == id);
+
+  // fill up output parameters
+  object->position.latitude = atof(parts[3].c_str());
+  object->position.longitude = atof(parts[4].c_str());
+  object->id = id.ToString();  // this is redundant
+  object->value = iter->value().ToString();
+  delete iter;
+  return Status::OK();
+}
+
+
+Status GeoDBImpl::Remove(const Slice& id) {
+  // Read the object from the database
+  GeoObject obj;
+  Status status = GetById(id, &obj);
+  if (!status.ok()) {
+    return status;
+  }
+
+  // remove the object by atomically deleting it from both tables
+  std::string quadkey = PositionToQuad(obj.position, Detail);
+  std::string key1 = MakeKey1(obj.position, obj.id, quadkey);
+  std::string key2 = MakeKey2(obj.id);
+  WriteBatch batch;
+  batch.Delete(Slice(key1));
+  batch.Delete(Slice(key2));
+  return db_->Write(woptions_, &batch);
+}
+
+Status GeoDBImpl::SearchRadial(const GeoPosition& pos,
+  double radius,
+  std::vector<GeoObject>* values,
+  int number_of_values) {
+  // Gather all bounding quadkeys
+  std::vector<std::string> qids;
+  Status s = searchQuadIds(pos, radius, &qids);
+  if (!s.ok()) {
+    return s;
+  }
+
+  // create an iterator
+  Iterator* iter = db_->NewIterator(ReadOptions());
+
+  // Process each prospective quadkey
+  for (std::string qid : qids) {
+    // The user is interested in only these many objects.
+    if (number_of_values == 0) {
+      break;
+    }
+
+    // convert quadkey to db key prefix
+    std::string dbkey = MakeQuadKeyPrefix(qid);
+
+    for (iter->Seek(dbkey);
+         number_of_values > 0 && iter->Valid() && iter->status().ok();
+         iter->Next()) {
+      // split the key into p + quadkey + id + lat + lon
+      std::vector<std::string> parts;
+      Slice key = iter->key();
+      StringSplit(&parts, key.ToString(), ':');
+      assert(parts.size() == 5);
+      assert(parts[0] == "p");
+      std::string* quadkey = &parts[1];
+
+      // If the key we are looking for is a prefix of the key
+      // we found from the database, then this is one of the keys
+      // we are looking for.
+      auto res = std::mismatch(qid.begin(), qid.end(), quadkey->begin());
+      if (res.first == qid.end()) {
+        GeoPosition pos(atof(parts[3].c_str()), atof(parts[4].c_str()));
+        GeoObject obj(pos, parts[4], iter->value().ToString());
+        values->push_back(obj);
+        number_of_values--;
+      } else {
+        break;
+      }
+    }
+  }
+  delete iter;
+  return Status::OK();
+}
+
+std::string GeoDBImpl::MakeKey1(const GeoPosition& pos, Slice id,
+                                std::string quadkey) {
+  std::string lat = std::to_string(pos.latitude);
+  std::string lon = std::to_string(pos.longitude);
+  std::string key = "p:";
+  key.reserve(5 + quadkey.size() + id.size() + lat.size() + lon.size());
+  key.append(quadkey);
+  key.append(":");
+  key.append(id.ToString());
+  key.append(":");
+  key.append(lat);
+  key.append(":");
+  key.append(lon);
+  return key;
+}
+
+std::string GeoDBImpl::MakeKey2(Slice id) {
+  std::string key = "k:";
+  key.append(id.ToString());
+  return key;
+}
+
+std::string GeoDBImpl::MakeKey1Prefix(std::string quadkey,
+                                      Slice id) {
+  std::string key = "p:";
+  key.reserve(3 + quadkey.size() + id.size());
+  key.append(quadkey);
+  key.append(":");
+  key.append(id.ToString());
+  return key;
+}
+
+std::string GeoDBImpl::MakeQuadKeyPrefix(std::string quadkey) {
+  std::string key = "p:";
+  key.append(quadkey);
+  return key;
+}
+
+void GeoDBImpl::StringSplit(std::vector<std::string>* tokens,
+                            const std::string &text, char sep) {
+  std::size_t start = 0, end = 0;
+  while ((end = text.find(sep, start)) != std::string::npos) {
+    tokens->push_back(text.substr(start, end - start));
+    start = end + 1;
+  }
+  tokens->push_back(text.substr(start));
+}
+
+// convert degrees to radians
+double GeoDBImpl::radians(double x) {
+  return (x * PI) / 180;
+}
+
+// convert radians to degrees
+double GeoDBImpl::degrees(double x) {
+  return (x * 180) / PI;
+}
+
+// convert a gps location to quad coordinate
+std::string GeoDBImpl::PositionToQuad(const GeoPosition& pos,
+                                      int levelOfDetail) {
+  Pixel p = PositionToPixel(pos, levelOfDetail);
+  Tile tile = PixelToTile(p);
+  return TileToQuadKey(tile, levelOfDetail);
+}
+
+GeoPosition GeoDBImpl::displaceLatLon(double lat, double lon,
+                                      double deltay, double deltax) {
+  double dLat = deltay / EarthRadius;
+  double dLon = deltax / (EarthRadius * cos(radians(lat)));
+  return GeoPosition(lat + degrees(dLat),
+                     lon + degrees(dLon));
+}
+
+//
+// Return the distance between two positions on the earth
+//
+double GeoDBImpl::distance(double lat1, double lon1,
+                           double lat2, double lon2) {
+  double lon = radians(lon2 - lon1);
+  double lat = radians(lat2 - lat1);
+
+  double a = (sin(lat / 2) * sin(lat / 2)) +
+              cos(radians(lat1)) * cos(radians(lat2)) *
+              (sin(lon / 2) * sin(lon / 2));
+  double angle = 2 * atan2(sqrt(a), sqrt(1 - a));
+  return angle * EarthRadius;
+}
+
+//
+// Returns all the quadkeys inside the search range
+//
+Status GeoDBImpl::searchQuadIds(const GeoPosition& position,
+                                double radius,
+                                std::vector<std::string>* quadKeys) {
+  // get the outline of the search square
+  GeoPosition topLeftPos = boundingTopLeft(position, radius);
+  GeoPosition bottomRightPos = boundingBottomRight(position, radius);
+
+  Pixel topLeft =  PositionToPixel(topLeftPos, Detail);
+  Pixel bottomRight =  PositionToPixel(bottomRightPos, Detail);
+
+  // how many level of details to look for
+  int numberOfTilesAtMaxDepth = floor((bottomRight.x - topLeft.x) / 256);
+  int zoomLevelsToRise = floor(log(numberOfTilesAtMaxDepth) / log(2));
+  zoomLevelsToRise++;
+  int levels = std::max(0, Detail - zoomLevelsToRise);
+
+  quadKeys->push_back(PositionToQuad(GeoPosition(topLeftPos.latitude,
+                                                 topLeftPos.longitude),
+                                     levels));
+  quadKeys->push_back(PositionToQuad(GeoPosition(topLeftPos.latitude,
+                                                 bottomRightPos.longitude),
+                                     levels));
+  quadKeys->push_back(PositionToQuad(GeoPosition(bottomRightPos.latitude,
+                                                 topLeftPos.longitude),
+                                     levels));
+  quadKeys->push_back(PositionToQuad(GeoPosition(bottomRightPos.latitude,
+                                                 bottomRightPos.longitude),
+                                     levels));
+  return Status::OK();
+}
+
+// Determines the ground resolution (in meters per pixel) at a specified
+// latitude and level of detail.
+// Latitude (in degrees) at which to measure the ground resolution.
+// Level of detail, from 1 (lowest detail) to 23 (highest detail).
+// Returns the ground resolution, in meters per pixel.
+double GeoDBImpl::GroundResolution(double latitude, int levelOfDetail) {
+  latitude = clip(latitude, MinLatitude, MaxLatitude);
+  return cos(latitude * PI / 180) * 2 * PI * EarthRadius /
+         MapSize(levelOfDetail);
+}
+
+// Converts a point from latitude/longitude WGS-84 coordinates (in degrees)
+// into pixel XY coordinates at a specified level of detail.
+GeoDBImpl::Pixel GeoDBImpl::PositionToPixel(const GeoPosition& pos,
+                                            int levelOfDetail) {
+  double latitude = clip(pos.latitude, MinLatitude, MaxLatitude);
+  double x = (pos.longitude + 180) / 360;
+  double sinLatitude = sin(latitude * PI / 180);
+  double y = 0.5 - log((1 + sinLatitude) / (1 - sinLatitude)) / (4 * PI);
+  double mapSize = MapSize(levelOfDetail);
+  double X = floor(clip(x * mapSize + 0.5, 0, mapSize - 1));
+  double Y = floor(clip(y * mapSize + 0.5, 0, mapSize - 1));
+  return Pixel((unsigned int)X, (unsigned int)Y);
+}
+
+GeoPosition GeoDBImpl::PixelToPosition(const Pixel& pixel, int levelOfDetail) {
+  double mapSize = MapSize(levelOfDetail);
+  double x = (clip(pixel.x, 0, mapSize - 1) / mapSize) - 0.5;
+  double y = 0.5 - (clip(pixel.y, 0, mapSize - 1) / mapSize);
+  double latitude = 90 - 360 * atan(exp(-y * 2 * PI)) / PI;
+  double longitude = 360 * x;
+  return GeoPosition(latitude, longitude);
+}
+
+// Converts a Pixel to a Tile
+GeoDBImpl::Tile GeoDBImpl::PixelToTile(const Pixel& pixel) {
+  unsigned int tileX = floor(pixel.x / 256);
+  unsigned int tileY = floor(pixel.y / 256);
+  return Tile(tileX, tileY);
+}
+
+GeoDBImpl::Pixel GeoDBImpl::TileToPixel(const Tile& tile) {
+  unsigned int pixelX = tile.x * 256;
+  unsigned int pixelY = tile.y * 256;
+  return Pixel(pixelX, pixelY);
+}
+
+// Convert a Tile to a quadkey
+std::string GeoDBImpl::TileToQuadKey(const Tile& tile, int levelOfDetail) {
+  std::stringstream quadKey;
+  for (int i = levelOfDetail; i > 0; i--) {
+    char digit = '0';
+    int mask = 1 << (i - 1);
+    if ((tile.x & mask) != 0) {
+      digit++;
+    }
+    if ((tile.y & mask) != 0) {
+      digit++;
+      digit++;
+    }
+    quadKey << digit;
+  }
+  return quadKey.str();
+}
+
+//
+// Convert a quadkey to a tile and its level of detail
+//
+void GeoDBImpl::QuadKeyToTile(std::string quadkey, Tile* tile,
+                                     int *levelOfDetail) {
+  tile->x = tile->y = 0;
+  *levelOfDetail = quadkey.size();
+  const char* key = reinterpret_cast<const char *>(quadkey.c_str());
+  for (int i = *levelOfDetail; i > 0; i--) {
+    int mask = 1 << (i - 1);
+    switch (key[*levelOfDetail - i]) {
+      case '0':
+        break;
+
+      case '1':
+        tile->x |= mask;
+        break;
+
+      case '2':
+        tile->y |= mask;
+        break;
+
+      case '3':
+        tile->x |= mask;
+        tile->y |= mask;
+        break;
+
+      default:
+        std::stringstream msg;
+        msg << quadkey;
+        msg << " Invalid QuadKey.";
+        throw std::runtime_error(msg.str());
+    }
+  }
+}
+}  // namespace rocksdb
+
+#endif  // ROCKSDB_LITE

utilities/geodb/geodb_impl.h

@@ -0,0 +1,191 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+
+#ifndef ROCKSDB_LITE
+
+#pragma once
+#include <algorithm>
+#include <cmath>
+#include <string>
+#include <sstream>
+#include <stdexcept>
+#include <vector>
+
+#include "utilities/geo_db.h"
+#include "utilities/stackable_db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/status.h"
+
+namespace rocksdb {
+
+// A specific implementation of GeoDB
+
+class GeoDBImpl : public GeoDB {
+ public:
+  GeoDBImpl(DB* db, const GeoDBOptions& options);
+  ~GeoDBImpl();
+
+  // Associate the GPS location with the identified by 'id'. The value
+  // is a blob that is associated with this object.
+  virtual Status Insert(const GeoObject& object);
+
+  // Retrieve the value of the object located at the specified GPS
+  // location and is identified by the 'id'.
+  virtual Status GetByPosition(const GeoPosition& pos,
+                               const Slice& id,
+                               std::string* value);
+
+  // Retrieve the value of the object identified by the 'id'. This method
+  // could be potentially slower than GetByPosition
+  virtual Status GetById(const Slice& id, GeoObject* object);
+
+  // Delete the specified object
+  virtual Status Remove(const Slice& id);
+
+  // Returns a list of all items within a circular radius from the
+  // specified gps location
+  virtual Status SearchRadial(const GeoPosition& pos,
+                              double radius,
+                              std::vector<GeoObject>* values,
+                              int number_of_values);
+
+ private:
+  DB* db_;
+  const GeoDBOptions options_;
+  const WriteOptions woptions_;
+  const ReadOptions roptions_;
+
+  // The value of PI
+  static constexpr double PI = 3.141592653589793;
+
+  // convert degrees to radians
+  static double radians(double x);
+
+  // convert radians to degrees
+  static double degrees(double x);
+
+  // A pixel class that captures X and Y coordinates
+  class Pixel {
+   public:
+    unsigned int x;
+    unsigned int y;
+    Pixel(unsigned int a, unsigned int b) :
+     x(a), y(b) {
+    }
+  };
+
+  // A Tile in the geoid
+  class Tile {
+   public:
+    unsigned int x;
+    unsigned int y;
+    Tile(unsigned int a, unsigned int b) :
+     x(a), y(b) {
+    }
+  };
+
+  // convert a gps location to quad coordinate
+  static std::string PositionToQuad(const GeoPosition& pos, int levelOfDetail);
+
+  // arbitrary constant use for WGS84 via
+  // http://en.wikipedia.org/wiki/World_Geodetic_System
+  // http://mathforum.org/library/drmath/view/51832.html
+  // http://msdn.microsoft.com/en-us/library/bb259689.aspx
+  // http://www.tuicool.com/articles/NBrE73
+  //
+  const int Detail = 23;
+  static constexpr double EarthRadius = 6378137;
+  static constexpr double MinLatitude = -85.05112878;
+  static constexpr double MaxLatitude = 85.05112878;
+  static constexpr double MinLongitude = -180;
+  static constexpr double MaxLongitude = 180;
+
+  // clips a number to the specified minimum and maximum values.
+  static double clip(double n, double minValue, double maxValue) {
+    return fmin(fmax(n, minValue), maxValue);
+  }
+
+  // Determines the map width and height (in pixels) at a specified level
+  // of detail, from 1 (lowest detail) to 23 (highest detail).
+  // Returns the map width and height in pixels.
+  static unsigned int MapSize(int levelOfDetail) {
+    return (unsigned int)(256 << levelOfDetail);
+  }
+
+  // Determines the ground resolution (in meters per pixel) at a specified
+  // latitude and level of detail.
+  // Latitude (in degrees) at which to measure the ground resolution.
+  // Level of detail, from 1 (lowest detail) to 23 (highest detail).
+  // Returns the ground resolution, in meters per pixel.
+  static double GroundResolution(double latitude, int levelOfDetail);
+
+  // Converts a point from latitude/longitude WGS-84 coordinates (in degrees)
+  // into pixel XY coordinates at a specified level of detail.
+  static Pixel PositionToPixel(const GeoPosition& pos, int levelOfDetail);
+
+  static GeoPosition PixelToPosition(const Pixel& pixel, int levelOfDetail);
+
+  // Converts a Pixel to a Tile
+  static Tile PixelToTile(const Pixel& pixel);
+
+  static Pixel TileToPixel(const Tile& tile);
+
+  // Convert a Tile to a quadkey
+  static std::string TileToQuadKey(const Tile& tile, int levelOfDetail);
+
+  // Convert a quadkey to a tile and its level of detail
+  static void QuadKeyToTile(std::string quadkey, Tile* tile,
+                            int *levelOfDetail);
+
+  // Return the distance between two positions on the earth
+  static double distance(double lat1, double lon1,
+                         double lat2, double lon2);
+  static GeoPosition displaceLatLon(double lat, double lon,
+                                    double deltay, double deltax);
+
+  //
+  // Returns the top left position after applying the delta to
+  // the specified position
+  //
+  static GeoPosition boundingTopLeft(const GeoPosition& in, double radius) {
+    return displaceLatLon(in.latitude, in.longitude, -radius, -radius);
+  }
+
+  //
+  // Returns the bottom right position after applying the delta to
+  // the specified position
+  static GeoPosition boundingBottomRight(const GeoPosition& in,
+                                         double radius) {
+    return displaceLatLon(in.latitude, in.longitude, radius, radius);
+  }
+
+  //
+  // Get all quadkeys within a radius of a specified position
+  //
+  Status searchQuadIds(const GeoPosition& position,
+                       double radius,
+                       std::vector<std::string>* quadKeys);
+
+  // splits a string into its components
+  static void StringSplit(std::vector<std::string>* tokens,
+                          const std::string &text,
+                          char sep);
+
+  //
+  // Create keys for accessing rocksdb table(s)
+  //
+  static std::string MakeKey1(const GeoPosition& pos,
+                              Slice id,
+                              std::string quadkey);
+  static std::string MakeKey2(Slice id);
+  static std::string MakeKey1Prefix(std::string quadkey,
+                                    Slice id);
+  static std::string MakeQuadKeyPrefix(std::string quadkey);
+};
+
+}  // namespace rocksdb
+
+#endif  // ROCKSDB_LITE

utilities/geodb/geodb_test.cc

@@ -0,0 +1,123 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+//
+#include "utilities/geodb/geodb_impl.h"
+
+#include <cctype>
+#include "util/testharness.h"
+
+namespace rocksdb {
+
+class GeoDBTest {
+ public:
+  static const std::string kDefaultDbName;
+  static Options options;
+  DB* db;
+  GeoDB* geodb;
+
+  GeoDBTest() {
+    GeoDBOptions geodb_options;
+    ASSERT_OK(DestroyDB(kDefaultDbName, options));
+    options.create_if_missing = true;
+    Status status = DB::Open(options, kDefaultDbName, &db);
+    geodb =  new GeoDBImpl(db, geodb_options);
+  }
+
+  ~GeoDBTest() {
+    delete geodb;
+  }
+
+  GeoDB* getdb() {
+    return geodb;
+  }
+};
+
+const std::string GeoDBTest::kDefaultDbName = "/tmp/geodefault";
+Options GeoDBTest::options = Options();
+
+// Insert, Get and Remove
+TEST(GeoDBTest, SimpleTest) {
+  GeoPosition pos1(100, 101);
+  std::string id1("id1");
+  std::string value1("value1");
+
+  // insert first object into database
+  GeoObject obj1(pos1, id1, value1);
+  Status status = getdb()->Insert(obj1);
+  ASSERT_TRUE(status.ok());
+
+  // insert second object into database
+  GeoPosition pos2(200, 201);
+  std::string id2("id2");
+  std::string value2 = "value2";
+  GeoObject obj2(pos2, id2, value2);
+  status = getdb()->Insert(obj2);
+  ASSERT_TRUE(status.ok());
+
+  // retrieve first object using position
+  std::string value;
+  status = getdb()->GetByPosition(pos1, Slice(id1), &value);
+  ASSERT_TRUE(status.ok());
+  ASSERT_EQ(value, value1);
+
+  // retrieve first object using id
+  GeoObject obj;
+  status = getdb()->GetById(Slice(id1), &obj);
+  ASSERT_TRUE(status.ok());
+  ASSERT_EQ(obj.position.latitude, 100);
+  ASSERT_EQ(obj.position.longitude, 101);
+  ASSERT_EQ(obj.id.compare(id1), 0);
+  ASSERT_EQ(obj.value, value1);
+
+  // delete first object
+  status = getdb()->Remove(Slice(id1));
+  ASSERT_TRUE(status.ok());
+  status = getdb()->GetByPosition(pos1, Slice(id1), &value);
+  ASSERT_TRUE(status.IsNotFound());
+  status = getdb()->GetById(id1, &obj);
+  ASSERT_TRUE(status.IsNotFound());
+
+  // check that we can still find second object
+  status = getdb()->GetByPosition(pos2, id2, &value);
+  ASSERT_TRUE(status.ok());
+  ASSERT_EQ(value, value2);
+  status = getdb()->GetById(id2, &obj);
+  ASSERT_TRUE(status.ok());
+}
+
+// Search.
+// Verify distances via http://www.stevemorse.org/nearest/distance.php
+TEST(GeoDBTest, Search) {
+  GeoPosition pos1(45, 45);
+  std::string id1("mid1");
+  std::string value1 = "midvalue1";
+
+  // insert object at 45 degree latitude
+  GeoObject obj1(pos1, id1, value1);
+  Status status = getdb()->Insert(obj1);
+  ASSERT_TRUE(status.ok());
+
+  // search all objects centered at 46 degree latitude with
+  // a radius of 200 kilometers. We should find the one object that
+  // we inserted earlier.
+  std::vector<GeoObject> values;
+  status = getdb()->SearchRadial(GeoPosition(46, 46), 200000, &values);
+  ASSERT_TRUE(status.ok());
+  ASSERT_EQ(values.size(), 1U);
+
+  // search all objects centered at 46 degree latitude with
+  // a radius of 2 kilometers. There should be none.
+  values.clear();
+  status = getdb()->SearchRadial(GeoPosition(46, 46), 2, &values);
+  ASSERT_TRUE(status.ok());
+  ASSERT_EQ(values.size(), 0U);
+}
+
+}  // namespace rocksdb
+
+int main(int argc, char* argv[]) {
+  return rocksdb::test::RunAllTests();
+}

utilities/merge_operators.h

@@ -0,0 +1,45 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+#ifndef MERGE_OPERATORS_H
+#define MERGE_OPERATORS_H
+
+#include <memory>
+#include <stdio.h>
+
+#include "rocksdb/merge_operator.h"
+
+namespace rocksdb {
+
+class MergeOperators {
+ public:
+  static std::shared_ptr<MergeOperator> CreatePutOperator();
+  static std::shared_ptr<MergeOperator> CreateUInt64AddOperator();
+  static std::shared_ptr<MergeOperator> CreateStringAppendOperator();
+  static std::shared_ptr<MergeOperator> CreateStringAppendTESTOperator();
+
+  // Will return a different merge operator depending on the string.
+  // TODO: Hook the "name" up to the actual Name() of the MergeOperators?
+  static std::shared_ptr<MergeOperator> CreateFromStringId(
+      const std::string& name) {
+    if (name == "put") {
+      return CreatePutOperator();
+    } else if ( name == "uint64add") {
+      return CreateUInt64AddOperator();
+    } else if (name == "stringappend") {
+      return CreateStringAppendOperator();
+    } else if (name == "stringappendtest") {
+      return CreateStringAppendTESTOperator();
+    } else {
+      // Empty or unknown, just return nullptr
+      return nullptr;
+    }
+  }
+
+};
+
+} // namespace rocksdb
+
+#endif

utilities/merge_operators/put.cc

@@ -0,0 +1,68 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#include <memory>
+#include "rocksdb/slice.h"
+#include "rocksdb/merge_operator.h"
+#include "utilities/merge_operators.h"
+
+using namespace rocksdb;
+
+namespace { // anonymous namespace
+
+// A merge operator that mimics Put semantics
+// Since this merge-operator will not be used in production,
+// it is implemented as a non-associative merge operator to illustrate the
+// new interface and for testing purposes. (That is, we inherit from
+// the MergeOperator class rather than the AssociativeMergeOperator
+// which would be simpler in this case).
+//
+// From the client-perspective, semantics are the same.
+class PutOperator : public MergeOperator {
+ public:
+  virtual bool FullMerge(const Slice& key,
+                         const Slice* existing_value,
+                         const std::deque<std::string>& operand_sequence,
+                         std::string* new_value,
+                         Logger* logger) const override {
+    // Put basically only looks at the current/latest value
+    assert(!operand_sequence.empty());
+    assert(new_value != nullptr);
+    new_value->assign(operand_sequence.back());
+    return true;
+  }
+
+  virtual bool PartialMerge(const Slice& key,
+                            const Slice& left_operand,
+                            const Slice& right_operand,
+                            std::string* new_value,
+                            Logger* logger) const override {
+    new_value->assign(right_operand.data(), right_operand.size());
+    return true;
+  }
+
+  using MergeOperator::PartialMergeMulti;
+  virtual bool PartialMergeMulti(const Slice& key,
+                                 const std::deque<Slice>& operand_list,
+                                 std::string* new_value, Logger* logger) const
+      override {
+    new_value->assign(operand_list.back().data(), operand_list.back().size());
+    return true;
+  }
+
+  virtual const char* Name() const override {
+    return "PutOperator";
+  }
+};
+
+} // end of anonymous namespace
+
+namespace rocksdb {
+
+std::shared_ptr<MergeOperator> MergeOperators::CreatePutOperator() {
+  return std::make_shared<PutOperator>();
+}
+
+}

utilities/merge_operators/string_append/stringappend.cc

@@ -0,0 +1,60 @@
+/**
+ * A MergeOperator for rocksdb that implements string append.
+ * @author Deon Nicholas (dnicholas@fb.com)
+ * Copyright 2013 Facebook
+ */
+
+#include "stringappend.h"
+
+#include <memory>
+#include <assert.h>
+
+#include "rocksdb/slice.h"
+#include "rocksdb/merge_operator.h"
+#include "utilities/merge_operators.h"
+
+namespace rocksdb {
+
+// Constructor: also specify the delimiter character.
+StringAppendOperator::StringAppendOperator(char delim_char)
+    : delim_(delim_char) {
+}
+
+// Implementation for the merge operation (concatenates two strings)
+bool StringAppendOperator::Merge(const Slice& key,
+                                 const Slice* existing_value,
+                                 const Slice& value,
+                                 std::string* new_value,
+                                 Logger* logger) const {
+
+  // Clear the *new_value for writing.
+  assert(new_value);
+  new_value->clear();
+
+  if (!existing_value) {
+    // No existing_value. Set *new_value = value
+    new_value->assign(value.data(),value.size());
+  } else {
+    // Generic append (existing_value != null).
+    // Reserve *new_value to correct size, and apply concatenation.
+    new_value->reserve(existing_value->size() + 1 + value.size());
+    new_value->assign(existing_value->data(),existing_value->size());
+    new_value->append(1,delim_);
+    new_value->append(value.data(), value.size());
+  }
+
+  return true;
+}
+
+const char* StringAppendOperator::Name() const  {
+  return "StringAppendOperator";
+}
+
+std::shared_ptr<MergeOperator> MergeOperators::CreateStringAppendOperator() {
+  return std::make_shared<StringAppendOperator>(',');
+}
+
+} // namespace rocksdb
+
+
+

utilities/merge_operators/string_append/stringappend.h

@@ -0,0 +1,31 @@
+/**
+ * A MergeOperator for rocksdb that implements string append.
+ * @author Deon Nicholas (dnicholas@fb.com)
+ * Copyright 2013 Facebook
+ */
+
+#pragma once
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/slice.h"
+
+namespace rocksdb {
+
+class StringAppendOperator : public AssociativeMergeOperator {
+ public:
+  StringAppendOperator(char delim_char);    /// Constructor: specify delimiter
+
+  virtual bool Merge(const Slice& key,
+                     const Slice* existing_value,
+                     const Slice& value,
+                     std::string* new_value,
+                     Logger* logger) const override;
+
+  virtual const char* Name() const override;
+
+ private:
+  char delim_;         // The delimiter is inserted between elements
+
+};
+
+} // namespace rocksdb
+

utilities/merge_operators/string_append/stringappend2.cc

@@ -0,0 +1,113 @@
+/**
+ * @author Deon Nicholas (dnicholas@fb.com)
+ * Copyright 2013 Facebook
+ */
+
+#include "stringappend2.h"
+
+#include <memory>
+#include <string>
+#include <assert.h>
+
+#include "rocksdb/slice.h"
+#include "rocksdb/merge_operator.h"
+#include "utilities/merge_operators.h"
+
+namespace rocksdb {
+
+// Constructor: also specify the delimiter character.
+StringAppendTESTOperator::StringAppendTESTOperator(char delim_char)
+    : delim_(delim_char) {
+}
+
+// Implementation for the merge operation (concatenates two strings)
+bool StringAppendTESTOperator::FullMerge(
+    const Slice& key,
+    const Slice* existing_value,
+    const std::deque<std::string>& operands,
+    std::string* new_value,
+    Logger* logger) const {
+
+  // Clear the *new_value for writing.
+  assert(new_value);
+  new_value->clear();
+
+  // Compute the space needed for the final result.
+  int numBytes = 0;
+  for(auto it = operands.begin(); it != operands.end(); ++it) {
+    numBytes += it->size() + 1;   // Plus 1 for the delimiter
+  }
+
+  // Only print the delimiter after the first entry has been printed
+  bool printDelim = false;
+
+  // Prepend the *existing_value if one exists.
+  if (existing_value) {
+    new_value->reserve(numBytes + existing_value->size());
+    new_value->append(existing_value->data(), existing_value->size());
+    printDelim = true;
+  } else if (numBytes) {
+    new_value->reserve(numBytes-1); // Minus 1 since we have one less delimiter
+  }
+
+  // Concatenate the sequence of strings (and add a delimiter between each)
+  for(auto it = operands.begin(); it != operands.end(); ++it) {
+    if (printDelim) {
+      new_value->append(1,delim_);
+    }
+    new_value->append(*it);
+    printDelim = true;
+  }
+
+  return true;
+}
+
+bool StringAppendTESTOperator::PartialMergeMulti(
+    const Slice& key, const std::deque<Slice>& operand_list,
+    std::string* new_value, Logger* logger) const {
+  return false;
+}
+
+// A version of PartialMerge that actually performs "partial merging".
+// Use this to simulate the exact behaviour of the StringAppendOperator.
+bool StringAppendTESTOperator::_AssocPartialMergeMulti(
+    const Slice& key, const std::deque<Slice>& operand_list,
+    std::string* new_value, Logger* logger) const {
+  // Clear the *new_value for writing
+  assert(new_value);
+  new_value->clear();
+  assert(operand_list.size() >= 2);
+
+  // Generic append
+  // Determine and reserve correct size for *new_value.
+  size_t size = 0;
+  for (const auto& operand : operand_list) {
+    size += operand.size();
+  }
+  size += operand_list.size() - 1;  // Delimiters
+  new_value->reserve(size);
+
+  // Apply concatenation
+  new_value->assign(operand_list.front().data(), operand_list.front().size());
+
+  for (std::deque<Slice>::const_iterator it = operand_list.begin() + 1;
+       it != operand_list.end(); ++it) {
+    new_value->append(1, delim_);
+    new_value->append(it->data(), it->size());
+  }
+
+  return true;
+}
+
+const char* StringAppendTESTOperator::Name() const  {
+  return "StringAppendTESTOperator";
+}
+
+
+std::shared_ptr<MergeOperator>
+MergeOperators::CreateStringAppendTESTOperator() {
+  return std::make_shared<StringAppendTESTOperator>(',');
+}
+
+} // namespace rocksdb
+

utilities/merge_operators/string_append/stringappend2.h

@@ -0,0 +1,51 @@
+/**
+ * A TEST MergeOperator for rocksdb that implements string append.
+ * It is built using the MergeOperator interface rather than the simpler
+ * AssociativeMergeOperator interface. This is useful for testing/benchmarking.
+ * While the two operators are semantically the same, all production code
+ * should use the StringAppendOperator defined in stringappend.{h,cc}. The
+ * operator defined in the present file is primarily for testing.
+ *
+ * @author Deon Nicholas (dnicholas@fb.com)
+ * Copyright 2013 Facebook
+ */
+
+#pragma once
+#include <deque>
+#include <string>
+
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/slice.h"
+
+namespace rocksdb {
+
+class StringAppendTESTOperator : public MergeOperator {
+ public:
+  // Constructor with delimiter
+  explicit StringAppendTESTOperator(char delim_char);
+
+  virtual bool FullMerge(const Slice& key,
+                         const Slice* existing_value,
+                         const std::deque<std::string>& operand_sequence,
+                         std::string* new_value,
+                         Logger* logger) const override;
+
+  virtual bool PartialMergeMulti(const Slice& key,
+                                 const std::deque<Slice>& operand_list,
+                                 std::string* new_value, Logger* logger) const
+      override;
+
+  virtual const char* Name() const override;
+
+ private:
+  // A version of PartialMerge that actually performs "partial merging".
+  // Use this to simulate the exact behaviour of the StringAppendOperator.
+  bool _AssocPartialMergeMulti(const Slice& key,
+                               const std::deque<Slice>& operand_list,
+                               std::string* new_value, Logger* logger) const;
+
+  char delim_;         // The delimiter is inserted between elements
+
+};
+
+} // namespace rocksdb

utilities/merge_operators/string_append/stringappend_test.cc

@@ -0,0 +1,595 @@
+/**
+ * An persistent map : key -> (list of strings), using rocksdb merge.
+ * This file is a test-harness / use-case for the StringAppendOperator.
+ *
+ * @author Deon Nicholas (dnicholas@fb.com)
+ * Copyright 2013 Facebook, Inc.
+*/
+
+#include <iostream>
+#include <map>
+
+#include "rocksdb/db.h"
+#include "rocksdb/merge_operator.h"
+#include "utilities/merge_operators.h"
+#include "utilities/merge_operators/string_append/stringappend.h"
+#include "utilities/merge_operators/string_append/stringappend2.h"
+#include "utilities/db_ttl.h"
+#include "util/testharness.h"
+#include "util/random.h"
+
+using namespace rocksdb;
+
+namespace rocksdb {
+
+// Path to the database on file system
+const std::string kDbName = "/tmp/mergetestdb";
+
+namespace {
+// OpenDb opens a (possibly new) rocksdb database with a StringAppendOperator
+std::shared_ptr<DB> OpenNormalDb(char delim_char) {
+  DB* db;
+  Options options;
+  options.create_if_missing = true;
+  options.merge_operator.reset(new StringAppendOperator(delim_char));
+  ASSERT_OK(DB::Open(options, kDbName,  &db));
+  return std::shared_ptr<DB>(db);
+}
+
+// Open a TtlDB with a non-associative StringAppendTESTOperator
+std::shared_ptr<DB> OpenTtlDb(char delim_char) {
+  DBWithTTL* db;
+  Options options;
+  options.create_if_missing = true;
+  options.merge_operator.reset(new StringAppendTESTOperator(delim_char));
+  ASSERT_OK(DBWithTTL::Open(options, kDbName, &db, 123456));
+  return std::shared_ptr<DB>(db);
+}
+}  // namespace
+
+/// StringLists represents a set of string-lists, each with a key-index.
+/// Supports Append(list, string) and Get(list)
+class StringLists {
+ public:
+
+  //Constructor: specifies the rocksdb db
+  /* implicit */
+  StringLists(std::shared_ptr<DB> db)
+      : db_(db),
+        merge_option_(),
+        get_option_() {
+    assert(db);
+  }
+
+  // Append string val onto the list defined by key; return true on success
+  bool Append(const std::string& key, const std::string& val){
+    Slice valSlice(val.data(), val.size());
+    auto s = db_->Merge(merge_option_, key, valSlice);
+
+    if (s.ok()) {
+      return true;
+    } else {
+      std::cerr << "ERROR " << s.ToString() << std::endl;
+      return false;
+    }
+  }
+
+  // Returns the list of strings associated with key (or "" if does not exist)
+  bool Get(const std::string& key, std::string* const result){
+    assert(result != nullptr); // we should have a place to store the result
+    auto s = db_->Get(get_option_, key, result);
+
+    if (s.ok()) {
+      return true;
+    }
+
+    // Either key does not exist, or there is some error.
+    *result = "";       // Always return empty string (just for convention)
+
+    //NotFound is okay; just return empty (similar to std::map)
+    //But network or db errors, etc, should fail the test (or at least yell)
+    if (!s.IsNotFound()) {
+      std::cerr << "ERROR " << s.ToString() << std::endl;
+    }
+
+    // Always return false if s.ok() was not true
+    return false;
+  }
+
+
+ private:
+  std::shared_ptr<DB> db_;
+  WriteOptions merge_option_;
+  ReadOptions get_option_;
+
+};
+
+
+// The class for unit-testing
+class StringAppendOperatorTest {
+ public:
+  StringAppendOperatorTest() {
+    DestroyDB(kDbName, Options());    // Start each test with a fresh DB
+  }
+
+  typedef std::shared_ptr<DB> (* OpenFuncPtr)(char);
+
+  // Allows user to open databases with different configurations.
+  // e.g.: Can open a DB or a TtlDB, etc.
+  static void SetOpenDbFunction(OpenFuncPtr func) {
+    OpenDb = func;
+  }
+
+ protected:
+  static OpenFuncPtr OpenDb;
+};
+StringAppendOperatorTest::OpenFuncPtr StringAppendOperatorTest::OpenDb = nullptr;
+
+// THE TEST CASES BEGIN HERE
+
+TEST(StringAppendOperatorTest, IteratorTest) {
+  auto db_ = OpenDb(',');
+  StringLists slists(db_);
+
+  slists.Append("k1", "v1");
+  slists.Append("k1", "v2");
+  slists.Append("k1", "v3");
+
+  slists.Append("k2", "a1");
+  slists.Append("k2", "a2");
+  slists.Append("k2", "a3");
+
+  std::string res;
+  std::unique_ptr<rocksdb::Iterator> it(db_->NewIterator(ReadOptions()));
+  std::string k1("k1");
+  std::string k2("k2");
+  bool first = true;
+  for (it->Seek(k1); it->Valid(); it->Next()) {
+    res = it->value().ToString();
+    if (first) {
+      ASSERT_EQ(res, "v1,v2,v3");
+      first = false;
+    } else {
+      ASSERT_EQ(res, "a1,a2,a3");
+    }
+  }
+  slists.Append("k2", "a4");
+  slists.Append("k1", "v4");
+
+  // Snapshot should still be the same. Should ignore a4 and v4.
+  first = true;
+  for (it->Seek(k1); it->Valid(); it->Next()) {
+    res = it->value().ToString();
+    if (first) {
+      ASSERT_EQ(res, "v1,v2,v3");
+      first = false;
+    } else {
+      ASSERT_EQ(res, "a1,a2,a3");
+    }
+  }
+
+
+  // Should release the snapshot and be aware of the new stuff now
+  it.reset(db_->NewIterator(ReadOptions()));
+  first = true;
+  for (it->Seek(k1); it->Valid(); it->Next()) {
+    res = it->value().ToString();
+    if (first) {
+      ASSERT_EQ(res, "v1,v2,v3,v4");
+      first = false;
+    } else {
+      ASSERT_EQ(res, "a1,a2,a3,a4");
+    }
+  }
+
+  // start from k2 this time.
+  for (it->Seek(k2); it->Valid(); it->Next()) {
+    res = it->value().ToString();
+    if (first) {
+      ASSERT_EQ(res, "v1,v2,v3,v4");
+      first = false;
+    } else {
+      ASSERT_EQ(res, "a1,a2,a3,a4");
+    }
+  }
+
+  slists.Append("k3", "g1");
+
+  it.reset(db_->NewIterator(ReadOptions()));
+  first = true;
+  std::string k3("k3");
+  for(it->Seek(k2); it->Valid(); it->Next()) {
+    res = it->value().ToString();
+    if (first) {
+      ASSERT_EQ(res, "a1,a2,a3,a4");
+      first = false;
+    } else {
+      ASSERT_EQ(res, "g1");
+    }
+  }
+  for(it->Seek(k3); it->Valid(); it->Next()) {
+    res = it->value().ToString();
+    if (first) {
+      // should not be hit
+      ASSERT_EQ(res, "a1,a2,a3,a4");
+      first = false;
+    } else {
+      ASSERT_EQ(res, "g1");
+    }
+  }
+
+}
+
+TEST(StringAppendOperatorTest, SimpleTest) {
+  auto db = OpenDb(',');
+  StringLists slists(db);
+
+  slists.Append("k1", "v1");
+  slists.Append("k1", "v2");
+  slists.Append("k1", "v3");
+
+  std::string res;
+  bool status = slists.Get("k1", &res);
+
+  ASSERT_TRUE(status);
+  ASSERT_EQ(res, "v1,v2,v3");
+}
+
+TEST(StringAppendOperatorTest, SimpleDelimiterTest) {
+  auto db = OpenDb('|');
+  StringLists slists(db);
+
+  slists.Append("k1", "v1");
+  slists.Append("k1", "v2");
+  slists.Append("k1", "v3");
+
+  std::string res;
+  slists.Get("k1", &res);
+  ASSERT_EQ(res, "v1|v2|v3");
+}
+
+TEST(StringAppendOperatorTest, OneValueNoDelimiterTest) {
+  auto db = OpenDb('!');
+  StringLists slists(db);
+
+  slists.Append("random_key", "single_val");
+
+  std::string res;
+  slists.Get("random_key", &res);
+  ASSERT_EQ(res, "single_val");
+}
+
+TEST(StringAppendOperatorTest, VariousKeys) {
+  auto db = OpenDb('\n');
+  StringLists slists(db);
+
+  slists.Append("c", "asdasd");
+  slists.Append("a", "x");
+  slists.Append("b", "y");
+  slists.Append("a", "t");
+  slists.Append("a", "r");
+  slists.Append("b", "2");
+  slists.Append("c", "asdasd");
+
+  std::string a, b, c;
+  bool sa, sb, sc;
+  sa = slists.Get("a", &a);
+  sb = slists.Get("b", &b);
+  sc = slists.Get("c", &c);
+
+  ASSERT_TRUE(sa && sb && sc); // All three keys should have been found
+
+  ASSERT_EQ(a, "x\nt\nr");
+  ASSERT_EQ(b, "y\n2");
+  ASSERT_EQ(c, "asdasd\nasdasd");
+}
+
+// Generate semi random keys/words from a small distribution.
+TEST(StringAppendOperatorTest, RandomMixGetAppend) {
+  auto db = OpenDb(' ');
+  StringLists slists(db);
+
+  // Generate a list of random keys and values
+  const int kWordCount = 15;
+  std::string words[] = {"sdasd", "triejf", "fnjsdfn", "dfjisdfsf", "342839",
+                         "dsuha", "mabuais", "sadajsid", "jf9834hf", "2d9j89",
+                         "dj9823jd", "a", "dk02ed2dh", "$(jd4h984$(*", "mabz"};
+  const int kKeyCount = 6;
+  std::string keys[] = {"dhaiusdhu", "denidw", "daisda", "keykey", "muki",
+                        "shzassdianmd"};
+
+  // Will store a local copy of all data in order to verify correctness
+  std::map<std::string, std::string> parallel_copy;
+
+  // Generate a bunch of random queries (Append and Get)!
+  enum query_t  { APPEND_OP, GET_OP, NUM_OPS };
+  Random randomGen(1337);       //deterministic seed; always get same results!
+
+  const int kNumQueries = 30;
+  for (int q=0; q<kNumQueries; ++q) {
+    // Generate a random query (Append or Get) and random parameters
+    query_t query = (query_t)randomGen.Uniform((int)NUM_OPS);
+    std::string key = keys[randomGen.Uniform((int)kKeyCount)];
+    std::string word = words[randomGen.Uniform((int)kWordCount)];
+
+    // Apply the query and any checks.
+    if (query == APPEND_OP) {
+
+      // Apply the rocksdb test-harness Append defined above
+      slists.Append(key, word);  //apply the rocksdb append
+
+      // Apply the similar "Append" to the parallel copy
+      if (parallel_copy[key].size() > 0) {
+        parallel_copy[key] += " " + word;
+      } else {
+        parallel_copy[key] = word;
+      }
+
+    } else if (query == GET_OP) {
+      // Assumes that a non-existent key just returns <empty>
+      std::string res;
+      slists.Get(key, &res);
+      ASSERT_EQ(res, parallel_copy[key]);
+    }
+
+  }
+
+}
+
+TEST(StringAppendOperatorTest, BIGRandomMixGetAppend) {
+  auto db = OpenDb(' ');
+  StringLists slists(db);
+
+  // Generate a list of random keys and values
+  const int kWordCount = 15;
+  std::string words[] = {"sdasd", "triejf", "fnjsdfn", "dfjisdfsf", "342839",
+                         "dsuha", "mabuais", "sadajsid", "jf9834hf", "2d9j89",
+                         "dj9823jd", "a", "dk02ed2dh", "$(jd4h984$(*", "mabz"};
+  const int kKeyCount = 6;
+  std::string keys[] = {"dhaiusdhu", "denidw", "daisda", "keykey", "muki",
+                        "shzassdianmd"};
+
+  // Will store a local copy of all data in order to verify correctness
+  std::map<std::string, std::string> parallel_copy;
+
+  // Generate a bunch of random queries (Append and Get)!
+  enum query_t  { APPEND_OP, GET_OP, NUM_OPS };
+  Random randomGen(9138204);       // deterministic seed
+
+  const int kNumQueries = 1000;
+  for (int q=0; q<kNumQueries; ++q) {
+    // Generate a random query (Append or Get) and random parameters
+    query_t query = (query_t)randomGen.Uniform((int)NUM_OPS);
+    std::string key = keys[randomGen.Uniform((int)kKeyCount)];
+    std::string word = words[randomGen.Uniform((int)kWordCount)];
+
+    //Apply the query and any checks.
+    if (query == APPEND_OP) {
+
+      // Apply the rocksdb test-harness Append defined above
+      slists.Append(key, word);  //apply the rocksdb append
+
+      // Apply the similar "Append" to the parallel copy
+      if (parallel_copy[key].size() > 0) {
+        parallel_copy[key] += " " + word;
+      } else {
+        parallel_copy[key] = word;
+      }
+
+    } else if (query == GET_OP) {
+      // Assumes that a non-existent key just returns <empty>
+      std::string res;
+      slists.Get(key, &res);
+      ASSERT_EQ(res, parallel_copy[key]);
+    }
+
+  }
+
+}
+
+
+TEST(StringAppendOperatorTest, PersistentVariousKeys) {
+  // Perform the following operations in limited scope
+  {
+    auto db = OpenDb('\n');
+    StringLists slists(db);
+
+    slists.Append("c", "asdasd");
+    slists.Append("a", "x");
+    slists.Append("b", "y");
+    slists.Append("a", "t");
+    slists.Append("a", "r");
+    slists.Append("b", "2");
+    slists.Append("c", "asdasd");
+
+    std::string a, b, c;
+    slists.Get("a", &a);
+    slists.Get("b", &b);
+    slists.Get("c", &c);
+
+    ASSERT_EQ(a, "x\nt\nr");
+    ASSERT_EQ(b, "y\n2");
+    ASSERT_EQ(c, "asdasd\nasdasd");
+  }
+
+  // Reopen the database (the previous changes should persist / be remembered)
+  {
+    auto db = OpenDb('\n');
+    StringLists slists(db);
+
+    slists.Append("c", "bbnagnagsx");
+    slists.Append("a", "sa");
+    slists.Append("b", "df");
+    slists.Append("a", "gh");
+    slists.Append("a", "jk");
+    slists.Append("b", "l;");
+    slists.Append("c", "rogosh");
+
+    // The previous changes should be on disk (L0)
+    // The most recent changes should be in memory (MemTable)
+    // Hence, this will test both Get() paths.
+    std::string a, b, c;
+    slists.Get("a", &a);
+    slists.Get("b", &b);
+    slists.Get("c", &c);
+
+    ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
+    ASSERT_EQ(b, "y\n2\ndf\nl;");
+    ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
+  }
+
+  // Reopen the database (the previous changes should persist / be remembered)
+  {
+    auto db = OpenDb('\n');
+    StringLists slists(db);
+
+    // All changes should be on disk. This will test VersionSet Get()
+    std::string a, b, c;
+    slists.Get("a", &a);
+    slists.Get("b", &b);
+    slists.Get("c", &c);
+
+    ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
+    ASSERT_EQ(b, "y\n2\ndf\nl;");
+    ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
+  }
+}
+
+TEST(StringAppendOperatorTest, PersistentFlushAndCompaction) {
+  // Perform the following operations in limited scope
+  {
+    auto db = OpenDb('\n');
+    StringLists slists(db);
+    std::string a, b, c;
+    bool success;
+
+    // Append, Flush, Get
+    slists.Append("c", "asdasd");
+    db->Flush(rocksdb::FlushOptions());
+    success = slists.Get("c", &c);
+    ASSERT_TRUE(success);
+    ASSERT_EQ(c, "asdasd");
+
+    // Append, Flush, Append, Get
+    slists.Append("a", "x");
+    slists.Append("b", "y");
+    db->Flush(rocksdb::FlushOptions());
+    slists.Append("a", "t");
+    slists.Append("a", "r");
+    slists.Append("b", "2");
+
+    success = slists.Get("a", &a);
+    assert(success == true);
+    ASSERT_EQ(a, "x\nt\nr");
+
+    success = slists.Get("b", &b);
+    assert(success == true);
+    ASSERT_EQ(b, "y\n2");
+
+    // Append, Get
+    success = slists.Append("c", "asdasd");
+    assert(success);
+    success = slists.Append("b", "monkey");
+    assert(success);
+
+    // I omit the "assert(success)" checks here.
+    slists.Get("a", &a);
+    slists.Get("b", &b);
+    slists.Get("c", &c);
+
+    ASSERT_EQ(a, "x\nt\nr");
+    ASSERT_EQ(b, "y\n2\nmonkey");
+    ASSERT_EQ(c, "asdasd\nasdasd");
+  }
+
+  // Reopen the database (the previous changes should persist / be remembered)
+  {
+    auto db = OpenDb('\n');
+    StringLists slists(db);
+    std::string a, b, c;
+
+    // Get (Quick check for persistence of previous database)
+    slists.Get("a", &a);
+    ASSERT_EQ(a, "x\nt\nr");
+
+    //Append, Compact, Get
+    slists.Append("c", "bbnagnagsx");
+    slists.Append("a", "sa");
+    slists.Append("b", "df");
+    db->CompactRange(nullptr, nullptr);
+    slists.Get("a", &a);
+    slists.Get("b", &b);
+    slists.Get("c", &c);
+    ASSERT_EQ(a, "x\nt\nr\nsa");
+    ASSERT_EQ(b, "y\n2\nmonkey\ndf");
+    ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx");
+
+    // Append, Get
+    slists.Append("a", "gh");
+    slists.Append("a", "jk");
+    slists.Append("b", "l;");
+    slists.Append("c", "rogosh");
+    slists.Get("a", &a);
+    slists.Get("b", &b);
+    slists.Get("c", &c);
+    ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
+    ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;");
+    ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
+
+    // Compact, Get
+    db->CompactRange(nullptr, nullptr);
+    ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
+    ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;");
+    ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
+
+    // Append, Flush, Compact, Get
+    slists.Append("b", "afcg");
+    db->Flush(rocksdb::FlushOptions());
+    db->CompactRange(nullptr, nullptr);
+    slists.Get("b", &b);
+    ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;\nafcg");
+  }
+}
+
+TEST(StringAppendOperatorTest, SimpleTestNullDelimiter) {
+  auto db = OpenDb('\0');
+  StringLists slists(db);
+
+  slists.Append("k1", "v1");
+  slists.Append("k1", "v2");
+  slists.Append("k1", "v3");
+
+  std::string res;
+  bool status = slists.Get("k1", &res);
+  ASSERT_TRUE(status);
+
+  // Construct the desired string. Default constructor doesn't like '\0' chars.
+  std::string checker("v1,v2,v3");    // Verify that the string is right size.
+  checker[2] = '\0';                  // Use null delimiter instead of comma.
+  checker[5] = '\0';
+  assert(checker.size() == 8);        // Verify it is still the correct size
+
+  // Check that the rocksdb result string matches the desired string
+  assert(res.size() == checker.size());
+  ASSERT_EQ(res, checker);
+}
+
+} // namespace rocksdb
+
+int main(int arc, char** argv) {
+  // Run with regular database
+  {
+    fprintf(stderr, "Running tests with regular db and operator.\n");
+    StringAppendOperatorTest::SetOpenDbFunction(&OpenNormalDb);
+    rocksdb::test::RunAllTests();
+  }
+
+  // Run with TTL
+  {
+    fprintf(stderr, "Running tests with ttl db and generic operator.\n");
+    StringAppendOperatorTest::SetOpenDbFunction(&OpenTtlDb);
+    rocksdb::test::RunAllTests();
+  }
+
+  return 0;
+}

utilities/merge_operators/uint64add.cc

@@ -0,0 +1,65 @@
+#include <memory>
+#include "rocksdb/env.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/slice.h"
+#include "util/coding.h"
+#include "utilities/merge_operators.h"
+
+using namespace rocksdb;
+
+namespace { // anonymous namespace
+
+// A 'model' merge operator with uint64 addition semantics
+// Implemented as an AssociativeMergeOperator for simplicity and example.
+class UInt64AddOperator : public AssociativeMergeOperator {
+ public:
+  virtual bool Merge(const Slice& key,
+                     const Slice* existing_value,
+                     const Slice& value,
+                     std::string* new_value,
+                     Logger* logger) const override {
+    uint64_t orig_value = 0;
+    if (existing_value){
+      orig_value = DecodeInteger(*existing_value, logger);
+    }
+    uint64_t operand = DecodeInteger(value, logger);
+
+    assert(new_value);
+    new_value->clear();
+    PutFixed64(new_value, orig_value + operand);
+
+    return true;  // Return true always since corruption will be treated as 0
+  }
+
+  virtual const char* Name() const override {
+    return "UInt64AddOperator";
+  }
+
+ private:
+  // Takes the string and decodes it into a uint64_t
+  // On error, prints a message and returns 0
+  uint64_t DecodeInteger(const Slice& value, Logger* logger) const {
+    uint64_t result = 0;
+
+    if (value.size() == sizeof(uint64_t)) {
+      result = DecodeFixed64(value.data());
+    } else if (logger != nullptr) {
+      // If value is corrupted, treat it as 0
+      Log(logger, "uint64 value corruption, size: %zu > %zu",
+          value.size(), sizeof(uint64_t));
+    }
+
+    return result;
+  }
+
+};
+
+}
+
+namespace rocksdb {
+
+std::shared_ptr<MergeOperator> MergeOperators::CreateUInt64AddOperator() {
+  return std::make_shared<UInt64AddOperator>();
+}
+
+}

utilities/redis/README

@@ -0,0 +1,14 @@
+This folder defines a REDIS-style interface for Rocksdb.
+Right now it is written as a simple tag-on in the rocksdb::RedisLists class.
+It implements Redis Lists, and supports only the "non-blocking operations".
+
+Internally, the set of lists are stored in a rocksdb database, mapping keys to
+values. Each "value" is the list itself, storing a sequence of "elements".
+Each element is stored as a 32-bit-integer, followed by a sequence of bytes.
+The 32-bit-integer represents the length of the element (that is, the number
+of bytes that follow). And then that many bytes follow.
+
+
+NOTE: This README file may be old. See the actual redis_lists.cc file for
+definitive details on the implementation. There should be a header at the top
+of that file, explaining a bit of the implementation details.

utilities/redis/redis_list_exception.h

@@ -0,0 +1,22 @@
+/**
+ * A simple structure for exceptions in RedisLists.
+ *
+ * @author Deon Nicholas (dnicholas@fb.com)
+ * Copyright 2013 Facebook
+ */
+
+#ifndef ROCKSDB_LITE
+#pragma once
+#include <exception>
+
+namespace rocksdb {
+
+class RedisListException: public std::exception {
+ public:
+  const char* what() const throw() {
+    return "Invalid operation or corrupt data in Redis List.";
+  }
+};
+
+} // namespace rocksdb
+#endif

utilities/redis/redis_list_iterator.h

@@ -0,0 +1,310 @@
+// Copyright 2013 Facebook
+/**
+ * RedisListIterator:
+ * An abstraction over the "list" concept (e.g.: for redis lists).
+ * Provides functionality to read, traverse, edit, and write these lists.
+ *
+ * Upon construction, the RedisListIterator is given a block of list data.
+ * Internally, it stores a pointer to the data and a pointer to current item.
+ * It also stores a "result" list that will be mutated over time.
+ *
+ * Traversal and mutation are done by "forward iteration".
+ * The Push() and Skip() methods will advance the iterator to the next item.
+ * However, Push() will also "write the current item to the result".
+ * Skip() will simply move to next item, causing current item to be dropped.
+ *
+ * Upon completion, the result (accessible by WriteResult()) will be saved.
+ * All "skipped" items will be gone; all "pushed" items will remain.
+ *
+ * @throws Any of the operations may throw a RedisListException if an invalid
+ *          operation is performed or if the data is found to be corrupt.
+ *
+ * @notes By default, if WriteResult() is called part-way through iteration,
+ *        it will automatically advance the iterator to the end, and Keep()
+ *        all items that haven't been traversed yet. This may be subject
+ *        to review.
+ *
+ * @notes Can access the "current" item via GetCurrent(), and other
+ *        list-specific information such as Length().
+ *
+ * @notes The internal representation is due to change at any time. Presently,
+ *        the list is represented as follows:
+ *          - 32-bit integer header: the number of items in the list
+ *          - For each item:
+ *              - 32-bit int (n): the number of bytes representing this item
+ *              - n bytes of data: the actual data.
+ *
+ * @author Deon Nicholas (dnicholas@fb.com)
+ */
+
+#ifndef ROCKSDB_LITE
+#pragma once
+
+#include <string>
+
+#include "redis_list_exception.h"
+#include "rocksdb/slice.h"
+#include "util/coding.h"
+
+namespace rocksdb {
+
+/// An abstraction over the "list" concept.
+/// All operations may throw a RedisListException
+class RedisListIterator {
+ public:
+  /// Construct a redis-list-iterator based on data.
+  /// If the data is non-empty, it must formatted according to @notes above.
+  ///
+  /// If the data is valid, we can assume the following invariant(s):
+  ///  a) length_, num_bytes_ are set correctly.
+  ///  b) cur_byte_ always refers to the start of the current element,
+  ///       just before the bytes that specify element length.
+  ///  c) cur_elem_ is always the index of the current element.
+  ///  d) cur_elem_length_ is always the number of bytes in current element,
+  ///       excluding the 4-byte header itself.
+  ///  e) result_ will always contain data_[0..cur_byte_) and a header
+  ///  f) Whenever corrupt data is encountered or an invalid operation is
+  ///      attempted, a RedisListException will immediately be thrown.
+  RedisListIterator(const std::string& list_data)
+      : data_(list_data.data()),
+        num_bytes_(list_data.size()),
+        cur_byte_(0),
+        cur_elem_(0),
+        cur_elem_length_(0),
+        length_(0),
+        result_() {
+
+    // Initialize the result_ (reserve enough space for header)
+    InitializeResult();
+
+    // Parse the data only if it is not empty.
+    if (num_bytes_ == 0) {
+      return;
+    }
+
+    // If non-empty, but less than 4 bytes, data must be corrupt
+    if (num_bytes_ < sizeof(length_)) {
+      ThrowError("Corrupt header.");    // Will break control flow
+    }
+
+    // Good. The first bytes specify the number of elements
+    length_ = DecodeFixed32(data_);
+    cur_byte_ = sizeof(length_);
+
+    // If we have at least one element, point to that element.
+    // Also, read the first integer of the element (specifying the size),
+    //   if possible.
+    if (length_ > 0) {
+      if (cur_byte_ + sizeof(cur_elem_length_) <= num_bytes_) {
+        cur_elem_length_ = DecodeFixed32(data_+cur_byte_);
+      } else {
+        ThrowError("Corrupt data for first element.");
+      }
+    }
+
+    // At this point, we are fully set-up.
+    // The invariants described in the header should now be true.
+  }
+
+  /// Reserve some space for the result_.
+  /// Equivalent to result_.reserve(bytes).
+  void Reserve(int bytes) {
+    result_.reserve(bytes);
+  }
+
+  /// Go to next element in data file.
+  /// Also writes the current element to result_.
+  RedisListIterator& Push() {
+    WriteCurrentElement();
+    MoveNext();
+    return *this;
+  }
+
+  /// Go to next element in data file.
+  /// Drops/skips the current element. It will not be written to result_.
+  RedisListIterator& Skip() {
+    MoveNext();
+    --length_;          // One less item
+    --cur_elem_;        // We moved one forward, but index did not change
+    return *this;
+  }
+
+  /// Insert elem into the result_ (just BEFORE the current element / byte)
+  /// Note: if Done() (i.e.: iterator points to end), this will append elem.
+  void InsertElement(const Slice& elem) {
+    // Ensure we are in a valid state
+    CheckErrors();
+
+    const int kOrigSize = result_.size();
+    result_.resize(kOrigSize + SizeOf(elem));
+    EncodeFixed32(result_.data() + kOrigSize, elem.size());
+    memcpy(result_.data() + kOrigSize + sizeof(uint32_t),
+           elem.data(),
+           elem.size());
+    ++length_;
+    ++cur_elem_;
+  }
+
+  /// Access the current element, and save the result into *curElem
+  void GetCurrent(Slice* curElem) {
+    // Ensure we are in a valid state
+    CheckErrors();
+
+    // Ensure that we are not past the last element.
+    if (Done()) {
+      ThrowError("Invalid dereferencing.");
+    }
+
+    // Dereference the element
+    *curElem = Slice(data_+cur_byte_+sizeof(cur_elem_length_),
+                     cur_elem_length_);
+  }
+
+  // Number of elements
+  int Length() const {
+    return length_;
+  }
+
+  // Number of bytes in the final representation (i.e: WriteResult().size())
+  int Size() const {
+    // result_ holds the currently written data
+    // data_[cur_byte..num_bytes-1] is the remainder of the data
+    return result_.size() + (num_bytes_ - cur_byte_);
+  }
+
+  // Reached the end?
+  bool Done() const {
+    return cur_byte_ >= num_bytes_ || cur_elem_ >= length_;
+  }
+
+  /// Returns a string representing the final, edited, data.
+  /// Assumes that all bytes of data_ in the range [0,cur_byte_) have been read
+  ///  and that result_ contains this data.
+  /// The rest of the data must still be written.
+  /// So, this method ADVANCES THE ITERATOR TO THE END before writing.
+  Slice WriteResult() {
+    CheckErrors();
+
+    // The header should currently be filled with dummy data (0's)
+    // Correctly update the header.
+    // Note, this is safe since result_ is a vector (guaranteed contiguous)
+    EncodeFixed32(&result_[0],length_);
+
+    // Append the remainder of the data to the result.
+    result_.insert(result_.end(),data_+cur_byte_, data_ +num_bytes_);
+
+    // Seek to end of file
+    cur_byte_ = num_bytes_;
+    cur_elem_ = length_;
+    cur_elem_length_ = 0;
+
+    // Return the result
+    return Slice(result_.data(),result_.size());
+  }
+
+ public: // Static public functions
+
+  /// An upper-bound on the amount of bytes needed to store this element.
+  /// This is used to hide representation information from the client.
+  /// E.G. This can be used to compute the bytes we want to Reserve().
+  static uint32_t SizeOf(const Slice& elem) {
+    // [Integer Length . Data]
+    return sizeof(uint32_t) + elem.size();
+  }
+
+ private: // Private functions
+
+  /// Initializes the result_ string.
+  /// It will fill the first few bytes with 0's so that there is
+  ///  enough space for header information when we need to write later.
+  /// Currently, "header information" means: the length (number of elements)
+  /// Assumes that result_ is empty to begin with
+  void InitializeResult() {
+    assert(result_.empty());            // Should always be true.
+    result_.resize(sizeof(uint32_t),0); // Put a block of 0's as the header
+  }
+
+  /// Go to the next element (used in Push() and Skip())
+  void MoveNext() {
+    CheckErrors();
+
+    // Check to make sure we are not already in a finished state
+    if (Done()) {
+      ThrowError("Attempting to iterate past end of list.");
+    }
+
+    // Move forward one element.
+    cur_byte_ += sizeof(cur_elem_length_) + cur_elem_length_;
+    ++cur_elem_;
+
+    // If we are at the end, finish
+    if (Done()) {
+      cur_elem_length_ = 0;
+      return;
+    }
+
+    // Otherwise, we should be able to read the new element's length
+    if (cur_byte_ + sizeof(cur_elem_length_) > num_bytes_) {
+      ThrowError("Corrupt element data.");
+    }
+
+    // Set the new element's length
+    cur_elem_length_ = DecodeFixed32(data_+cur_byte_);
+
+    return;
+  }
+
+  /// Append the current element (pointed to by cur_byte_) to result_
+  /// Assumes result_ has already been reserved appropriately.
+  void WriteCurrentElement() {
+    // First verify that the iterator is still valid.
+    CheckErrors();
+    if (Done()) {
+      ThrowError("Attempting to write invalid element.");
+    }
+
+    // Append the cur element.
+    result_.insert(result_.end(),
+                   data_+cur_byte_,
+                   data_+cur_byte_+ sizeof(uint32_t) + cur_elem_length_);
+  }
+
+  /// Will ThrowError() if neccessary.
+  /// Checks for common/ubiquitous errors that can arise after most operations.
+  /// This method should be called before any reading operation.
+  /// If this function succeeds, then we are guaranteed to be in a valid state.
+  /// Other member functions should check for errors and ThrowError() also
+  ///  if an error occurs that is specific to it even while in a valid state.
+  void CheckErrors() {
+    // Check if any crazy thing has happened recently
+    if ((cur_elem_ > length_) ||                              // Bad index
+        (cur_byte_ > num_bytes_) ||                           // No more bytes
+        (cur_byte_ + cur_elem_length_ > num_bytes_) ||        // Item too large
+        (cur_byte_ == num_bytes_ && cur_elem_ != length_) ||  // Too many items
+        (cur_elem_ == length_ && cur_byte_ != num_bytes_)) {  // Too many bytes
+      ThrowError("Corrupt data.");
+    }
+  }
+
+  /// Will throw an exception based on the passed-in message.
+  /// This function is guaranteed to STOP THE CONTROL-FLOW.
+  /// (i.e.: you do not have to call "return" after calling ThrowError)
+  void ThrowError(const char* const msg = NULL) {
+    // TODO: For now we ignore the msg parameter. This can be expanded later.
+    throw RedisListException();
+  }
+
+ private:
+  const char* const data_;      // A pointer to the data (the first byte)
+  const uint32_t num_bytes_;    // The number of bytes in this list
+
+  uint32_t cur_byte_;           // The current byte being read
+  uint32_t cur_elem_;           // The current element being read
+  uint32_t cur_elem_length_;    // The number of bytes in current element
+
+  uint32_t length_;             // The number of elements in this list
+  std::vector<char> result_;    // The output data
+};
+
+} // namespace rocksdb
+#endif  // ROCKSDB_LITE

utilities/redis/redis_lists.cc

@@ -0,0 +1,552 @@
+// Copyright 2013 Facebook
+/**
+ * A (persistent) Redis API built using the rocksdb backend.
+ * Implements Redis Lists as described on: http://redis.io/commands#list
+ *
+ * @throws All functions may throw a RedisListException on error/corruption.
+ *
+ * @notes Internally, the set of lists is stored in a rocksdb database,
+ *        mapping keys to values. Each "value" is the list itself, storing
+ *        some kind of internal representation of the data. All the
+ *        representation details are handled by the RedisListIterator class.
+ *        The present file should be oblivious to the representation details,
+ *        handling only the client (Redis) API, and the calls to rocksdb.
+ *
+ * @TODO  Presently, all operations take at least O(NV) time where
+ *        N is the number of elements in the list, and V is the average
+ *        number of bytes per value in the list. So maybe, with merge operator
+ *        we can improve this to an optimal O(V) amortized time, since we
+ *        wouldn't have to read and re-write the entire list.
+ *
+ * @author Deon Nicholas (dnicholas@fb.com)
+ */
+
+#ifndef ROCKSDB_LITE
+#include "redis_lists.h"
+
+#include <iostream>
+#include <memory>
+#include <cmath>
+
+#include "rocksdb/slice.h"
+#include "util/coding.h"
+
+namespace rocksdb
+{
+
+/// Constructors
+
+RedisLists::RedisLists(const std::string& db_path,
+                       Options options, bool destructive)
+    : put_option_(),
+      get_option_() {
+
+  // Store the name of the database
+  db_name_ = db_path;
+
+  // If destructive, destroy the DB before re-opening it.
+  if (destructive) {
+    DestroyDB(db_name_, Options());
+  }
+
+  // Now open and deal with the db
+  DB* db;
+  Status s = DB::Open(options, db_name_, &db);
+  if (!s.ok()) {
+    std::cerr << "ERROR " << s.ToString() << std::endl;
+    assert(false);
+  }
+
+  db_ = std::unique_ptr<DB>(db);
+}
+
+
+/// Accessors
+
+// Number of elements in the list associated with key
+//   : throws RedisListException
+int RedisLists::Length(const std::string& key) {
+  // Extract the string data representing the list.
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Return the length
+  RedisListIterator it(data);
+  return it.Length();
+}
+
+// Get the element at the specified index in the (list: key)
+// Returns <empty> ("") on out-of-bounds
+//   : throws RedisListException
+bool RedisLists::Index(const std::string& key, int32_t index,
+                       std::string* result) {
+  // Extract the string data representing the list.
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Handle REDIS negative indices (from the end); fast iff Length() takes O(1)
+  if (index < 0) {
+    index = Length(key) - (-index);  //replace (-i) with (N-i).
+  }
+
+  // Iterate through the list until the desired index is found.
+  int curIndex = 0;
+  RedisListIterator it(data);
+  while(curIndex < index && !it.Done()) {
+    ++curIndex;
+    it.Skip();
+  }
+
+  // If we actually found the index
+  if (curIndex == index && !it.Done()) {
+    Slice elem;
+    it.GetCurrent(&elem);
+    if (result != NULL) {
+      *result = elem.ToString();
+    }
+
+    return true;
+  } else {
+    return false;
+  }
+}
+
+// Return a truncated version of the list.
+// First, negative values for first/last are interpreted as "end of list".
+// So, if first == -1, then it is re-set to index: (Length(key) - 1)
+// Then, return exactly those indices i such that first <= i <= last.
+//   : throws RedisListException
+std::vector<std::string> RedisLists::Range(const std::string& key,
+                                           int32_t first, int32_t last) {
+  // Extract the string data representing the list.
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Handle negative bounds (-1 means last element, etc.)
+  int listLen = Length(key);
+  if (first < 0) {
+    first = listLen - (-first);           // Replace (-x) with (N-x)
+  }
+  if (last < 0) {
+    last = listLen - (-last);
+  }
+
+  // Verify bounds (and truncate the range so that it is valid)
+  first = std::max(first, 0);
+  last = std::min(last, listLen-1);
+  int len = std::max(last-first+1, 0);
+
+  // Initialize the resulting list
+  std::vector<std::string> result(len);
+
+  // Traverse the list and update the vector
+  int curIdx = 0;
+  Slice elem;
+  for (RedisListIterator it(data); !it.Done() && curIdx<=last; it.Skip()) {
+    if (first <= curIdx && curIdx <= last) {
+      it.GetCurrent(&elem);
+      result[curIdx-first].assign(elem.data(),elem.size());
+    }
+
+    ++curIdx;
+  }
+
+  // Return the result. Might be empty
+  return result;
+}
+
+// Print the (list: key) out to stdout. For debugging mostly. Public for now.
+void RedisLists::Print(const std::string& key) {
+  // Extract the string data representing the list.
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Iterate through the list and print the items
+  Slice elem;
+  for (RedisListIterator it(data); !it.Done(); it.Skip()) {
+    it.GetCurrent(&elem);
+    std::cout << "ITEM " << elem.ToString() << std::endl;
+  }
+
+  //Now print the byte data
+  RedisListIterator it(data);
+  std::cout << "==Printing data==" << std::endl;
+  std::cout << data.size() << std::endl;
+  std::cout << it.Size() << " " << it.Length() << std::endl;
+  Slice result = it.WriteResult();
+  std::cout << result.data() << std::endl;
+  if (true) {
+    std::cout << "size: " << result.size() << std::endl;
+    const char* val = result.data();
+    for(int i=0; i<(int)result.size(); ++i) {
+      std::cout << (int)val[i] << " " << (val[i]>=32?val[i]:' ') << std::endl;
+    }
+    std::cout << std::endl;
+  }
+}
+
+/// Insert/Update Functions
+/// Note: The "real" insert function is private. See below.
+
+// InsertBefore and InsertAfter are simply wrappers around the Insert function.
+int RedisLists::InsertBefore(const std::string& key, const std::string& pivot,
+                             const std::string& value) {
+  return Insert(key, pivot, value, false);
+}
+
+int RedisLists::InsertAfter(const std::string& key, const std::string& pivot,
+                            const std::string& value) {
+  return Insert(key, pivot, value, true);
+}
+
+// Prepend value onto beginning of (list: key)
+//   : throws RedisListException
+int RedisLists::PushLeft(const std::string& key, const std::string& value) {
+  // Get the original list data
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Construct the result
+  RedisListIterator it(data);
+  it.Reserve(it.Size() + it.SizeOf(value));
+  it.InsertElement(value);
+
+  // Push the data back to the db and return the length
+  db_->Put(put_option_, key, it.WriteResult());
+  return it.Length();
+}
+
+// Append value onto end of (list: key)
+// TODO: Make this O(1) time. Might require MergeOperator.
+//   : throws RedisListException
+int RedisLists::PushRight(const std::string& key, const std::string& value) {
+  // Get the original list data
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Create an iterator to the data and seek to the end.
+  RedisListIterator it(data);
+  it.Reserve(it.Size() + it.SizeOf(value));
+  while (!it.Done()) {
+    it.Push();    // Write each element as we go
+  }
+
+  // Insert the new element at the current position (the end)
+  it.InsertElement(value);
+
+  // Push it back to the db, and return length
+  db_->Put(put_option_, key, it.WriteResult());
+  return it.Length();
+}
+
+// Set (list: key)[idx] = val. Return true on success, false on fail.
+//   : throws RedisListException
+bool RedisLists::Set(const std::string& key, int32_t index,
+                     const std::string& value) {
+  // Get the original list data
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Handle negative index for REDIS (meaning -index from end of list)
+  if (index < 0) {
+    index = Length(key) - (-index);
+  }
+
+  // Iterate through the list until we find the element we want
+  int curIndex = 0;
+  RedisListIterator it(data);
+  it.Reserve(it.Size() + it.SizeOf(value));  // Over-estimate is fine
+  while(curIndex < index && !it.Done()) {
+    it.Push();
+    ++curIndex;
+  }
+
+  // If not found, return false (this occurs when index was invalid)
+  if (it.Done() || curIndex != index) {
+    return false;
+  }
+
+  // Write the new element value, and drop the previous element value
+  it.InsertElement(value);
+  it.Skip();
+
+  // Write the data to the database
+  // Check status, since it needs to return true/false guarantee
+  Status s = db_->Put(put_option_, key, it.WriteResult());
+
+  // Success
+  return s.ok();
+}
+
+/// Delete / Remove / Pop functions
+
+// Trim (list: key) so that it will only contain the indices from start..stop
+//  Invalid indices will not generate an error, just empty,
+//  or the portion of the list that fits in this interval
+//   : throws RedisListException
+bool RedisLists::Trim(const std::string& key, int32_t start, int32_t stop) {
+  // Get the original list data
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Handle negative indices in REDIS
+  int listLen = Length(key);
+  if (start < 0) {
+    start = listLen - (-start);
+  }
+  if (stop < 0) {
+    stop = listLen - (-stop);
+  }
+
+  // Truncate bounds to only fit in the list
+  start = std::max(start, 0);
+  stop = std::min(stop, listLen-1);
+
+  // Construct an iterator for the list. Drop all undesired elements.
+  int curIndex = 0;
+  RedisListIterator it(data);
+  it.Reserve(it.Size());          // Over-estimate
+  while(!it.Done()) {
+    // If not within the range, just skip the item (drop it).
+    // Otherwise, continue as usual.
+    if (start <= curIndex && curIndex <= stop) {
+      it.Push();
+    } else {
+      it.Skip();
+    }
+
+    // Increment the current index
+    ++curIndex;
+  }
+
+  // Write the (possibly empty) result to the database
+  Status s = db_->Put(put_option_, key, it.WriteResult());
+
+  // Return true as long as the write succeeded
+  return s.ok();
+}
+
+// Return and remove the first element in the list (or "" if empty)
+//   : throws RedisListException
+bool RedisLists::PopLeft(const std::string& key, std::string* result) {
+  // Get the original list data
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Point to first element in the list (if it exists), and get its value/size
+  RedisListIterator it(data);
+  if (it.Length() > 0) {            // Proceed only if list is non-empty
+    Slice elem;
+    it.GetCurrent(&elem);           // Store the value of the first element
+    it.Reserve(it.Size() - it.SizeOf(elem));
+    it.Skip();                      // DROP the first item and move to next
+
+    // Update the db
+    db_->Put(put_option_, key, it.WriteResult());
+
+    // Return the value
+    if (result != NULL) {
+      *result = elem.ToString();
+    }
+    return true;
+  } else {
+    return false;
+  }
+}
+
+// Remove and return the last element in the list (or "" if empty)
+// TODO: Make this O(1). Might require MergeOperator.
+//   : throws RedisListException
+bool RedisLists::PopRight(const std::string& key, std::string* result) {
+  // Extract the original list data
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Construct an iterator to the data and move to last element
+  RedisListIterator it(data);
+  it.Reserve(it.Size());
+  int len = it.Length();
+  int curIndex = 0;
+  while(curIndex < (len-1) && !it.Done()) {
+    it.Push();
+    ++curIndex;
+  }
+
+  // Extract and drop/skip the last element
+  if (curIndex == len-1) {
+    assert(!it.Done());         // Sanity check. Should not have ended here.
+
+    // Extract and pop the element
+    Slice elem;
+    it.GetCurrent(&elem);       // Save value of element.
+    it.Skip();                  // Skip the element
+
+    // Write the result to the database
+    db_->Put(put_option_, key, it.WriteResult());
+
+    // Return the value
+    if (result != NULL) {
+      *result = elem.ToString();
+    }
+    return true;
+  } else {
+    // Must have been an empty list
+    assert(it.Done() && len==0 && curIndex == 0);
+    return false;
+  }
+}
+
+// Remove the (first or last) "num" occurrences of value in (list: key)
+//   : throws RedisListException
+int RedisLists::Remove(const std::string& key, int32_t num,
+                       const std::string& value) {
+  // Negative num ==> RemoveLast; Positive num ==> Remove First
+  if (num < 0) {
+    return RemoveLast(key, -num, value);
+  } else if (num > 0) {
+    return RemoveFirst(key, num, value);
+  } else {
+    return RemoveFirst(key, Length(key), value);
+  }
+}
+
+// Remove the first "num" occurrences of value in (list: key).
+//   : throws RedisListException
+int RedisLists::RemoveFirst(const std::string& key, int32_t num,
+                            const std::string& value) {
+  // Ensure that the number is positive
+  assert(num >= 0);
+
+  // Extract the original list data
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Traverse the list, appending all but the desired occurrences of value
+  int numSkipped = 0;         // Keep track of the number of times value is seen
+  Slice elem;
+  RedisListIterator it(data);
+  it.Reserve(it.Size());
+  while (!it.Done()) {
+    it.GetCurrent(&elem);
+
+    if (elem == value && numSkipped < num) {
+      // Drop this item if desired
+      it.Skip();
+      ++numSkipped;
+    } else {
+      // Otherwise keep the item and proceed as normal
+      it.Push();
+    }
+  }
+
+  // Put the result back to the database
+  db_->Put(put_option_, key, it.WriteResult());
+
+  // Return the number of elements removed
+  return numSkipped;
+}
+
+
+// Remove the last "num" occurrences of value in (list: key).
+// TODO: I traverse the list 2x. Make faster. Might require MergeOperator.
+//   : throws RedisListException
+int RedisLists::RemoveLast(const std::string& key, int32_t num,
+                           const std::string& value) {
+  // Ensure that the number is positive
+  assert(num >= 0);
+
+  // Extract the original list data
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Temporary variable to hold the "current element" in the blocks below
+  Slice elem;
+
+  // Count the total number of occurrences of value
+  int totalOccs = 0;
+  for (RedisListIterator it(data); !it.Done(); it.Skip()) {
+    it.GetCurrent(&elem);
+    if (elem == value) {
+      ++totalOccs;
+    }
+  }
+
+  // Construct an iterator to the data. Reserve enough space for the result.
+  RedisListIterator it(data);
+  int bytesRemoved = std::min(num,totalOccs)*it.SizeOf(value);
+  it.Reserve(it.Size() - bytesRemoved);
+
+  // Traverse the list, appending all but the desired occurrences of value.
+  // Note: "Drop the last k occurrences" is equivalent to
+  //  "keep only the first n-k occurrences", where n is total occurrences.
+  int numKept = 0;          // Keep track of the number of times value is kept
+  while(!it.Done()) {
+    it.GetCurrent(&elem);
+
+    // If we are within the deletion range and equal to value, drop it.
+    // Otherwise, append/keep/push it.
+    if (elem == value) {
+      if (numKept < totalOccs - num) {
+        it.Push();
+        ++numKept;
+      } else {
+        it.Skip();
+      }
+    } else {
+      // Always append the others
+      it.Push();
+    }
+  }
+
+  // Put the result back to the database
+  db_->Put(put_option_, key, it.WriteResult());
+
+  // Return the number of elements removed
+  return totalOccs - numKept;
+}
+
+/// Private functions
+
+// Insert element value into (list: key), right before/after
+//  the first occurrence of pivot
+//   : throws RedisListException
+int RedisLists::Insert(const std::string& key, const std::string& pivot,
+                       const std::string& value, bool insert_after) {
+  // Get the original list data
+  std::string data;
+  db_->Get(get_option_, key, &data);
+
+  // Construct an iterator to the data and reserve enough space for result.
+  RedisListIterator it(data);
+  it.Reserve(it.Size() + it.SizeOf(value));
+
+  // Iterate through the list until we find the element we want
+  Slice elem;
+  bool found = false;
+  while(!it.Done() && !found) {
+    it.GetCurrent(&elem);
+
+    // When we find the element, insert the element and mark found
+    if (elem == pivot) {                // Found it!
+      found = true;
+      if (insert_after == true) {       // Skip one more, if inserting after it
+        it.Push();
+      }
+      it.InsertElement(value);
+    } else {
+      it.Push();
+    }
+
+  }
+
+  // Put the data (string) into the database
+  if (found) {
+    db_->Put(put_option_, key, it.WriteResult());
+  }
+
+  // Returns the new (possibly unchanged) length of the list
+  return it.Length();
+}
+
+}  // namespace rocksdb
+#endif  // ROCKSDB_LITE

utilities/redis/redis_lists.h

@@ -0,0 +1,108 @@
+/**
+ * A (persistent) Redis API built using the rocksdb backend.
+ * Implements Redis Lists as described on: http://redis.io/commands#list
+ *
+ * @throws All functions may throw a RedisListException
+ *
+ * @author Deon Nicholas (dnicholas@fb.com)
+ * Copyright 2013 Facebook
+ */
+
+#ifndef ROCKSDB_LITE
+#pragma once
+
+#include <string>
+#include "rocksdb/db.h"
+#include "redis_list_iterator.h"
+#include "redis_list_exception.h"
+
+namespace rocksdb {
+
+/// The Redis functionality (see http://redis.io/commands#list)
+/// All functions may THROW a RedisListException
+class RedisLists {
+ public: // Constructors / Destructors
+  /// Construct a new RedisLists database, with name/path of db.
+  /// Will clear the database on open iff destructive is true (default false).
+  /// Otherwise, it will restore saved changes.
+  /// May throw RedisListException
+  RedisLists(const std::string& db_path,
+             Options options, bool destructive = false);
+
+ public:  // Accessors
+  /// The number of items in (list: key)
+  int Length(const std::string& key);
+
+  /// Search the list for the (index)'th item (0-based) in (list:key)
+  /// A negative index indicates: "from end-of-list"
+  /// If index is within range: return true, and return the value in *result.
+  /// If (index < -length OR index>=length), then index is out of range:
+  ///   return false (and *result is left unchanged)
+  /// May throw RedisListException
+  bool Index(const std::string& key, int32_t index,
+             std::string* result);
+
+  /// Return (list: key)[first..last] (inclusive)
+  /// May throw RedisListException
+  std::vector<std::string> Range(const std::string& key,
+                                 int32_t first, int32_t last);
+
+  /// Prints the entire (list: key), for debugging.
+  void Print(const std::string& key);
+
+ public: // Insert/Update
+  /// Insert value before/after pivot in (list: key). Return the length.
+  /// May throw RedisListException
+  int InsertBefore(const std::string& key, const std::string& pivot,
+                   const std::string& value);
+  int InsertAfter(const std::string& key, const std::string& pivot,
+                  const std::string& value);
+
+  /// Push / Insert value at beginning/end of the list. Return the length.
+  /// May throw RedisListException
+  int PushLeft(const std::string& key, const std::string& value);
+  int PushRight(const std::string& key, const std::string& value);
+
+  /// Set (list: key)[idx] = val. Return true on success, false on fail
+  /// May throw RedisListException
+  bool Set(const std::string& key, int32_t index, const std::string& value);
+
+ public: // Delete / Remove / Pop / Trim
+  /// Trim (list: key) so that it will only contain the indices from start..stop
+  /// Returns true on success
+  /// May throw RedisListException
+  bool Trim(const std::string& key, int32_t start, int32_t stop);
+
+  /// If list is empty, return false and leave *result unchanged.
+  /// Else, remove the first/last elem, store it in *result, and return true
+  bool PopLeft(const std::string& key, std::string* result);  // First
+  bool PopRight(const std::string& key, std::string* result); // Last
+
+  /// Remove the first (or last) num occurrences of value from the list (key)
+  /// Return the number of elements removed.
+  /// May throw RedisListException
+  int Remove(const std::string& key, int32_t num,
+             const std::string& value);
+  int RemoveFirst(const std::string& key, int32_t num,
+                  const std::string& value);
+  int RemoveLast(const std::string& key, int32_t num,
+                 const std::string& value);
+
+ private: // Private Functions
+  /// Calls InsertBefore or InsertAfter
+  int Insert(const std::string& key, const std::string& pivot,
+             const std::string& value, bool insert_after);
+ private:
+  std::string db_name_;       // The actual database name/path
+  WriteOptions put_option_;
+  ReadOptions get_option_;
+
+  /// The backend rocksdb database.
+  /// Map : key --> list
+  ///       where a list is a sequence of elements
+  ///       and an element is a 4-byte integer (n), followed by n bytes of data
+  std::unique_ptr<DB> db_;
+};
+
+} // namespace rocksdb
+#endif  // ROCKSDB_LITE

utilities/redis/redis_lists_test.cc

@@ -0,0 +1,884 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+/**
+ * A test harness for the Redis API built on rocksdb.
+ *
+ * USAGE: Build with: "make redis_test" (in rocksdb directory).
+ *        Run unit tests with: "./redis_test"
+ *        Manual/Interactive user testing: "./redis_test -m"
+ *        Manual user testing + restart database: "./redis_test -m -d"
+ *
+ * TODO:  Add LARGE random test cases to verify efficiency and scalability
+ *
+ * @author Deon Nicholas (dnicholas@fb.com)
+ */
+
+
+#include <iostream>
+#include <cctype>
+
+#include "redis_lists.h"
+#include "util/testharness.h"
+#include "util/random.h"
+
+using namespace rocksdb;
+using namespace std;
+
+namespace rocksdb {
+
+class RedisListsTest {
+ public:
+  static const string kDefaultDbName;
+  static Options options;
+
+  RedisListsTest() {
+    options.create_if_missing = true;
+  }
+};
+
+const string RedisListsTest::kDefaultDbName = "/tmp/redisdefaultdb/";
+Options RedisListsTest::options = Options();
+
+// operator== and operator<< are defined below for vectors (lists)
+// Needed for ASSERT_EQ
+
+namespace {
+void AssertListEq(const std::vector<std::string>& result,
+                  const std::vector<std::string>& expected_result) {
+  ASSERT_EQ(result.size(), expected_result.size());
+  for (size_t i = 0; i < result.size(); ++i) {
+    ASSERT_EQ(result[i], expected_result[i]);
+  }
+}
+}  // namespace
+
+// PushRight, Length, Index, Range
+TEST(RedisListsTest, SimpleTest) {
+  RedisLists redis(kDefaultDbName, options, true);   // Destructive
+
+  string tempv; // Used below for all Index(), PopRight(), PopLeft()
+
+  // Simple PushRight (should return the new length each time)
+  ASSERT_EQ(redis.PushRight("k1", "v1"), 1);
+  ASSERT_EQ(redis.PushRight("k1", "v2"), 2);
+  ASSERT_EQ(redis.PushRight("k1", "v3"), 3);
+
+  // Check Length and Index() functions
+  ASSERT_EQ(redis.Length("k1"), 3);        // Check length
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "v1");   // Check valid indices
+  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
+  ASSERT_EQ(tempv, "v2");
+  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
+  ASSERT_EQ(tempv, "v3");
+
+  // Check range function and vectors
+  std::vector<std::string> result = redis.Range("k1", 0, 2);   // Get the list
+  std::vector<std::string> expected_result(3);
+  expected_result[0] = "v1";
+  expected_result[1] = "v2";
+  expected_result[2] = "v3";
+  AssertListEq(result, expected_result);
+}
+
+// PushLeft, Length, Index, Range
+TEST(RedisListsTest, SimpleTest2) {
+  RedisLists redis(kDefaultDbName, options, true);   // Destructive
+
+  string tempv; // Used below for all Index(), PopRight(), PopLeft()
+
+  // Simple PushRight
+  ASSERT_EQ(redis.PushLeft("k1", "v3"), 1);
+  ASSERT_EQ(redis.PushLeft("k1", "v2"), 2);
+  ASSERT_EQ(redis.PushLeft("k1", "v1"), 3);
+
+  // Check Length and Index() functions
+  ASSERT_EQ(redis.Length("k1"), 3);        // Check length
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "v1");   // Check valid indices
+  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
+  ASSERT_EQ(tempv, "v2");
+  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
+  ASSERT_EQ(tempv, "v3");
+
+  // Check range function and vectors
+  std::vector<std::string> result = redis.Range("k1", 0, 2);   // Get the list
+  std::vector<std::string> expected_result(3);
+  expected_result[0] = "v1";
+  expected_result[1] = "v2";
+  expected_result[2] = "v3";
+  AssertListEq(result, expected_result);
+}
+
+// Exhaustive test of the Index() function
+TEST(RedisListsTest, IndexTest) {
+  RedisLists redis(kDefaultDbName, options, true);   // Destructive
+
+  string tempv; // Used below for all Index(), PopRight(), PopLeft()
+
+  // Empty Index check (return empty and should not crash or edit tempv)
+  tempv = "yo";
+  ASSERT_TRUE(!redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "yo");
+  ASSERT_TRUE(!redis.Index("fda", 3, &tempv));
+  ASSERT_EQ(tempv, "yo");
+  ASSERT_TRUE(!redis.Index("random", -12391, &tempv));
+  ASSERT_EQ(tempv, "yo");
+
+  // Simple Pushes (will yield: [v6, v4, v4, v1, v2, v3]
+  redis.PushRight("k1", "v1");
+  redis.PushRight("k1", "v2");
+  redis.PushRight("k1", "v3");
+  redis.PushLeft("k1", "v4");
+  redis.PushLeft("k1", "v4");
+  redis.PushLeft("k1", "v6");
+
+  // Simple, non-negative indices
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "v6");
+  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
+  ASSERT_EQ(tempv, "v4");
+  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
+  ASSERT_EQ(tempv, "v4");
+  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
+  ASSERT_EQ(tempv, "v1");
+  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
+  ASSERT_EQ(tempv, "v2");
+  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
+  ASSERT_EQ(tempv, "v3");
+
+  // Negative indices
+  ASSERT_TRUE(redis.Index("k1", -6, &tempv));
+  ASSERT_EQ(tempv, "v6");
+  ASSERT_TRUE(redis.Index("k1", -5, &tempv));
+  ASSERT_EQ(tempv, "v4");
+  ASSERT_TRUE(redis.Index("k1", -4, &tempv));
+  ASSERT_EQ(tempv, "v4");
+  ASSERT_TRUE(redis.Index("k1", -3, &tempv));
+  ASSERT_EQ(tempv, "v1");
+  ASSERT_TRUE(redis.Index("k1", -2, &tempv));
+  ASSERT_EQ(tempv, "v2");
+  ASSERT_TRUE(redis.Index("k1", -1, &tempv));
+  ASSERT_EQ(tempv, "v3");
+
+  // Out of bounds (return empty, no crash)
+  ASSERT_TRUE(!redis.Index("k1", 6, &tempv));
+  ASSERT_TRUE(!redis.Index("k1", 123219, &tempv));
+  ASSERT_TRUE(!redis.Index("k1", -7, &tempv));
+  ASSERT_TRUE(!redis.Index("k1", -129, &tempv));
+}
+
+
+// Exhaustive test of the Range() function
+TEST(RedisListsTest, RangeTest) {
+  RedisLists redis(kDefaultDbName, options, true);   // Destructive
+
+  string tempv; // Used below for all Index(), PopRight(), PopLeft()
+
+  // Simple Pushes (will yield: [v6, v4, v4, v1, v2, v3])
+  redis.PushRight("k1", "v1");
+  redis.PushRight("k1", "v2");
+  redis.PushRight("k1", "v3");
+  redis.PushLeft("k1", "v4");
+  redis.PushLeft("k1", "v4");
+  redis.PushLeft("k1", "v6");
+
+  // Sanity check (check the length;  make sure it's 6)
+  ASSERT_EQ(redis.Length("k1"), 6);
+
+  // Simple range
+  std::vector<std::string> res = redis.Range("k1", 1, 4);
+  ASSERT_EQ((int)res.size(), 4);
+  ASSERT_EQ(res[0], "v4");
+  ASSERT_EQ(res[1], "v4");
+  ASSERT_EQ(res[2], "v1");
+  ASSERT_EQ(res[3], "v2");
+
+  // Negative indices (i.e.: measured from the end)
+  res = redis.Range("k1", 2, -1);
+  ASSERT_EQ((int)res.size(), 4);
+  ASSERT_EQ(res[0], "v4");
+  ASSERT_EQ(res[1], "v1");
+  ASSERT_EQ(res[2], "v2");
+  ASSERT_EQ(res[3], "v3");
+
+  res = redis.Range("k1", -6, -4);
+  ASSERT_EQ((int)res.size(), 3);
+  ASSERT_EQ(res[0], "v6");
+  ASSERT_EQ(res[1], "v4");
+  ASSERT_EQ(res[2], "v4");
+
+  res = redis.Range("k1", -1, 5);
+  ASSERT_EQ((int)res.size(), 1);
+  ASSERT_EQ(res[0], "v3");
+
+  // Partial / Broken indices
+  res = redis.Range("k1", -3, 1000000);
+  ASSERT_EQ((int)res.size(), 3);
+  ASSERT_EQ(res[0], "v1");
+  ASSERT_EQ(res[1], "v2");
+  ASSERT_EQ(res[2], "v3");
+
+  res = redis.Range("k1", -1000000, 1);
+  ASSERT_EQ((int)res.size(), 2);
+  ASSERT_EQ(res[0], "v6");
+  ASSERT_EQ(res[1], "v4");
+
+  // Invalid indices
+  res = redis.Range("k1", 7, 9);
+  ASSERT_EQ((int)res.size(), 0);
+
+  res = redis.Range("k1", -8, -7);
+  ASSERT_EQ((int)res.size(), 0);
+
+  res = redis.Range("k1", 3, 2);
+  ASSERT_EQ((int)res.size(), 0);
+
+  res = redis.Range("k1", 5, -2);
+  ASSERT_EQ((int)res.size(), 0);
+
+  // Range matches Index
+  res = redis.Range("k1", -6, -4);
+  ASSERT_TRUE(redis.Index("k1", -6, &tempv));
+  ASSERT_EQ(tempv, res[0]);
+  ASSERT_TRUE(redis.Index("k1", -5, &tempv));
+  ASSERT_EQ(tempv, res[1]);
+  ASSERT_TRUE(redis.Index("k1", -4, &tempv));
+  ASSERT_EQ(tempv, res[2]);
+
+  // Last check
+  res = redis.Range("k1", 0, -6);
+  ASSERT_EQ((int)res.size(), 1);
+  ASSERT_EQ(res[0], "v6");
+}
+
+// Exhaustive test for InsertBefore(), and InsertAfter()
+TEST(RedisListsTest, InsertTest) {
+  RedisLists redis(kDefaultDbName, options, true);
+
+  string tempv; // Used below for all Index(), PopRight(), PopLeft()
+
+  // Insert on empty list (return 0, and do not crash)
+  ASSERT_EQ(redis.InsertBefore("k1", "non-exist", "a"), 0);
+  ASSERT_EQ(redis.InsertAfter("k1", "other-non-exist", "c"), 0);
+  ASSERT_EQ(redis.Length("k1"), 0);
+
+  // Push some preliminary stuff [g, f, e, d, c, b, a]
+  redis.PushLeft("k1", "a");
+  redis.PushLeft("k1", "b");
+  redis.PushLeft("k1", "c");
+  redis.PushLeft("k1", "d");
+  redis.PushLeft("k1", "e");
+  redis.PushLeft("k1", "f");
+  redis.PushLeft("k1", "g");
+  ASSERT_EQ(redis.Length("k1"), 7);
+
+  // Test InsertBefore
+  int newLength = redis.InsertBefore("k1", "e", "hello");
+  ASSERT_EQ(newLength, 8);
+  ASSERT_EQ(redis.Length("k1"), newLength);
+  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
+  ASSERT_EQ(tempv, "f");
+  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
+  ASSERT_EQ(tempv, "e");
+  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
+  ASSERT_EQ(tempv, "hello");
+
+  // Test InsertAfter
+  newLength =  redis.InsertAfter("k1", "c", "bye");
+  ASSERT_EQ(newLength, 9);
+  ASSERT_EQ(redis.Length("k1"), newLength);
+  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
+  ASSERT_EQ(tempv, "bye");
+
+  // Test bad value on InsertBefore
+  newLength = redis.InsertBefore("k1", "yo", "x");
+  ASSERT_EQ(newLength, 9);
+  ASSERT_EQ(redis.Length("k1"), newLength);
+
+  // Test bad value on InsertAfter
+  newLength = redis.InsertAfter("k1", "xxxx", "y");
+  ASSERT_EQ(newLength, 9);
+  ASSERT_EQ(redis.Length("k1"), newLength);
+
+  // Test InsertBefore beginning
+  newLength = redis.InsertBefore("k1", "g", "begggggggggggggggg");
+  ASSERT_EQ(newLength, 10);
+  ASSERT_EQ(redis.Length("k1"), newLength);
+
+  // Test InsertAfter end
+  newLength = redis.InsertAfter("k1", "a", "enddd");
+  ASSERT_EQ(newLength, 11);
+  ASSERT_EQ(redis.Length("k1"), newLength);
+
+  // Make sure nothing weird happened.
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "begggggggggggggggg");
+  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
+  ASSERT_EQ(tempv, "g");
+  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
+  ASSERT_EQ(tempv, "f");
+  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
+  ASSERT_EQ(tempv, "hello");
+  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
+  ASSERT_EQ(tempv, "e");
+  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
+  ASSERT_EQ(tempv, "d");
+  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
+  ASSERT_EQ(tempv, "c");
+  ASSERT_TRUE(redis.Index("k1", 7, &tempv));
+  ASSERT_EQ(tempv, "bye");
+  ASSERT_TRUE(redis.Index("k1", 8, &tempv));
+  ASSERT_EQ(tempv, "b");
+  ASSERT_TRUE(redis.Index("k1", 9, &tempv));
+  ASSERT_EQ(tempv, "a");
+  ASSERT_TRUE(redis.Index("k1", 10, &tempv));
+  ASSERT_EQ(tempv, "enddd");
+}
+
+// Exhaustive test of Set function
+TEST(RedisListsTest, SetTest) {
+  RedisLists redis(kDefaultDbName, options, true);
+
+  string tempv; // Used below for all Index(), PopRight(), PopLeft()
+
+  // Set on empty list (return false, and do not crash)
+  ASSERT_EQ(redis.Set("k1", 7, "a"), false);
+  ASSERT_EQ(redis.Set("k1", 0, "a"), false);
+  ASSERT_EQ(redis.Set("k1", -49, "cx"), false);
+  ASSERT_EQ(redis.Length("k1"), 0);
+
+  // Push some preliminary stuff [g, f, e, d, c, b, a]
+  redis.PushLeft("k1", "a");
+  redis.PushLeft("k1", "b");
+  redis.PushLeft("k1", "c");
+  redis.PushLeft("k1", "d");
+  redis.PushLeft("k1", "e");
+  redis.PushLeft("k1", "f");
+  redis.PushLeft("k1", "g");
+  ASSERT_EQ(redis.Length("k1"), 7);
+
+  // Test Regular Set
+  ASSERT_TRUE(redis.Set("k1", 0, "0"));
+  ASSERT_TRUE(redis.Set("k1", 3, "3"));
+  ASSERT_TRUE(redis.Set("k1", 6, "6"));
+  ASSERT_TRUE(redis.Set("k1", 2, "2"));
+  ASSERT_TRUE(redis.Set("k1", 5, "5"));
+  ASSERT_TRUE(redis.Set("k1", 1, "1"));
+  ASSERT_TRUE(redis.Set("k1", 4, "4"));
+
+  ASSERT_EQ(redis.Length("k1"), 7); // Size should not change
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "0");
+  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
+  ASSERT_EQ(tempv, "1");
+  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
+  ASSERT_EQ(tempv, "2");
+  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
+  ASSERT_EQ(tempv, "3");
+  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
+  ASSERT_EQ(tempv, "4");
+  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
+  ASSERT_EQ(tempv, "5");
+  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
+  ASSERT_EQ(tempv, "6");
+
+  // Set with negative indices
+  ASSERT_TRUE(redis.Set("k1", -7, "a"));
+  ASSERT_TRUE(redis.Set("k1", -4, "d"));
+  ASSERT_TRUE(redis.Set("k1", -1, "g"));
+  ASSERT_TRUE(redis.Set("k1", -5, "c"));
+  ASSERT_TRUE(redis.Set("k1", -2, "f"));
+  ASSERT_TRUE(redis.Set("k1", -6, "b"));
+  ASSERT_TRUE(redis.Set("k1", -3, "e"));
+
+  ASSERT_EQ(redis.Length("k1"), 7); // Size should not change
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "a");
+  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
+  ASSERT_EQ(tempv, "b");
+  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
+  ASSERT_EQ(tempv, "c");
+  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
+  ASSERT_EQ(tempv, "d");
+  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
+  ASSERT_EQ(tempv, "e");
+  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
+  ASSERT_EQ(tempv, "f");
+  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
+  ASSERT_EQ(tempv, "g");
+
+  // Bad indices (just out-of-bounds / off-by-one check)
+  ASSERT_EQ(redis.Set("k1", -8, "off-by-one in negative index"), false);
+  ASSERT_EQ(redis.Set("k1", 7, "off-by-one-error in positive index"), false);
+  ASSERT_EQ(redis.Set("k1", 43892, "big random index should fail"), false);
+  ASSERT_EQ(redis.Set("k1", -21391, "large negative index should fail"), false);
+
+  // One last check (to make sure nothing weird happened)
+  ASSERT_EQ(redis.Length("k1"), 7); // Size should not change
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "a");
+  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
+  ASSERT_EQ(tempv, "b");
+  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
+  ASSERT_EQ(tempv, "c");
+  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
+  ASSERT_EQ(tempv, "d");
+  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
+  ASSERT_EQ(tempv, "e");
+  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
+  ASSERT_EQ(tempv, "f");
+  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
+  ASSERT_EQ(tempv, "g");
+}
+
+// Testing Insert, Push, and Set, in a mixed environment
+TEST(RedisListsTest, InsertPushSetTest) {
+  RedisLists redis(kDefaultDbName, options, true);   // Destructive
+
+  string tempv; // Used below for all Index(), PopRight(), PopLeft()
+
+  // A series of pushes and insertions
+  // Will result in [newbegin, z, a, aftera, x, newend]
+  // Also, check the return value sometimes (should return length)
+  int lengthCheck;
+  lengthCheck = redis.PushLeft("k1", "a");
+  ASSERT_EQ(lengthCheck, 1);
+  redis.PushLeft("k1", "z");
+  redis.PushRight("k1", "x");
+  lengthCheck = redis.InsertAfter("k1", "a", "aftera");
+  ASSERT_EQ(lengthCheck , 4);
+  redis.InsertBefore("k1", "z", "newbegin");  // InsertBefore beginning of list
+  redis.InsertAfter("k1", "x", "newend");     // InsertAfter end of list
+
+  // Check
+  std::vector<std::string> res = redis.Range("k1", 0, -1); // Get the list
+  ASSERT_EQ((int)res.size(), 6);
+  ASSERT_EQ(res[0], "newbegin");
+  ASSERT_EQ(res[5], "newend");
+  ASSERT_EQ(res[3], "aftera");
+
+  // Testing duplicate values/pivots (multiple occurrences of 'a')
+  ASSERT_TRUE(redis.Set("k1", 0, "a"));     // [a, z, a, aftera, x, newend]
+  redis.InsertAfter("k1", "a", "happy");    // [a, happy, z, a, aftera, ...]
+  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
+  ASSERT_EQ(tempv, "happy");
+  redis.InsertBefore("k1", "a", "sad");     // [sad, a, happy, z, a, aftera, ...]
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "sad");
+  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
+  ASSERT_EQ(tempv, "happy");
+  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
+  ASSERT_EQ(tempv, "aftera");
+  redis.InsertAfter("k1", "a", "zz");         // [sad, a, zz, happy, z, a, aftera, ...]
+  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
+  ASSERT_EQ(tempv, "zz");
+  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
+  ASSERT_EQ(tempv, "aftera");
+  ASSERT_TRUE(redis.Set("k1", 1, "nota"));    // [sad, nota, zz, happy, z, a, ...]
+  redis.InsertBefore("k1", "a", "ba");        // [sad, nota, zz, happy, z, ba, a, ...]
+  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
+  ASSERT_EQ(tempv, "z");
+  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
+  ASSERT_EQ(tempv, "ba");
+  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
+  ASSERT_EQ(tempv, "a");
+
+  // We currently have: [sad, nota, zz, happy, z, ba, a, aftera, x, newend]
+  // redis.Print("k1");   // manually check
+
+  // Test Inserting before/after non-existent values
+  lengthCheck = redis.Length("k1"); // Ensure that the length doesn't change
+  ASSERT_EQ(lengthCheck, 10);
+  ASSERT_EQ(redis.InsertBefore("k1", "non-exist", "randval"), lengthCheck);
+  ASSERT_EQ(redis.InsertAfter("k1", "nothing", "a"), lengthCheck);
+  ASSERT_EQ(redis.InsertAfter("randKey", "randVal", "ranValue"), 0); // Empty
+  ASSERT_EQ(redis.Length("k1"), lengthCheck); // The length should not change
+
+  // Simply Test the Set() function
+  redis.Set("k1", 5, "ba2");
+  redis.InsertBefore("k1", "ba2", "beforeba2");
+  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
+  ASSERT_EQ(tempv, "z");
+  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
+  ASSERT_EQ(tempv, "beforeba2");
+  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
+  ASSERT_EQ(tempv, "ba2");
+  ASSERT_TRUE(redis.Index("k1", 7, &tempv));
+  ASSERT_EQ(tempv, "a");
+
+  // We have: [sad, nota, zz, happy, z, beforeba2, ba2, a, aftera, x, newend]
+
+  // Set() with negative indices
+  redis.Set("k1", -1, "endprank");
+  ASSERT_TRUE(!redis.Index("k1", 11, &tempv));
+  ASSERT_TRUE(redis.Index("k1", 10, &tempv));
+  ASSERT_EQ(tempv, "endprank"); // Ensure Set worked correctly
+  redis.Set("k1", -11, "t");
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "t");
+
+  // Test out of bounds Set
+  ASSERT_EQ(redis.Set("k1", -12, "ssd"), false);
+  ASSERT_EQ(redis.Set("k1", 11, "sasd"), false);
+  ASSERT_EQ(redis.Set("k1", 1200, "big"), false);
+}
+
+// Testing Trim, Pop
+TEST(RedisListsTest, TrimPopTest) {
+  RedisLists redis(kDefaultDbName, options, true);   // Destructive
+
+  string tempv; // Used below for all Index(), PopRight(), PopLeft()
+
+  // A series of pushes and insertions
+  // Will result in [newbegin, z, a, aftera, x, newend]
+  redis.PushLeft("k1", "a");
+  redis.PushLeft("k1", "z");
+  redis.PushRight("k1", "x");
+  redis.InsertBefore("k1", "z", "newbegin");    // InsertBefore start of list
+  redis.InsertAfter("k1", "x", "newend");       // InsertAfter end of list
+  redis.InsertAfter("k1", "a", "aftera");
+
+  // Simple PopLeft/Right test
+  ASSERT_TRUE(redis.PopLeft("k1", &tempv));
+  ASSERT_EQ(tempv, "newbegin");
+  ASSERT_EQ(redis.Length("k1"), 5);
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "z");
+  ASSERT_TRUE(redis.PopRight("k1", &tempv));
+  ASSERT_EQ(tempv, "newend");
+  ASSERT_EQ(redis.Length("k1"), 4);
+  ASSERT_TRUE(redis.Index("k1", -1, &tempv));
+  ASSERT_EQ(tempv, "x");
+
+  // Now have: [z, a, aftera, x]
+
+  // Test Trim
+  ASSERT_TRUE(redis.Trim("k1", 0, -1));       // [z, a, aftera, x] (do nothing)
+  ASSERT_EQ(redis.Length("k1"), 4);
+  ASSERT_TRUE(redis.Trim("k1", 0, 2));                     // [z, a, aftera]
+  ASSERT_EQ(redis.Length("k1"), 3);
+  ASSERT_TRUE(redis.Index("k1", -1, &tempv));
+  ASSERT_EQ(tempv, "aftera");
+  ASSERT_TRUE(redis.Trim("k1", 1, 1));                     // [a]
+  ASSERT_EQ(redis.Length("k1"), 1);
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "a");
+
+  // Test out of bounds (empty) trim
+  ASSERT_TRUE(redis.Trim("k1", 1, 0));
+  ASSERT_EQ(redis.Length("k1"), 0);
+
+  // Popping with empty list (return empty without error)
+  ASSERT_TRUE(!redis.PopLeft("k1", &tempv));
+  ASSERT_TRUE(!redis.PopRight("k1", &tempv));
+  ASSERT_TRUE(redis.Trim("k1", 0, 5));
+
+  // Exhaustive Trim test (negative and invalid indices)
+  // Will start in [newbegin, z, a, aftera, x, newend]
+  redis.PushLeft("k1", "a");
+  redis.PushLeft("k1", "z");
+  redis.PushRight("k1", "x");
+  redis.InsertBefore("k1", "z", "newbegin");    // InsertBefore start of list
+  redis.InsertAfter("k1", "x", "newend");       // InsertAfter end of list
+  redis.InsertAfter("k1", "a", "aftera");
+  ASSERT_TRUE(redis.Trim("k1", -6, -1));                     // Should do nothing
+  ASSERT_EQ(redis.Length("k1"), 6);
+  ASSERT_TRUE(redis.Trim("k1", 1, -2));
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "z");
+  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
+  ASSERT_EQ(tempv, "x");
+  ASSERT_EQ(redis.Length("k1"), 4);
+  ASSERT_TRUE(redis.Trim("k1", -3, -2));
+  ASSERT_EQ(redis.Length("k1"), 2);
+}
+
+// Testing Remove, RemoveFirst, RemoveLast
+TEST(RedisListsTest, RemoveTest) {
+  RedisLists redis(kDefaultDbName, options, true);   // Destructive
+
+  string tempv; // Used below for all Index(), PopRight(), PopLeft()
+
+  // A series of pushes and insertions
+  // Will result in [newbegin, z, a, aftera, x, newend, a, a]
+  redis.PushLeft("k1", "a");
+  redis.PushLeft("k1", "z");
+  redis.PushRight("k1", "x");
+  redis.InsertBefore("k1", "z", "newbegin");    // InsertBefore start of list
+  redis.InsertAfter("k1", "x", "newend");       // InsertAfter end of list
+  redis.InsertAfter("k1", "a", "aftera");
+  redis.PushRight("k1", "a");
+  redis.PushRight("k1", "a");
+
+  // Verify
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "newbegin");
+  ASSERT_TRUE(redis.Index("k1", -1, &tempv));
+  ASSERT_EQ(tempv, "a");
+
+  // Check RemoveFirst (Remove the first two 'a')
+  // Results in [newbegin, z, aftera, x, newend, a]
+  int numRemoved = redis.Remove("k1", 2, "a");
+  ASSERT_EQ(numRemoved, 2);
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "newbegin");
+  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
+  ASSERT_EQ(tempv, "z");
+  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
+  ASSERT_EQ(tempv, "newend");
+  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
+  ASSERT_EQ(tempv, "a");
+  ASSERT_EQ(redis.Length("k1"), 6);
+
+  // Repopulate some stuff
+  // Results in: [x, x, x, x, x, newbegin, z, x, aftera, x, newend, a, x]
+  redis.PushLeft("k1", "x");
+  redis.PushLeft("k1", "x");
+  redis.PushLeft("k1", "x");
+  redis.PushLeft("k1", "x");
+  redis.PushLeft("k1", "x");
+  redis.PushRight("k1", "x");
+  redis.InsertAfter("k1", "z", "x");
+
+  // Test removal from end
+  numRemoved = redis.Remove("k1", -2, "x");
+  ASSERT_EQ(numRemoved, 2);
+  ASSERT_TRUE(redis.Index("k1", 8, &tempv));
+  ASSERT_EQ(tempv, "aftera");
+  ASSERT_TRUE(redis.Index("k1", 9, &tempv));
+  ASSERT_EQ(tempv, "newend");
+  ASSERT_TRUE(redis.Index("k1", 10, &tempv));
+  ASSERT_EQ(tempv, "a");
+  ASSERT_TRUE(!redis.Index("k1", 11, &tempv));
+  numRemoved = redis.Remove("k1", -2, "x");
+  ASSERT_EQ(numRemoved, 2);
+  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
+  ASSERT_EQ(tempv, "newbegin");
+  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
+  ASSERT_EQ(tempv, "aftera");
+
+  // We now have: [x, x, x, x, newbegin, z, aftera, newend, a]
+  ASSERT_EQ(redis.Length("k1"), 9);
+  ASSERT_TRUE(redis.Index("k1", -1, &tempv));
+  ASSERT_EQ(tempv, "a");
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "x");
+
+  // Test over-shooting (removing more than there exists)
+  numRemoved = redis.Remove("k1", -9000, "x");
+  ASSERT_EQ(numRemoved , 4);    // Only really removed 4
+  ASSERT_EQ(redis.Length("k1"), 5);
+  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+  ASSERT_EQ(tempv, "newbegin");
+  numRemoved = redis.Remove("k1", 1, "x");
+  ASSERT_EQ(numRemoved, 0);
+
+  // Try removing ALL!
+  numRemoved = redis.Remove("k1", 0, "newbegin");   // REMOVE 0 will remove all!
+  ASSERT_EQ(numRemoved, 1);
+
+  // Removal from an empty-list
+  ASSERT_TRUE(redis.Trim("k1", 1, 0));
+  numRemoved = redis.Remove("k1", 1, "z");
+  ASSERT_EQ(numRemoved, 0);
+}
+
+
+// Test Multiple keys and Persistence
+TEST(RedisListsTest, PersistenceMultiKeyTest) {
+
+  string tempv; // Used below for all Index(), PopRight(), PopLeft()
+
+  // Block one: populate a single key in the database
+  {
+    RedisLists redis(kDefaultDbName, options, true);   // Destructive
+
+    // A series of pushes and insertions
+    // Will result in [newbegin, z, a, aftera, x, newend, a, a]
+    redis.PushLeft("k1", "a");
+    redis.PushLeft("k1", "z");
+    redis.PushRight("k1", "x");
+    redis.InsertBefore("k1", "z", "newbegin");    // InsertBefore start of list
+    redis.InsertAfter("k1", "x", "newend");       // InsertAfter end of list
+    redis.InsertAfter("k1", "a", "aftera");
+    redis.PushRight("k1", "a");
+    redis.PushRight("k1", "a");
+
+    ASSERT_TRUE(redis.Index("k1", 3, &tempv));
+    ASSERT_EQ(tempv, "aftera");
+  }
+
+  // Block two: make sure changes were saved and add some other key
+  {
+    RedisLists redis(kDefaultDbName, options, false); // Persistent, non-destructive
+
+    // Check
+    ASSERT_EQ(redis.Length("k1"), 8);
+    ASSERT_TRUE(redis.Index("k1", 3, &tempv));
+    ASSERT_EQ(tempv, "aftera");
+
+    redis.PushRight("k2", "randomkey");
+    redis.PushLeft("k2", "sas");
+
+    redis.PopLeft("k1", &tempv);
+  }
+
+  // Block three: Verify the changes from block 2
+  {
+    RedisLists redis(kDefaultDbName, options, false); // Persistent, non-destructive
+
+    // Check
+    ASSERT_EQ(redis.Length("k1"), 7);
+    ASSERT_EQ(redis.Length("k2"), 2);
+    ASSERT_TRUE(redis.Index("k1", 0, &tempv));
+    ASSERT_EQ(tempv, "z");
+    ASSERT_TRUE(redis.Index("k2", -2, &tempv));
+    ASSERT_EQ(tempv, "sas");
+  }
+}
+
+/// THE manual REDIS TEST begins here
+/// THIS WILL ONLY OCCUR IF YOU RUN: ./redis_test -m
+
+namespace {
+void MakeUpper(std::string* const s) {
+  int len = s->length();
+  for(int i=0; i<len; ++i) {
+    (*s)[i] = toupper((*s)[i]); // C-version defined in <ctype.h>
+  }
+}
+
+/// Allows the user to enter in REDIS commands into the command-line.
+/// This is useful for manual / interacticve testing / debugging.
+///  Use destructive=true to clean the database before use.
+///  Use destructive=false to remember the previous state (i.e.: persistent)
+/// Should be called from main function.
+int manual_redis_test(bool destructive){
+  RedisLists redis(RedisListsTest::kDefaultDbName,
+                   RedisListsTest::options,
+                   destructive);
+
+  // TODO: Right now, please use spaces to separate each word.
+  //  In actual redis, you can use quotes to specify compound values
+  //  Example: RPUSH mylist "this is a compound value"
+
+  std::string command;
+  while(true) {
+    cin >> command;
+    MakeUpper(&command);
+
+    if (command == "LINSERT") {
+      std::string k, t, p, v;
+      cin >> k >> t >> p >> v;
+      MakeUpper(&t);
+      if (t=="BEFORE") {
+        std::cout << redis.InsertBefore(k, p, v) << std::endl;
+      } else if (t=="AFTER") {
+        std::cout << redis.InsertAfter(k, p, v) << std::endl;
+      }
+    } else if (command == "LPUSH") {
+      std::string k, v;
+      std::cin >> k >> v;
+      redis.PushLeft(k, v);
+    } else if (command == "RPUSH") {
+      std::string k, v;
+      std::cin >> k >> v;
+      redis.PushRight(k, v);
+    } else if (command == "LPOP") {
+      std::string k;
+      std::cin >> k;
+      string res;
+      redis.PopLeft(k, &res);
+      std::cout << res << std::endl;
+    } else if (command == "RPOP") {
+      std::string k;
+      std::cin >> k;
+      string res;
+      redis.PopRight(k, &res);
+      std::cout << res << std::endl;
+    } else if (command == "LREM") {
+      std::string k;
+      int amt;
+      std::string v;
+
+      std::cin >> k >> amt >> v;
+      std::cout << redis.Remove(k, amt, v) << std::endl;
+    } else if (command == "LLEN") {
+      std::string k;
+      std::cin >> k;
+      std::cout << redis.Length(k) << std::endl;
+    } else if (command == "LRANGE") {
+      std::string k;
+      int i, j;
+      std::cin >> k >> i >> j;
+      std::vector<std::string> res = redis.Range(k, i, j);
+      for (auto it = res.begin(); it != res.end(); ++it) {
+        std::cout << " " << (*it);
+      }
+      std::cout << std::endl;
+    } else if (command == "LTRIM") {
+      std::string k;
+      int i, j;
+      std::cin >> k >> i >> j;
+      redis.Trim(k, i, j);
+    } else if (command == "LSET") {
+      std::string k;
+      int idx;
+      std::string v;
+      cin >> k >> idx >> v;
+      redis.Set(k, idx, v);
+    } else if (command == "LINDEX") {
+      std::string k;
+      int idx;
+      std::cin >> k >> idx;
+      string res;
+      redis.Index(k, idx, &res);
+      std::cout << res << std::endl;
+    } else if (command == "PRINT") {      // Added by Deon
+      std::string k;
+      cin >> k;
+      redis.Print(k);
+    } else if (command == "QUIT") {
+      return 0;
+    } else {
+      std::cout << "unknown command: " << command << std::endl;
+    }
+  }
+}
+}  // namespace
+
+} // namespace rocksdb
+
+
+// USAGE: "./redis_test" for default (unit tests)
+//        "./redis_test -m" for manual testing (redis command api)
+//        "./redis_test -m -d" for destructive manual test (erase db before use)
+
+
+namespace {
+// Check for "want" argument in the argument list
+bool found_arg(int argc, char* argv[], const char* want){
+  for(int i=1; i<argc; ++i){
+    if (strcmp(argv[i], want) == 0) {
+      return true;
+    }
+  }
+  return false;
+}
+}  // namespace
+
+// Will run unit tests.
+// However, if -m is specified, it will do user manual/interactive testing
+// -m -d is manual and destructive (will clear the database before use)
+int main(int argc, char* argv[]) {
+  if (found_arg(argc, argv, "-m")) {
+    bool destructive = found_arg(argc, argv, "-d");
+    return rocksdb::manual_redis_test(destructive);
+  } else {
+    return rocksdb::test::RunAllTests();
+  }
+}
+

utilities/ttl/db_ttl_impl.cc

@@ -0,0 +1,284 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#ifndef ROCKSDB_LITE
+
+#include "utilities/ttl/db_ttl_impl.h"
+
+#include "utilities/db_ttl.h"
+#include "db/filename.h"
+#include "db/write_batch_internal.h"
+#include "util/coding.h"
+#include "rocksdb/env.h"
+#include "rocksdb/iterator.h"
+
+namespace rocksdb {
+
+void DBWithTTLImpl::SanitizeOptions(int32_t ttl, ColumnFamilyOptions* options,
+                                    Env* env) {
+  if (options->compaction_filter) {
+    options->compaction_filter =
+        new TtlCompactionFilter(ttl, env, options->compaction_filter);
+  } else {
+    options->compaction_filter_factory =
+        std::shared_ptr<CompactionFilterFactory>(new TtlCompactionFilterFactory(
+            ttl, env, options->compaction_filter_factory));
+  }
+
+  if (options->merge_operator) {
+    options->merge_operator.reset(
+        new TtlMergeOperator(options->merge_operator, env));
+  }
+}
+
+// Open the db inside DBWithTTLImpl because options needs pointer to its ttl
+DBWithTTLImpl::DBWithTTLImpl(DB* db) : DBWithTTL(db) {}
+
+DBWithTTLImpl::~DBWithTTLImpl() { delete GetOptions().compaction_filter; }
+
+Status UtilityDB::OpenTtlDB(const Options& options, const std::string& dbname,
+                            StackableDB** dbptr, int32_t ttl, bool read_only) {
+  DBWithTTL* db;
+  Status s = DBWithTTL::Open(options, dbname, &db, ttl, read_only);
+  if (s.ok()) {
+    *dbptr = db;
+  } else {
+    *dbptr = nullptr;
+  }
+  return s;
+}
+
+Status DBWithTTL::Open(const Options& options, const std::string& dbname,
+                       DBWithTTL** dbptr, int32_t ttl, bool read_only) {
+
+  DBOptions db_options(options);
+  ColumnFamilyOptions cf_options(options);
+  std::vector<ColumnFamilyDescriptor> column_families;
+  column_families.push_back(
+      ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options));
+  std::vector<ColumnFamilyHandle*> handles;
+  Status s = DBWithTTL::Open(db_options, dbname, column_families, &handles,
+                             dbptr, {ttl}, read_only);
+  if (s.ok()) {
+    assert(handles.size() == 1);
+    // i can delete the handle since DBImpl is always holding a reference to
+    // default column family
+    delete handles[0];
+  }
+  return s;
+}
+
+Status DBWithTTL::Open(
+    const DBOptions& db_options, const std::string& dbname,
+    const std::vector<ColumnFamilyDescriptor>& column_families,
+    std::vector<ColumnFamilyHandle*>* handles, DBWithTTL** dbptr,
+    std::vector<int32_t> ttls, bool read_only) {
+
+  if (ttls.size() != column_families.size()) {
+    return Status::InvalidArgument(
+        "ttls size has to be the same as number of column families");
+  }
+
+  std::vector<ColumnFamilyDescriptor> column_families_sanitized =
+      column_families;
+  for (size_t i = 0; i < column_families_sanitized.size(); ++i) {
+    DBWithTTLImpl::SanitizeOptions(
+        ttls[i], &column_families_sanitized[i].options,
+        db_options.env == nullptr ? Env::Default() : db_options.env);
+  }
+  DB* db;
+
+  Status st;
+  if (read_only) {
+    st = DB::OpenForReadOnly(db_options, dbname, column_families_sanitized,
+                             handles, &db);
+  } else {
+    st = DB::Open(db_options, dbname, column_families_sanitized, handles, &db);
+  }
+  if (st.ok()) {
+    *dbptr = new DBWithTTLImpl(db);
+  } else {
+    *dbptr = nullptr;
+  }
+  return st;
+}
+
+Status DBWithTTLImpl::CreateColumnFamilyWithTtl(
+    const ColumnFamilyOptions& options, const std::string& column_family_name,
+    ColumnFamilyHandle** handle, int ttl) {
+  ColumnFamilyOptions sanitized_options = options;
+  DBWithTTLImpl::SanitizeOptions(ttl, &sanitized_options, GetEnv());
+
+  return DBWithTTL::CreateColumnFamily(sanitized_options, column_family_name,
+                                       handle);
+}
+
+Status DBWithTTLImpl::CreateColumnFamily(const ColumnFamilyOptions& options,
+                                         const std::string& column_family_name,
+                                         ColumnFamilyHandle** handle) {
+  return CreateColumnFamilyWithTtl(options, column_family_name, handle, 0);
+}
+
+// Appends the current timestamp to the string.
+// Returns false if could not get the current_time, true if append succeeds
+Status DBWithTTLImpl::AppendTS(const Slice& val, std::string* val_with_ts,
+                               Env* env) {
+  val_with_ts->reserve(kTSLength + val.size());
+  char ts_string[kTSLength];
+  int64_t curtime;
+  Status st = env->GetCurrentTime(&curtime);
+  if (!st.ok()) {
+    return st;
+  }
+  EncodeFixed32(ts_string, (int32_t)curtime);
+  val_with_ts->append(val.data(), val.size());
+  val_with_ts->append(ts_string, kTSLength);
+  return st;
+}
+
+// Returns corruption if the length of the string is lesser than timestamp, or
+// timestamp refers to a time lesser than ttl-feature release time
+Status DBWithTTLImpl::SanityCheckTimestamp(const Slice& str) {
+  if (str.size() < kTSLength) {
+    return Status::Corruption("Error: value's length less than timestamp's\n");
+  }
+  // Checks that TS is not lesser than kMinTimestamp
+  // Gaurds against corruption & normal database opened incorrectly in ttl mode
+  int32_t timestamp_value = DecodeFixed32(str.data() + str.size() - kTSLength);
+  if (timestamp_value < kMinTimestamp) {
+    return Status::Corruption("Error: Timestamp < ttl feature release time!\n");
+  }
+  return Status::OK();
+}
+
+// Checks if the string is stale or not according to TTl provided
+bool DBWithTTLImpl::IsStale(const Slice& value, int32_t ttl, Env* env) {
+  if (ttl <= 0) {  // Data is fresh if TTL is non-positive
+    return false;
+  }
+  int64_t curtime;
+  if (!env->GetCurrentTime(&curtime).ok()) {
+    return false;  // Treat the data as fresh if could not get current time
+  }
+  int32_t timestamp_value =
+      DecodeFixed32(value.data() + value.size() - kTSLength);
+  return (timestamp_value + ttl) < curtime;
+}
+
+// Strips the TS from the end of the string
+Status DBWithTTLImpl::StripTS(std::string* str) {
+  Status st;
+  if (str->length() < kTSLength) {
+    return Status::Corruption("Bad timestamp in key-value");
+  }
+  // Erasing characters which hold the TS
+  str->erase(str->length() - kTSLength, kTSLength);
+  return st;
+}
+
+Status DBWithTTLImpl::Put(const WriteOptions& options,
+                          ColumnFamilyHandle* column_family, const Slice& key,
+                          const Slice& val) {
+  WriteBatch batch;
+  batch.Put(column_family, key, val);
+  return Write(options, &batch);
+}
+
+Status DBWithTTLImpl::Get(const ReadOptions& options,
+                          ColumnFamilyHandle* column_family, const Slice& key,
+                          std::string* value) {
+  Status st = db_->Get(options, column_family, key, value);
+  if (!st.ok()) {
+    return st;
+  }
+  st = SanityCheckTimestamp(*value);
+  if (!st.ok()) {
+    return st;
+  }
+  return StripTS(value);
+}
+
+std::vector<Status> DBWithTTLImpl::MultiGet(
+    const ReadOptions& options,
+    const std::vector<ColumnFamilyHandle*>& column_family,
+    const std::vector<Slice>& keys, std::vector<std::string>* values) {
+  return std::vector<Status>(
+      keys.size(), Status::NotSupported("MultiGet not supported with TTL"));
+}
+
+bool DBWithTTLImpl::KeyMayExist(const ReadOptions& options,
+                                ColumnFamilyHandle* column_family,
+                                const Slice& key, std::string* value,
+                                bool* value_found) {
+  bool ret = db_->KeyMayExist(options, column_family, key, value, value_found);
+  if (ret && value != nullptr && value_found != nullptr && *value_found) {
+    if (!SanityCheckTimestamp(*value).ok() || !StripTS(value).ok()) {
+      return false;
+    }
+  }
+  return ret;
+}
+
+Status DBWithTTLImpl::Merge(const WriteOptions& options,
+                            ColumnFamilyHandle* column_family, const Slice& key,
+                            const Slice& value) {
+  WriteBatch batch;
+  batch.Merge(column_family, key, value);
+  return Write(options, &batch);
+}
+
+Status DBWithTTLImpl::Write(const WriteOptions& opts, WriteBatch* updates) {
+  class Handler : public WriteBatch::Handler {
+   public:
+    explicit Handler(Env* env) : env_(env) {}
+    WriteBatch updates_ttl;
+    Status batch_rewrite_status;
+    virtual Status PutCF(uint32_t column_family_id, const Slice& key,
+                         const Slice& value) {
+      std::string value_with_ts;
+      Status st = AppendTS(value, &value_with_ts, env_);
+      if (!st.ok()) {
+        batch_rewrite_status = st;
+      } else {
+        WriteBatchInternal::Put(&updates_ttl, column_family_id, key,
+                                value_with_ts);
+      }
+      return Status::OK();
+    }
+    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
+                           const Slice& value) {
+      std::string value_with_ts;
+      Status st = AppendTS(value, &value_with_ts, env_);
+      if (!st.ok()) {
+        batch_rewrite_status = st;
+      } else {
+        WriteBatchInternal::Merge(&updates_ttl, column_family_id, key,
+                                  value_with_ts);
+      }
+      return Status::OK();
+    }
+    virtual Status DeleteCF(uint32_t column_family_id, const Slice& key) {
+      WriteBatchInternal::Delete(&updates_ttl, column_family_id, key);
+      return Status::OK();
+    }
+    virtual void LogData(const Slice& blob) { updates_ttl.PutLogData(blob); }
+
+   private:
+    Env* env_;
+  };
+  Handler handler(GetEnv());
+  updates->Iterate(&handler);
+  if (!handler.batch_rewrite_status.ok()) {
+    return handler.batch_rewrite_status;
+  } else {
+    return db_->Write(opts, &(handler.updates_ttl));
+  }
+}
+
+Iterator* DBWithTTLImpl::NewIterator(const ReadOptions& opts,
+                                     ColumnFamilyHandle* column_family) {
+  return new TtlIterator(db_->NewIterator(opts, column_family));
+}
+
+}  // namespace rocksdb
+#endif  // ROCKSDB_LITE

utilities/ttl/db_ttl_impl.h

@@ -0,0 +1,314 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+
+#ifndef ROCKSDB_LITE
+#include <deque>
+#include <string>
+#include <vector>
+
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/merge_operator.h"
+#include "utilities/utility_db.h"
+#include "utilities/db_ttl.h"
+#include "db/db_impl.h"
+
+namespace rocksdb {
+
+class DBWithTTLImpl : public DBWithTTL {
+ public:
+  static void SanitizeOptions(int32_t ttl, ColumnFamilyOptions* options,
+                              Env* env);
+
+  explicit DBWithTTLImpl(DB* db);
+
+  virtual ~DBWithTTLImpl();
+
+  Status CreateColumnFamilyWithTtl(const ColumnFamilyOptions& options,
+                                   const std::string& column_family_name,
+                                   ColumnFamilyHandle** handle,
+                                   int ttl) override;
+
+  Status CreateColumnFamily(const ColumnFamilyOptions& options,
+                            const std::string& column_family_name,
+                            ColumnFamilyHandle** handle) override;
+
+  using StackableDB::Put;
+  virtual Status Put(const WriteOptions& options,
+                     ColumnFamilyHandle* column_family, const Slice& key,
+                     const Slice& val) override;
+
+  using StackableDB::Get;
+  virtual Status Get(const ReadOptions& options,
+                     ColumnFamilyHandle* column_family, const Slice& key,
+                     std::string* value) override;
+
+  using StackableDB::MultiGet;
+  virtual std::vector<Status> MultiGet(
+      const ReadOptions& options,
+      const std::vector<ColumnFamilyHandle*>& column_family,
+      const std::vector<Slice>& keys,
+      std::vector<std::string>* values) override;
+
+  using StackableDB::KeyMayExist;
+  virtual bool KeyMayExist(const ReadOptions& options,
+                           ColumnFamilyHandle* column_family, const Slice& key,
+                           std::string* value,
+                           bool* value_found = nullptr) override;
+
+  using StackableDB::Merge;
+  virtual Status Merge(const WriteOptions& options,
+                       ColumnFamilyHandle* column_family, const Slice& key,
+                       const Slice& value) override;
+
+  virtual Status Write(const WriteOptions& opts, WriteBatch* updates) override;
+
+  using StackableDB::NewIterator;
+  virtual Iterator* NewIterator(const ReadOptions& opts,
+                                ColumnFamilyHandle* column_family) override;
+
+  virtual DB* GetBaseDB() { return db_; }
+
+  static bool IsStale(const Slice& value, int32_t ttl, Env* env);
+
+  static Status AppendTS(const Slice& val, std::string* val_with_ts, Env* env);
+
+  static Status SanityCheckTimestamp(const Slice& str);
+
+  static Status StripTS(std::string* str);
+
+  static const uint32_t kTSLength = sizeof(int32_t);  // size of timestamp
+
+  static const int32_t kMinTimestamp = 1368146402;  // 05/09/2013:5:40PM GMT-8
+
+  static const int32_t kMaxTimestamp = 2147483647;  // 01/18/2038:7:14PM GMT-8
+};
+
+class TtlIterator : public Iterator {
+
+ public:
+  explicit TtlIterator(Iterator* iter) : iter_(iter) { assert(iter_); }
+
+  ~TtlIterator() { delete iter_; }
+
+  bool Valid() const { return iter_->Valid(); }
+
+  void SeekToFirst() { iter_->SeekToFirst(); }
+
+  void SeekToLast() { iter_->SeekToLast(); }
+
+  void Seek(const Slice& target) { iter_->Seek(target); }
+
+  void Next() { iter_->Next(); }
+
+  void Prev() { iter_->Prev(); }
+
+  Slice key() const { return iter_->key(); }
+
+  int32_t timestamp() const {
+    return DecodeFixed32(iter_->value().data() + iter_->value().size() -
+                         DBWithTTLImpl::kTSLength);
+  }
+
+  Slice value() const {
+    // TODO: handle timestamp corruption like in general iterator semantics
+    assert(DBWithTTLImpl::SanityCheckTimestamp(iter_->value()).ok());
+    Slice trimmed_value = iter_->value();
+    trimmed_value.size_ -= DBWithTTLImpl::kTSLength;
+    return trimmed_value;
+  }
+
+  Status status() const { return iter_->status(); }
+
+ private:
+  Iterator* iter_;
+};
+
+class TtlCompactionFilter : public CompactionFilter {
+ public:
+  TtlCompactionFilter(
+      int32_t ttl, Env* env, const CompactionFilter* user_comp_filter,
+      std::unique_ptr<const CompactionFilter> user_comp_filter_from_factory =
+          nullptr)
+      : ttl_(ttl),
+        env_(env),
+        user_comp_filter_(user_comp_filter),
+        user_comp_filter_from_factory_(
+            std::move(user_comp_filter_from_factory)) {
+    // Unlike the merge operator, compaction filter is necessary for TTL, hence
+    // this would be called even if user doesn't specify any compaction-filter
+    if (!user_comp_filter_) {
+      user_comp_filter_ = user_comp_filter_from_factory_.get();
+    }
+  }
+
+  virtual bool Filter(int level, const Slice& key, const Slice& old_val,
+                      std::string* new_val, bool* value_changed) const
+      override {
+    if (DBWithTTLImpl::IsStale(old_val, ttl_, env_)) {
+      return true;
+    }
+    if (user_comp_filter_ == nullptr) {
+      return false;
+    }
+    assert(old_val.size() >= DBWithTTLImpl::kTSLength);
+    Slice old_val_without_ts(old_val.data(),
+                             old_val.size() - DBWithTTLImpl::kTSLength);
+    if (user_comp_filter_->Filter(level, key, old_val_without_ts, new_val,
+                                  value_changed)) {
+      return true;
+    }
+    if (*value_changed) {
+      new_val->append(
+          old_val.data() + old_val.size() - DBWithTTLImpl::kTSLength,
+          DBWithTTLImpl::kTSLength);
+    }
+    return false;
+  }
+
+  virtual const char* Name() const override { return "Delete By TTL"; }
+
+ private:
+  int32_t ttl_;
+  Env* env_;
+  const CompactionFilter* user_comp_filter_;
+  std::unique_ptr<const CompactionFilter> user_comp_filter_from_factory_;
+};
+
+class TtlCompactionFilterFactory : public CompactionFilterFactory {
+ public:
+  TtlCompactionFilterFactory(
+      int32_t ttl, Env* env,
+      std::shared_ptr<CompactionFilterFactory> comp_filter_factory)
+      : ttl_(ttl), env_(env), user_comp_filter_factory_(comp_filter_factory) {}
+
+  virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
+      const CompactionFilter::Context& context) {
+    return std::unique_ptr<TtlCompactionFilter>(new TtlCompactionFilter(
+        ttl_, env_, nullptr,
+        std::move(user_comp_filter_factory_->CreateCompactionFilter(context))));
+  }
+
+  virtual const char* Name() const override {
+    return "TtlCompactionFilterFactory";
+  }
+
+ private:
+  int32_t ttl_;
+  Env* env_;
+  std::shared_ptr<CompactionFilterFactory> user_comp_filter_factory_;
+};
+
+class TtlMergeOperator : public MergeOperator {
+
+ public:
+  explicit TtlMergeOperator(const std::shared_ptr<MergeOperator> merge_op,
+                            Env* env)
+      : user_merge_op_(merge_op), env_(env) {
+    assert(merge_op);
+    assert(env);
+  }
+
+  virtual bool FullMerge(const Slice& key, const Slice* existing_value,
+                         const std::deque<std::string>& operands,
+                         std::string* new_value, Logger* logger) const
+      override {
+    const uint32_t ts_len = DBWithTTLImpl::kTSLength;
+    if (existing_value && existing_value->size() < ts_len) {
+      Log(logger, "Error: Could not remove timestamp from existing value.");
+      return false;
+    }
+
+    // Extract time-stamp from each operand to be passed to user_merge_op_
+    std::deque<std::string> operands_without_ts;
+    for (const auto& operand : operands) {
+      if (operand.size() < ts_len) {
+        Log(logger, "Error: Could not remove timestamp from operand value.");
+        return false;
+      }
+      operands_without_ts.push_back(operand.substr(0, operand.size() - ts_len));
+    }
+
+    // Apply the user merge operator (store result in *new_value)
+    bool good = true;
+    if (existing_value) {
+      Slice existing_value_without_ts(existing_value->data(),
+                                      existing_value->size() - ts_len);
+      good = user_merge_op_->FullMerge(key, &existing_value_without_ts,
+                                       operands_without_ts, new_value, logger);
+    } else {
+      good = user_merge_op_->FullMerge(key, nullptr, operands_without_ts,
+                                       new_value, logger);
+    }
+
+    // Return false if the user merge operator returned false
+    if (!good) {
+      return false;
+    }
+
+    // Augment the *new_value with the ttl time-stamp
+    int64_t curtime;
+    if (!env_->GetCurrentTime(&curtime).ok()) {
+      Log(logger,
+          "Error: Could not get current time to be attached internally "
+          "to the new value.");
+      return false;
+    } else {
+      char ts_string[ts_len];
+      EncodeFixed32(ts_string, (int32_t)curtime);
+      new_value->append(ts_string, ts_len);
+      return true;
+    }
+  }
+
+  virtual bool PartialMergeMulti(const Slice& key,
+                                 const std::deque<Slice>& operand_list,
+                                 std::string* new_value, Logger* logger) const
+      override {
+    const uint32_t ts_len = DBWithTTLImpl::kTSLength;
+    std::deque<Slice> operands_without_ts;
+
+    for (const auto& operand : operand_list) {
+      if (operand.size() < ts_len) {
+        Log(logger, "Error: Could not remove timestamp from value.");
+        return false;
+      }
+
+      operands_without_ts.push_back(
+          Slice(operand.data(), operand.size() - ts_len));
+    }
+
+    // Apply the user partial-merge operator (store result in *new_value)
+    assert(new_value);
+    if (!user_merge_op_->PartialMergeMulti(key, operands_without_ts, new_value,
+                                           logger)) {
+      return false;
+    }
+
+    // Augment the *new_value with the ttl time-stamp
+    int64_t curtime;
+    if (!env_->GetCurrentTime(&curtime).ok()) {
+      Log(logger,
+          "Error: Could not get current time to be attached internally "
+          "to the new value.");
+      return false;
+    } else {
+      char ts_string[ts_len];
+      EncodeFixed32(ts_string, (int32_t)curtime);
+      new_value->append(ts_string, ts_len);
+      return true;
+    }
+  }
+
+  virtual const char* Name() const override { return "Merge By TTL"; }
+
+ private:
+  std::shared_ptr<MergeOperator> user_merge_op_;
+  Env* env_;
+};
+}
+#endif  // ROCKSDB_LITE

utilities/ttl/ttl_test.cc

@@ -0,0 +1,595 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include <memory>
+#include "rocksdb/compaction_filter.h"
+#include "utilities/db_ttl.h"
+#include "util/testharness.h"
+#include "util/logging.h"
+#include <map>
+#include <unistd.h>
+
+namespace rocksdb {
+
+namespace {
+
+typedef std::map<std::string, std::string> KVMap;
+
+enum BatchOperation {
+  PUT = 0,
+  DELETE = 1
+};
+}
+
+class SpecialTimeEnv : public EnvWrapper {
+ public:
+  explicit SpecialTimeEnv(Env* base) : EnvWrapper(base) {
+    base->GetCurrentTime(&current_time_);
+  }
+
+  void Sleep(int64_t sleep_time) { current_time_ += sleep_time; }
+  virtual Status GetCurrentTime(int64_t* current_time) {
+    *current_time = current_time_;
+    return Status::OK();
+  }
+
+ private:
+  int64_t current_time_;
+};
+
+class TtlTest {
+ public:
+  TtlTest() {
+    env_.reset(new SpecialTimeEnv(Env::Default()));
+    dbname_ = test::TmpDir() + "/db_ttl";
+    options_.create_if_missing = true;
+    options_.env = env_.get();
+    // ensure that compaction is kicked in to always strip timestamp from kvs
+    options_.max_grandparent_overlap_factor = 0;
+    // compaction should take place always from level0 for determinism
+    options_.max_mem_compaction_level = 0;
+    db_ttl_ = nullptr;
+    DestroyDB(dbname_, Options());
+  }
+
+  ~TtlTest() {
+    CloseTtl();
+    DestroyDB(dbname_, Options());
+  }
+
+  // Open database with TTL support when TTL not provided with db_ttl_ pointer
+  void OpenTtl() {
+    ASSERT_TRUE(db_ttl_ ==
+                nullptr);  //  db should be closed before opening again
+    ASSERT_OK(DBWithTTL::Open(options_, dbname_, &db_ttl_));
+  }
+
+  // Open database with TTL support when TTL provided with db_ttl_ pointer
+  void OpenTtl(int32_t ttl) {
+    ASSERT_TRUE(db_ttl_ == nullptr);
+    ASSERT_OK(DBWithTTL::Open(options_, dbname_, &db_ttl_, ttl));
+  }
+
+  // Open with TestFilter compaction filter
+  void OpenTtlWithTestCompaction(int32_t ttl) {
+    options_.compaction_filter_factory =
+      std::shared_ptr<CompactionFilterFactory>(
+          new TestFilterFactory(kSampleSize_, kNewValue_));
+    OpenTtl(ttl);
+  }
+
+  // Open database with TTL support in read_only mode
+  void OpenReadOnlyTtl(int32_t ttl) {
+    ASSERT_TRUE(db_ttl_ == nullptr);
+    ASSERT_OK(DBWithTTL::Open(options_, dbname_, &db_ttl_, ttl, true));
+  }
+
+  void CloseTtl() {
+    delete db_ttl_;
+    db_ttl_ = nullptr;
+  }
+
+  // Populates and returns a kv-map
+  void MakeKVMap(int64_t num_entries) {
+    kvmap_.clear();
+    int digits = 1;
+    for (int dummy = num_entries; dummy /= 10 ; ++digits);
+    int digits_in_i = 1;
+    for (int64_t i = 0; i < num_entries; i++) {
+      std::string key = "key";
+      std::string value = "value";
+      if (i % 10 == 0) {
+        digits_in_i++;
+      }
+      for(int j = digits_in_i; j < digits; j++) {
+        key.append("0");
+        value.append("0");
+      }
+      AppendNumberTo(&key, i);
+      AppendNumberTo(&value, i);
+      kvmap_[key] = value;
+    }
+    ASSERT_EQ((int)kvmap_.size(), num_entries);//check all insertions done
+  }
+
+  // Makes a write-batch with key-vals from kvmap_ and 'Write''s it
+  void MakePutWriteBatch(const BatchOperation* batch_ops, int num_ops) {
+    ASSERT_LE(num_ops, (int)kvmap_.size());
+    static WriteOptions wopts;
+    static FlushOptions flush_opts;
+    WriteBatch batch;
+    kv_it_ = kvmap_.begin();
+    for (int i = 0; i < num_ops && kv_it_ != kvmap_.end(); i++, kv_it_++) {
+      switch (batch_ops[i]) {
+        case PUT:
+          batch.Put(kv_it_->first, kv_it_->second);
+          break;
+        case DELETE:
+          batch.Delete(kv_it_->first);
+          break;
+        default:
+          ASSERT_TRUE(false);
+      }
+    }
+    db_ttl_->Write(wopts, &batch);
+    db_ttl_->Flush(flush_opts);
+  }
+
+  // Puts num_entries starting from start_pos_map from kvmap_ into the database
+  void PutValues(int start_pos_map, int num_entries, bool flush = true,
+                 ColumnFamilyHandle* cf = nullptr) {
+    ASSERT_TRUE(db_ttl_);
+    ASSERT_LE(start_pos_map + num_entries, (int)kvmap_.size());
+    static WriteOptions wopts;
+    static FlushOptions flush_opts;
+    kv_it_ = kvmap_.begin();
+    advance(kv_it_, start_pos_map);
+    for (int i = 0; kv_it_ != kvmap_.end() && i < num_entries; i++, kv_it_++) {
+      ASSERT_OK(cf == nullptr
+                    ? db_ttl_->Put(wopts, kv_it_->first, kv_it_->second)
+                    : db_ttl_->Put(wopts, cf, kv_it_->first, kv_it_->second));
+    }
+    // Put a mock kv at the end because CompactionFilter doesn't delete last key
+    ASSERT_OK(cf == nullptr ? db_ttl_->Put(wopts, "keymock", "valuemock")
+                            : db_ttl_->Put(wopts, cf, "keymock", "valuemock"));
+    if (flush) {
+      if (cf == nullptr) {
+        db_ttl_->Flush(flush_opts);
+      } else {
+        db_ttl_->Flush(flush_opts, cf);
+      }
+    }
+  }
+
+  // Runs a manual compaction
+  void ManualCompact(ColumnFamilyHandle* cf = nullptr) {
+    if (cf == nullptr) {
+      db_ttl_->CompactRange(nullptr, nullptr);
+    } else {
+      db_ttl_->CompactRange(cf, nullptr, nullptr);
+    }
+  }
+
+  // checks the whole kvmap_ to return correct values using KeyMayExist
+  void SimpleKeyMayExistCheck() {
+    static ReadOptions ropts;
+    bool value_found;
+    std::string val;
+    for(auto &kv : kvmap_) {
+      bool ret = db_ttl_->KeyMayExist(ropts, kv.first, &val, &value_found);
+      if (ret == false || value_found == false) {
+        fprintf(stderr, "KeyMayExist could not find key=%s in the database but"
+                        " should have\n", kv.first.c_str());
+        ASSERT_TRUE(false);
+      } else if (val.compare(kv.second) != 0) {
+        fprintf(stderr, " value for key=%s present in database is %s but"
+                        " should be %s\n", kv.first.c_str(), val.c_str(),
+                        kv.second.c_str());
+        ASSERT_TRUE(false);
+      }
+    }
+  }
+
+  // Sleeps for slp_tim then runs a manual compaction
+  // Checks span starting from st_pos from kvmap_ in the db and
+  // Gets should return true if check is true and false otherwise
+  // Also checks that value that we got is the same as inserted; and =kNewValue
+  //   if test_compaction_change is true
+  void SleepCompactCheck(int slp_tim, int st_pos, int span, bool check = true,
+                         bool test_compaction_change = false,
+                         ColumnFamilyHandle* cf = nullptr) {
+    ASSERT_TRUE(db_ttl_);
+
+    env_->Sleep(slp_tim);
+    ManualCompact(cf);
+    static ReadOptions ropts;
+    kv_it_ = kvmap_.begin();
+    advance(kv_it_, st_pos);
+    std::string v;
+    for (int i = 0; kv_it_ != kvmap_.end() && i < span; i++, kv_it_++) {
+      Status s = (cf == nullptr) ? db_ttl_->Get(ropts, kv_it_->first, &v)
+                                 : db_ttl_->Get(ropts, cf, kv_it_->first, &v);
+      if (s.ok() != check) {
+        fprintf(stderr, "key=%s ", kv_it_->first.c_str());
+        if (!s.ok()) {
+          fprintf(stderr, "is absent from db but was expected to be present\n");
+        } else {
+          fprintf(stderr, "is present in db but was expected to be absent\n");
+        }
+        ASSERT_TRUE(false);
+      } else if (s.ok()) {
+          if (test_compaction_change && v.compare(kNewValue_) != 0) {
+            fprintf(stderr, " value for key=%s present in database is %s but "
+                            " should be %s\n", kv_it_->first.c_str(), v.c_str(),
+                            kNewValue_.c_str());
+            ASSERT_TRUE(false);
+          } else if (!test_compaction_change && v.compare(kv_it_->second) !=0) {
+            fprintf(stderr, " value for key=%s present in database is %s but "
+                            " should be %s\n", kv_it_->first.c_str(), v.c_str(),
+                            kv_it_->second.c_str());
+            ASSERT_TRUE(false);
+          }
+      }
+    }
+  }
+
+  // Similar as SleepCompactCheck but uses TtlIterator to read from db
+  void SleepCompactCheckIter(int slp, int st_pos, int span, bool check=true) {
+    ASSERT_TRUE(db_ttl_);
+    env_->Sleep(slp);
+    ManualCompact();
+    static ReadOptions ropts;
+    Iterator *dbiter = db_ttl_->NewIterator(ropts);
+    kv_it_ = kvmap_.begin();
+    advance(kv_it_, st_pos);
+
+    dbiter->Seek(kv_it_->first);
+    if (!check) {
+      if (dbiter->Valid()) {
+        ASSERT_NE(dbiter->value().compare(kv_it_->second), 0);
+      }
+    } else {  // dbiter should have found out kvmap_[st_pos]
+      for (int i = st_pos;
+           kv_it_ != kvmap_.end() && i < st_pos + span;
+           i++, kv_it_++)  {
+        ASSERT_TRUE(dbiter->Valid());
+        ASSERT_EQ(dbiter->value().compare(kv_it_->second), 0);
+        dbiter->Next();
+      }
+    }
+    delete dbiter;
+  }
+
+  class TestFilter : public CompactionFilter {
+   public:
+    TestFilter(const int64_t kSampleSize, const std::string kNewValue)
+      : kSampleSize_(kSampleSize),
+        kNewValue_(kNewValue) {
+    }
+
+    // Works on keys of the form "key<number>"
+    // Drops key if number at the end of key is in [0, kSampleSize_/3),
+    // Keeps key if it is in [kSampleSize_/3, 2*kSampleSize_/3),
+    // Change value if it is in [2*kSampleSize_/3, kSampleSize_)
+    // Eg. kSampleSize_=6. Drop:key0-1...Keep:key2-3...Change:key4-5...
+    virtual bool Filter(int level, const Slice& key,
+                        const Slice& value, std::string* new_value,
+                        bool* value_changed) const override {
+      assert(new_value != nullptr);
+
+      std::string search_str = "0123456789";
+      std::string key_string = key.ToString();
+      size_t pos = key_string.find_first_of(search_str);
+      int num_key_end;
+      if (pos != std::string::npos) {
+        num_key_end = stoi(key_string.substr(pos, key.size() - pos));
+      } else {
+        return false; // Keep keys not matching the format "key<NUMBER>"
+      }
+
+      int partition = kSampleSize_ / 3;
+      if (num_key_end < partition) {
+        return true;
+      } else if (num_key_end < partition * 2) {
+        return false;
+      } else {
+        *new_value = kNewValue_;
+        *value_changed = true;
+        return false;
+      }
+    }
+
+    virtual const char* Name() const override {
+      return "TestFilter";
+    }
+
+   private:
+    const int64_t kSampleSize_;
+    const std::string kNewValue_;
+  };
+
+  class TestFilterFactory : public CompactionFilterFactory {
+    public:
+      TestFilterFactory(const int64_t kSampleSize, const std::string kNewValue)
+        : kSampleSize_(kSampleSize),
+          kNewValue_(kNewValue) {
+      }
+
+      virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
+          const CompactionFilter::Context& context) override {
+        return std::unique_ptr<CompactionFilter>(
+            new TestFilter(kSampleSize_, kNewValue_));
+      }
+
+      virtual const char* Name() const override {
+        return "TestFilterFactory";
+      }
+
+    private:
+      const int64_t kSampleSize_;
+      const std::string kNewValue_;
+  };
+
+
+  // Choose carefully so that Put, Gets & Compaction complete in 1 second buffer
+  const int64_t kSampleSize_ = 100;
+  std::string dbname_;
+  DBWithTTL* db_ttl_;
+  unique_ptr<SpecialTimeEnv> env_;
+
+ private:
+  Options options_;
+  KVMap kvmap_;
+  KVMap::iterator kv_it_;
+  const std::string kNewValue_ = "new_value";
+  unique_ptr<CompactionFilter> test_comp_filter_;
+}; // class TtlTest
+
+// If TTL is non positive or not provided, the behaviour is TTL = infinity
+// This test opens the db 3 times with such default behavior and inserts a
+// bunch of kvs each time. All kvs should accumulate in the db till the end
+// Partitions the sample-size provided into 3 sets over boundary1 and boundary2
+TEST(TtlTest, NoEffect) {
+  MakeKVMap(kSampleSize_);
+  int boundary1 = kSampleSize_ / 3;
+  int boundary2 = 2 * boundary1;
+
+  OpenTtl();
+  PutValues(0, boundary1);                       //T=0: Set1 never deleted
+  SleepCompactCheck(1, 0, boundary1);            //T=1: Set1 still there
+  CloseTtl();
+
+  OpenTtl(0);
+  PutValues(boundary1, boundary2 - boundary1);   //T=1: Set2 never deleted
+  SleepCompactCheck(1, 0, boundary2);            //T=2: Sets1 & 2 still there
+  CloseTtl();
+
+  OpenTtl(-1);
+  PutValues(boundary2, kSampleSize_ - boundary2); //T=3: Set3 never deleted
+  SleepCompactCheck(1, 0, kSampleSize_, true);    //T=4: Sets 1,2,3 still there
+  CloseTtl();
+}
+
+// Puts a set of values and checks its presence using Get during ttl
+TEST(TtlTest, PresentDuringTTL) {
+  MakeKVMap(kSampleSize_);
+
+  OpenTtl(2);                                 // T=0:Open the db with ttl = 2
+  PutValues(0, kSampleSize_);                  // T=0:Insert Set1. Delete at t=2
+  SleepCompactCheck(1, 0, kSampleSize_, true); // T=1:Set1 should still be there
+  CloseTtl();
+}
+
+// Puts a set of values and checks its absence using Get after ttl
+TEST(TtlTest, AbsentAfterTTL) {
+  MakeKVMap(kSampleSize_);
+
+  OpenTtl(1);                                  // T=0:Open the db with ttl = 2
+  PutValues(0, kSampleSize_);                  // T=0:Insert Set1. Delete at t=2
+  SleepCompactCheck(2, 0, kSampleSize_, false); // T=2:Set1 should not be there
+  CloseTtl();
+}
+
+// Resets the timestamp of a set of kvs by updating them and checks that they
+// are not deleted according to the old timestamp
+TEST(TtlTest, ResetTimestamp) {
+  MakeKVMap(kSampleSize_);
+
+  OpenTtl(3);
+  PutValues(0, kSampleSize_);            // T=0: Insert Set1. Delete at t=3
+  env_->Sleep(2);                        // T=2
+  PutValues(0, kSampleSize_);            // T=2: Insert Set1. Delete at t=5
+  SleepCompactCheck(2, 0, kSampleSize_); // T=4: Set1 should still be there
+  CloseTtl();
+}
+
+// Similar to PresentDuringTTL but uses Iterator
+TEST(TtlTest, IterPresentDuringTTL) {
+  MakeKVMap(kSampleSize_);
+
+  OpenTtl(2);
+  PutValues(0, kSampleSize_);                 // T=0: Insert. Delete at t=2
+  SleepCompactCheckIter(1, 0, kSampleSize_);  // T=1: Set should be there
+  CloseTtl();
+}
+
+// Similar to AbsentAfterTTL but uses Iterator
+TEST(TtlTest, IterAbsentAfterTTL) {
+  MakeKVMap(kSampleSize_);
+
+  OpenTtl(1);
+  PutValues(0, kSampleSize_);                      // T=0: Insert. Delete at t=1
+  SleepCompactCheckIter(2, 0, kSampleSize_, false); // T=2: Should not be there
+  CloseTtl();
+}
+
+// Checks presence while opening the same db more than once with the same ttl
+// Note: The second open will open the same db
+TEST(TtlTest, MultiOpenSamePresent) {
+  MakeKVMap(kSampleSize_);
+
+  OpenTtl(2);
+  PutValues(0, kSampleSize_);                   // T=0: Insert. Delete at t=2
+  CloseTtl();
+
+  OpenTtl(2);                                  // T=0. Delete at t=2
+  SleepCompactCheck(1, 0, kSampleSize_);        // T=1: Set should be there
+  CloseTtl();
+}
+
+// Checks absence while opening the same db more than once with the same ttl
+// Note: The second open will open the same db
+TEST(TtlTest, MultiOpenSameAbsent) {
+  MakeKVMap(kSampleSize_);
+
+  OpenTtl(1);
+  PutValues(0, kSampleSize_);                   // T=0: Insert. Delete at t=1
+  CloseTtl();
+
+  OpenTtl(1);                                  // T=0.Delete at t=1
+  SleepCompactCheck(2, 0, kSampleSize_, false); // T=2: Set should not be there
+  CloseTtl();
+}
+
+// Checks presence while opening the same db more than once with bigger ttl
+TEST(TtlTest, MultiOpenDifferent) {
+  MakeKVMap(kSampleSize_);
+
+  OpenTtl(1);
+  PutValues(0, kSampleSize_);            // T=0: Insert. Delete at t=1
+  CloseTtl();
+
+  OpenTtl(3);                           // T=0: Set deleted at t=3
+  SleepCompactCheck(2, 0, kSampleSize_); // T=2: Set should be there
+  CloseTtl();
+}
+
+// Checks presence during ttl in read_only mode
+TEST(TtlTest, ReadOnlyPresentForever) {
+  MakeKVMap(kSampleSize_);
+
+  OpenTtl(1);                                 // T=0:Open the db normally
+  PutValues(0, kSampleSize_);                  // T=0:Insert Set1. Delete at t=1
+  CloseTtl();
+
+  OpenReadOnlyTtl(1);
+  SleepCompactCheck(2, 0, kSampleSize_);       // T=2:Set1 should still be there
+  CloseTtl();
+}
+
+// Checks whether WriteBatch works well with TTL
+// Puts all kvs in kvmap_ in a batch and writes first, then deletes first half
+TEST(TtlTest, WriteBatchTest) {
+  MakeKVMap(kSampleSize_);
+  BatchOperation batch_ops[kSampleSize_];
+  for (int i = 0; i < kSampleSize_; i++) {
+    batch_ops[i] = PUT;
+  }
+
+  OpenTtl(2);
+  MakePutWriteBatch(batch_ops, kSampleSize_);
+  for (int i = 0; i < kSampleSize_ / 2; i++) {
+    batch_ops[i] = DELETE;
+  }
+  MakePutWriteBatch(batch_ops, kSampleSize_ / 2);
+  SleepCompactCheck(0, 0, kSampleSize_ / 2, false);
+  SleepCompactCheck(0, kSampleSize_ / 2, kSampleSize_ - kSampleSize_ / 2);
+  CloseTtl();
+}
+
+// Checks user's compaction filter for correctness with TTL logic
+TEST(TtlTest, CompactionFilter) {
+  MakeKVMap(kSampleSize_);
+
+  OpenTtlWithTestCompaction(1);
+  PutValues(0, kSampleSize_);                  // T=0:Insert Set1. Delete at t=1
+  // T=2: TTL logic takes precedence over TestFilter:-Set1 should not be there
+  SleepCompactCheck(2, 0, kSampleSize_, false);
+  CloseTtl();
+
+  OpenTtlWithTestCompaction(3);
+  PutValues(0, kSampleSize_);                   // T=0:Insert Set1.
+  int partition = kSampleSize_ / 3;
+  SleepCompactCheck(1, 0, partition, false);   // Part dropped
+  SleepCompactCheck(0, partition, partition);  // Part kept
+  SleepCompactCheck(0, 2 * partition, partition, true, true); // Part changed
+  CloseTtl();
+}
+
+// Insert some key-values which KeyMayExist should be able to get and check that
+// values returned are fine
+TEST(TtlTest, KeyMayExist) {
+  MakeKVMap(kSampleSize_);
+
+  OpenTtl();
+  PutValues(0, kSampleSize_, false);
+
+  SimpleKeyMayExistCheck();
+
+  CloseTtl();
+}
+
+TEST(TtlTest, ColumnFamiliesTest) {
+  DB* db;
+  Options options;
+  options.create_if_missing = true;
+  options.env = env_.get();
+
+  DB::Open(options, dbname_, &db);
+  ColumnFamilyHandle* handle;
+  ASSERT_OK(db->CreateColumnFamily(ColumnFamilyOptions(options),
+                                   "ttl_column_family", &handle));
+
+  delete handle;
+  delete db;
+
+  std::vector<ColumnFamilyDescriptor> column_families;
+  column_families.push_back(ColumnFamilyDescriptor(
+      kDefaultColumnFamilyName, ColumnFamilyOptions(options)));
+  column_families.push_back(ColumnFamilyDescriptor(
+      "ttl_column_family", ColumnFamilyOptions(options)));
+
+  std::vector<ColumnFamilyHandle*> handles;
+
+  ASSERT_OK(DBWithTTL::Open(DBOptions(options), dbname_, column_families,
+                            &handles, &db_ttl_, {3, 5}, false));
+  ASSERT_EQ(handles.size(), 2U);
+  ColumnFamilyHandle* new_handle;
+  ASSERT_OK(db_ttl_->CreateColumnFamilyWithTtl(options, "ttl_column_family_2",
+                                               &new_handle, 2));
+  handles.push_back(new_handle);
+
+  MakeKVMap(kSampleSize_);
+  PutValues(0, kSampleSize_, false, handles[0]);
+  PutValues(0, kSampleSize_, false, handles[1]);
+  PutValues(0, kSampleSize_, false, handles[2]);
+
+  // everything should be there after 1 second
+  SleepCompactCheck(1, 0, kSampleSize_, true, false, handles[0]);
+  SleepCompactCheck(0, 0, kSampleSize_, true, false, handles[1]);
+  SleepCompactCheck(0, 0, kSampleSize_, true, false, handles[2]);
+
+  // only column family 1 should be alive after 4 seconds
+  SleepCompactCheck(3, 0, kSampleSize_, false, false, handles[0]);
+  SleepCompactCheck(0, 0, kSampleSize_, true, false, handles[1]);
+  SleepCompactCheck(0, 0, kSampleSize_, false, false, handles[2]);
+
+  // nothing should be there after 6 seconds
+  SleepCompactCheck(2, 0, kSampleSize_, false, false, handles[0]);
+  SleepCompactCheck(0, 0, kSampleSize_, false, false, handles[1]);
+  SleepCompactCheck(0, 0, kSampleSize_, false, false, handles[2]);
+
+  for (auto h : handles) {
+    delete h;
+  }
+  delete db_ttl_;
+  db_ttl_ = nullptr;
+}
+
+} //  namespace rocksdb
+
+// A black-box test for the ttl wrapper around rocksdb
+int main(int argc, char** argv) {
+  return rocksdb::test::RunAllTests();
+}