mirror of
https://github.com/XRPLF/rippled.git
synced 2025-12-06 17:27:55 +00:00
A Simple Plain Table
Summary: A Simple plain table format. No block structure. When creating the table reader, scanning the full table to create indexes. Test Plan:Add unit test Reviewers:haobo,dhruba,kailiu CC: Task ID: # Blame Rev:
This commit is contained in:
Siying Dong
332
db/plain_table_db_test.cc
Normal file
332
db/plain_table_db_test.cc
Normal file
@@ -0,0 +1,332 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
#include <algorithm>
|
||||
#include <set>
|
||||
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/filter_policy.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/version_set.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "db/db_statistics.h"
|
||||
#include "rocksdb/cache.h"
|
||||
#include "rocksdb/compaction_filter.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "rocksdb/plain_table_factory.h"
|
||||
#include "util/hash.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/mutexlock.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
#include "utilities/merge_operators.h"
|
||||
|
||||
using std::unique_ptr;
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
class PlainTableDBTest {
|
||||
protected:
|
||||
public:
|
||||
std::string dbname_;
|
||||
Env* env_;
|
||||
DB* db_;
|
||||
|
||||
Options last_options_;
|
||||
|
||||
PlainTableDBTest() :
|
||||
env_(Env::Default()) {
|
||||
dbname_ = test::TmpDir() + "/plain_table_db_test";
|
||||
ASSERT_OK(DestroyDB(dbname_, Options()));
|
||||
db_ = nullptr;
|
||||
Reopen();
|
||||
}
|
||||
|
||||
~PlainTableDBTest() {
|
||||
delete db_;
|
||||
ASSERT_OK(DestroyDB(dbname_, Options()));
|
||||
}
|
||||
|
||||
// Return the current option configuration.
|
||||
Options CurrentOptions() {
|
||||
Options options;
|
||||
options.table_factory.reset(new PlainTableFactory(16, 8));
|
||||
options.allow_mmap_reads = true;
|
||||
return options;
|
||||
}
|
||||
|
||||
DBImpl* dbfull() {
|
||||
return reinterpret_cast<DBImpl*>(db_);
|
||||
}
|
||||
|
||||
void Reopen(Options* options = nullptr) {
|
||||
ASSERT_OK(TryReopen(options));
|
||||
}
|
||||
|
||||
void Close() {
|
||||
delete db_;
|
||||
db_ = nullptr;
|
||||
}
|
||||
|
||||
void DestroyAndReopen(Options* options = nullptr) {
|
||||
//Destroy using last options
|
||||
Destroy(&last_options_);
|
||||
ASSERT_OK(TryReopen(options));
|
||||
}
|
||||
|
||||
void Destroy(Options* options) {
|
||||
delete db_;
|
||||
db_ = nullptr;
|
||||
ASSERT_OK(DestroyDB(dbname_, *options));
|
||||
}
|
||||
|
||||
Status PureReopen(Options* options, DB** db) {
|
||||
return DB::Open(*options, dbname_, db);
|
||||
}
|
||||
|
||||
Status TryReopen(Options* options = nullptr) {
|
||||
delete db_;
|
||||
db_ = nullptr;
|
||||
Options opts;
|
||||
if (options != nullptr) {
|
||||
opts = *options;
|
||||
} else {
|
||||
opts = CurrentOptions();
|
||||
opts.create_if_missing = true;
|
||||
}
|
||||
last_options_ = opts;
|
||||
|
||||
return DB::Open(opts, dbname_, &db_);
|
||||
}
|
||||
|
||||
Status Put(const Slice& k, const Slice& v) {
|
||||
return db_->Put(WriteOptions(), k, v);
|
||||
}
|
||||
|
||||
Status Delete(const std::string& k) {
|
||||
return db_->Delete(WriteOptions(), k);
|
||||
}
|
||||
|
||||
std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
|
||||
ReadOptions options;
|
||||
options.snapshot = snapshot;
|
||||
std::string result;
|
||||
Status s = db_->Get(options, k, &result);
|
||||
if (s.IsNotFound()) {
|
||||
result = "NOT_FOUND";
|
||||
} else if (!s.ok()) {
|
||||
result = s.ToString();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
int NumTableFilesAtLevel(int level) {
|
||||
std::string property;
|
||||
ASSERT_TRUE(
|
||||
db_->GetProperty("rocksdb.num-files-at-level" + NumberToString(level),
|
||||
&property));
|
||||
return atoi(property.c_str());
|
||||
}
|
||||
|
||||
// Return spread of files per level
|
||||
std::string FilesPerLevel() {
|
||||
std::string result;
|
||||
int last_non_zero_offset = 0;
|
||||
for (int level = 0; level < db_->NumberLevels(); level++) {
|
||||
int f = NumTableFilesAtLevel(level);
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
|
||||
result += buf;
|
||||
if (f > 0) {
|
||||
last_non_zero_offset = result.size();
|
||||
}
|
||||
}
|
||||
result.resize(last_non_zero_offset);
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string IterStatus(Iterator* iter) {
|
||||
std::string result;
|
||||
if (iter->Valid()) {
|
||||
result = iter->key().ToString() + "->" + iter->value().ToString();
|
||||
} else {
|
||||
result = "(invalid)";
|
||||
}
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
TEST(PlainTableDBTest, Empty) {
|
||||
ASSERT_TRUE(db_ != nullptr);
|
||||
ASSERT_EQ("NOT_FOUND", Get("0000000000000foo"));
|
||||
}
|
||||
|
||||
TEST(PlainTableDBTest, ReadWrite) {
|
||||
ASSERT_OK(Put("1000000000000foo", "v1"));
|
||||
ASSERT_EQ("v1", Get("1000000000000foo"));
|
||||
ASSERT_OK(Put("0000000000000bar", "v2"));
|
||||
ASSERT_OK(Put("1000000000000foo", "v3"));
|
||||
ASSERT_EQ("v3", Get("1000000000000foo"));
|
||||
ASSERT_EQ("v2", Get("0000000000000bar"));
|
||||
}
|
||||
|
||||
TEST(PlainTableDBTest, Flush) {
|
||||
ASSERT_OK(Put("1000000000000foo", "v1"));
|
||||
ASSERT_OK(Put("0000000000000bar", "v2"));
|
||||
ASSERT_OK(Put("1000000000000foo", "v3"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
ASSERT_EQ("v3", Get("1000000000000foo"));
|
||||
ASSERT_EQ("v2", Get("0000000000000bar"));
|
||||
}
|
||||
|
||||
TEST(PlainTableDBTest, Iterator) {
|
||||
ASSERT_OK(Put("1000000000foo002", "v_2"));
|
||||
ASSERT_OK(Put("0000000000000bar", "random"));
|
||||
ASSERT_OK(Put("1000000000foo001", "v1"));
|
||||
ASSERT_OK(Put("3000000000000bar", "bar_v"));
|
||||
ASSERT_OK(Put("1000000000foo003", "v__3"));
|
||||
ASSERT_OK(Put("1000000000foo004", "v__4"));
|
||||
ASSERT_OK(Put("1000000000foo005", "v__5"));
|
||||
ASSERT_OK(Put("1000000000foo007", "v__7"));
|
||||
ASSERT_OK(Put("1000000000foo008", "v__8"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
ASSERT_EQ("v1", Get("1000000000foo001"));
|
||||
ASSERT_EQ("v__3", Get("1000000000foo003"));
|
||||
ReadOptions ro;
|
||||
Iterator* iter = dbfull()->NewIterator(ro);
|
||||
iter->Seek("1000000000foo001");
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("1000000000foo001", iter->key().ToString());
|
||||
ASSERT_EQ("v1", iter->value().ToString());
|
||||
|
||||
iter->Next();
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("1000000000foo002", iter->key().ToString());
|
||||
ASSERT_EQ("v_2", iter->value().ToString());
|
||||
|
||||
iter->Next();
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("1000000000foo003", iter->key().ToString());
|
||||
ASSERT_EQ("v__3", iter->value().ToString());
|
||||
|
||||
iter->Next();
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("1000000000foo004", iter->key().ToString());
|
||||
ASSERT_EQ("v__4", iter->value().ToString());
|
||||
|
||||
iter->Seek("3000000000000bar");
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("3000000000000bar", iter->key().ToString());
|
||||
ASSERT_EQ("bar_v", iter->value().ToString());
|
||||
|
||||
iter->Seek("1000000000foo000");
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("1000000000foo001", iter->key().ToString());
|
||||
ASSERT_EQ("v1", iter->value().ToString());
|
||||
|
||||
iter->Seek("1000000000foo005");
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("1000000000foo005", iter->key().ToString());
|
||||
ASSERT_EQ("v__5", iter->value().ToString());
|
||||
|
||||
iter->Seek("1000000000foo006");
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("1000000000foo007", iter->key().ToString());
|
||||
ASSERT_EQ("v__7", iter->value().ToString());
|
||||
|
||||
iter->Seek("1000000000foo008");
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("1000000000foo008", iter->key().ToString());
|
||||
ASSERT_EQ("v__8", iter->value().ToString());
|
||||
|
||||
iter->Seek("1000000000foo009");
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("3000000000000bar", iter->key().ToString());
|
||||
|
||||
|
||||
delete iter;
|
||||
}
|
||||
|
||||
TEST(PlainTableDBTest, Flush2) {
|
||||
ASSERT_OK(Put("0000000000000bar", "b"));
|
||||
ASSERT_OK(Put("1000000000000foo", "v1"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
|
||||
ASSERT_OK(Put("1000000000000foo", "v2"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
ASSERT_EQ("v2", Get("1000000000000foo"));
|
||||
|
||||
ASSERT_OK(Put("0000000000000eee", "v3"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
ASSERT_EQ("v3", Get("0000000000000eee"));
|
||||
|
||||
ASSERT_OK(Delete("0000000000000bar"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
ASSERT_EQ("NOT_FOUND", Get("0000000000000bar"));
|
||||
|
||||
ASSERT_OK(Put("0000000000000eee", "v5"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
ASSERT_EQ("v5", Get("0000000000000eee"));
|
||||
}
|
||||
|
||||
static std::string Key(int i) {
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "key_______%06d", i);
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
static std::string RandomString(Random* rnd, int len) {
|
||||
std::string r;
|
||||
test::RandomString(rnd, len, &r);
|
||||
return r;
|
||||
}
|
||||
|
||||
TEST(PlainTableDBTest, CompactionTrigger) {
|
||||
Options options = CurrentOptions();
|
||||
options.write_buffer_size = 100 << 10; //100KB
|
||||
options.num_levels = 3;
|
||||
options.max_mem_compaction_level = 0;
|
||||
options.level0_file_num_compaction_trigger = 3;
|
||||
Reopen(&options);
|
||||
|
||||
Random rnd(301);
|
||||
|
||||
for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
|
||||
num++) {
|
||||
std::vector<std::string> values;
|
||||
// Write 120KB (12 values, each 10K)
|
||||
for (int i = 0; i < 12; i++) {
|
||||
values.push_back(RandomString(&rnd, 10000));
|
||||
ASSERT_OK(Put(Key(i), values[i]));
|
||||
}
|
||||
dbfull()->TEST_WaitForFlushMemTable();
|
||||
ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
|
||||
}
|
||||
|
||||
//generate one more file in level-0, and should trigger level-0 compaction
|
||||
std::vector<std::string> values;
|
||||
for (int i = 0; i < 12; i++) {
|
||||
values.push_back(RandomString(&rnd, 10000));
|
||||
ASSERT_OK(Put(Key(i), values[i]));
|
||||
}
|
||||
dbfull()->TEST_WaitForCompact();
|
||||
|
||||
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
||||
ASSERT_EQ(NumTableFilesAtLevel(1), 1);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
}
|
||||
Reference in New Issue
Block a user