Add the property block for the plain table

Summary:
This is the last diff that adds the property block to plain table.
The format resembles that of the block-based table: https://github.com/facebook/rocksdb/wiki/Rocksdb-table-format

  [data block]
  [meta block 1: stats block]
  [meta block 2: future extended block]
  ...
  [meta block K: future extended block]  (we may add more meta blocks in the future)
  [metaindex block]
  [index block: we only have the placeholder here, we can add persistent index block in the future]
  [Footer: contains magic number, handle to metaindex block and index block]
  <end_of_file>

Test Plan: extended existing property block test.

Reviewers: haobo, sdong, dhruba

CC: leveldb

Differential Revision: https://reviews.facebook.net/D14523
This commit is contained in:
Kai Liu
2013-12-05 16:51:26 -08:00
parent 5f5e5fc2e9
commit 2e9efcd6d8
13 changed files with 578 additions and 307 deletions

View File

@@ -15,17 +15,22 @@
#include "db/db_statistics.h"
#include "db/memtable.h"
#include "db/write_batch_internal.h"
#include "rocksdb/cache.h"
#include "rocksdb/db.h"
#include "rocksdb/plain_table_factory.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "rocksdb/memtablerep.h"
#include "table/meta_blocks.h"
#include "table/block_based_table_builder.h"
#include "table/block_based_table_factory.h"
#include "table/block_based_table_reader.h"
#include "table/block_builder.h"
#include "table/block.h"
#include "table/format.h"
#include "util/random.h"
#include "util/testharness.h"
#include "util/testutil.h"
@@ -743,49 +748,6 @@ class Harness {
Constructor* constructor_;
};
// Test the empty key
TEST(Harness, SimpleEmptyKey) {
std::vector<TestArgs> args = GenerateArgList();
for (unsigned int i = 0; i < args.size(); i++) {
Init(args[i]);
Random rnd(test::RandomSeed() + 1);
Add("", "v");
Test(&rnd);
}
}
TEST(Harness, SimpleSingle) {
std::vector<TestArgs> args = GenerateArgList();
for (unsigned int i = 0; i < args.size(); i++) {
Init(args[i]);
Random rnd(test::RandomSeed() + 2);
Add("abc", "v");
Test(&rnd);
}
}
TEST(Harness, SimpleMulti) {
std::vector<TestArgs> args = GenerateArgList();
for (unsigned int i = 0; i < args.size(); i++) {
Init(args[i]);
Random rnd(test::RandomSeed() + 3);
Add("abc", "v");
Add("abcd", "v");
Add("ac", "v2");
Test(&rnd);
}
}
TEST(Harness, SimpleSpecialKey) {
std::vector<TestArgs> args = GenerateArgList();
for (unsigned int i = 0; i < args.size(); i++) {
Init(args[i]);
Random rnd(test::RandomSeed() + 4);
Add("\xff\xff", "v3");
Test(&rnd);
}
}
static bool Between(uint64_t val, uint64_t low, uint64_t high) {
bool result = (val >= low) && (val <= high);
if (!result) {
@@ -801,7 +763,7 @@ class TableTest { };
// This test include all the basic checks except those for index size and block
// size, which will be conducted in separated unit tests.
TEST(TableTest, BasicTableProperties) {
TEST(TableTest, BasicBlockedBasedTableProperties) {
BlockBasedTableConstructor c(BytewiseComparator());
c.Add("a1", "val1");
@@ -845,6 +807,47 @@ TEST(TableTest, BasicTableProperties) {
);
}
extern const uint64_t kPlainTableMagicNumber;
TEST(TableTest, BasicPlainTableProperties) {
PlainTableFactory factory(8, 8, 0);
StringSink sink;
std::unique_ptr<TableBuilder> builder(factory.GetTableBuilder(
Options(),
&sink,
kNoCompression
));
for (char c = 'a'; c <= 'z'; ++c) {
std::string key(16, c);
std::string value(28, c + 42);
builder->Add(key, value);
}
ASSERT_OK(builder->Finish());
StringSource source(sink.contents(), 72242);
TableProperties props;
auto s = ReadTableProperties(
&source,
sink.contents().size(),
kPlainTableMagicNumber,
Env::Default(),
nullptr,
&props
);
ASSERT_OK(s);
ASSERT_EQ(0ul, props.index_size);
ASSERT_EQ(0ul, props.filter_size);
ASSERT_EQ(16ul * 26, props.raw_key_size);
ASSERT_EQ(28ul * 26, props.raw_value_size);
ASSERT_EQ(26ul, props.num_entries);
ASSERT_EQ(1ul, props.num_data_blocks);
// User collected keys
// internal keys
}
TEST(TableTest, FilterPolicyNameProperties) {
BlockBasedTableConstructor c(BytewiseComparator());
c.Add("a1", "val1");
@@ -1292,6 +1295,48 @@ TEST(MemTableTest, Simple) {
delete memtable->Unref();
}
// Test the empty key
TEST(Harness, SimpleEmptyKey) {
std::vector<TestArgs> args = GenerateArgList();
for (unsigned int i = 0; i < args.size(); i++) {
Init(args[i]);
Random rnd(test::RandomSeed() + 1);
Add("", "v");
Test(&rnd);
}
}
TEST(Harness, SimpleSingle) {
std::vector<TestArgs> args = GenerateArgList();
for (unsigned int i = 0; i < args.size(); i++) {
Init(args[i]);
Random rnd(test::RandomSeed() + 2);
Add("abc", "v");
Test(&rnd);
}
}
TEST(Harness, SimpleMulti) {
std::vector<TestArgs> args = GenerateArgList();
for (unsigned int i = 0; i < args.size(); i++) {
Init(args[i]);
Random rnd(test::RandomSeed() + 3);
Add("abc", "v");
Add("abcd", "v");
Add("ac", "v2");
Test(&rnd);
}
}
TEST(Harness, SimpleSpecialKey) {
std::vector<TestArgs> args = GenerateArgList();
for (unsigned int i = 0; i < args.size(); i++) {
Init(args[i]);
Random rnd(test::RandomSeed() + 4);
Add("\xff\xff", "v3");
Test(&rnd);
}
}
} // namespace rocksdb