Add the property block for the plain table

Summary:
This is the last diff that adds the property block to plain table.
The format resembles that of the block-based table: https://github.com/facebook/rocksdb/wiki/Rocksdb-table-format

  [data block]
  [meta block 1: stats block]
  [meta block 2: future extended block]
  ...
  [meta block K: future extended block]  (we may add more meta blocks in the future)
  [metaindex block]
  [index block: we only have the placeholder here, we can add persistent index block in the future]
  [Footer: contains magic number, handle to metaindex block and index block]
  <end_of_file>

Test Plan: extended existing property block test.

Reviewers: haobo, sdong, dhruba

CC: leveldb

Differential Revision: https://reviews.facebook.net/D14523
This commit is contained in:
Kai Liu
2013-12-05 16:51:26 -08:00
parent 5f5e5fc2e9
commit 2e9efcd6d8
13 changed files with 578 additions and 307 deletions

View File

@@ -19,6 +19,7 @@
#include "table/block.h"
#include "table/filter_block.h"
#include "table/format.h"
#include "table/meta_blocks.h"
#include "table/two_level_iterator.h"
#include "util/coding.h"
@@ -41,6 +42,7 @@ public:
namespace rocksdb {
extern const uint64_t kPlainTableMagicNumber;
static uint32_t getBucketId(Slice const& s, size_t prefix_len,
uint32_t num_buckets) {
return MurmurHash(s.data(), prefix_len, 397) % num_buckets;
@@ -49,18 +51,16 @@ static uint32_t getBucketId(Slice const& s, size_t prefix_len,
PlainTableReader::PlainTableReader(const EnvOptions& storage_options,
uint64_t file_size, int user_key_size,
int key_prefix_len, int bloom_bits_per_key,
double hash_table_ratio) :
double hash_table_ratio,
const TableProperties& table_properties) :
hash_table_size_(0), soptions_(storage_options), file_size_(file_size),
user_key_size_(user_key_size), key_prefix_len_(key_prefix_len),
hash_table_ratio_(hash_table_ratio) {
if (bloom_bits_per_key > 0) {
filter_policy_ = NewBloomFilterPolicy(bloom_bits_per_key);
} else {
filter_policy_ = nullptr;
}
hash_table_ = nullptr;
data_start_offset_ = 0;
data_end_offset_ = file_size;
hash_table_ratio_(hash_table_ratio),
filter_policy_(bloom_bits_per_key > 0 ?
NewBloomFilterPolicy(bloom_bits_per_key) : nullptr),
table_properties_(table_properties),
data_start_offset_(0),
data_end_offset_(table_properties_.data_size) {
}
PlainTableReader::~PlainTableReader() {
@@ -87,19 +87,38 @@ Status PlainTableReader::Open(const Options& options,
return Status::NotSupported("File is too large for PlainTableReader!");
}
PlainTableReader* t = new PlainTableReader(soptions, file_size,
user_key_size,
key_prefix_len,
bloom_num_bits,
hash_table_ratio);
t->file_ = std::move(file);
t->options_ = options;
Status s = t->PopulateIndex(file_size);
TableProperties table_properties;
auto s = ReadTableProperties(
file.get(),
file_size,
kPlainTableMagicNumber,
options.env,
options.info_log.get(),
&table_properties
);
if (!s.ok()) {
delete t;
return s;
}
table_reader->reset(t);
std::unique_ptr<PlainTableReader> new_reader(new PlainTableReader(
soptions,
file_size,
user_key_size,
key_prefix_len,
bloom_num_bits,
hash_table_ratio,
table_properties
));
new_reader->file_ = std::move(file);
new_reader->options_ = options;
// -- Populate Index
s = new_reader->PopulateIndex();
if (!s.ok()) {
return s;
}
*table_reader = std::move(new_reader);
return s;
}
@@ -114,7 +133,7 @@ Iterator* PlainTableReader::NewIterator(const ReadOptions& options) {
return new PlainTableIterator(this);
}
Status PlainTableReader::PopulateIndex(uint64_t file_size) {
Status PlainTableReader::PopulateIndex() {
// Get mmapped memory to file_data_.
Status s = file_->Read(0, file_size_, &file_data_, nullptr);
if (!s.ok()) {
@@ -124,7 +143,6 @@ Status PlainTableReader::PopulateIndex(uint64_t file_size) {
version_ ^= 0x80000000;
assert(version_ == 1);
data_start_offset_ = 4;
data_end_offset_ = file_size;
Slice key_slice;
Slice key_prefix_slice;
@@ -140,7 +158,7 @@ Status PlainTableReader::PopulateIndex(uint64_t file_size) {
// are in order.
std::vector<std::pair<Slice, std::string>> prefix_index_pairs;
std::string current_prefix_index;
while (pos < file_size) {
while (pos < data_end_offset_) {
uint32_t key_offset = pos;
status_ = Next(pos, &key_slice, &value_slice, pos);
key_prefix_slice = Slice(key_slice.data(), key_prefix_len_);