Add the property block for the plain table

Summary:
This is the last diff that adds the property block to plain table.
The format resembles that of the block-based table: https://github.com/facebook/rocksdb/wiki/Rocksdb-table-format

  [data block]
  [meta block 1: stats block]
  [meta block 2: future extended block]
  ...
  [meta block K: future extended block]  (we may add more meta blocks in the future)
  [metaindex block]
  [index block: we only have the placeholder here, we can add persistent index block in the future]
  [Footer: contains magic number, handle to metaindex block and index block]
  <end_of_file>

Test Plan: extended existing property block test.

Reviewers: haobo, sdong, dhruba

CC: leveldb

Differential Revision: https://reviews.facebook.net/D14523
This commit is contained in:
Kai Liu
2013-12-05 16:51:26 -08:00
parent 5f5e5fc2e9
commit 2e9efcd6d8
13 changed files with 578 additions and 307 deletions

View File

@@ -14,12 +14,40 @@
#include "table/block_builder.h"
#include "table/filter_block.h"
#include "table/format.h"
#include "table/meta_blocks.h"
#include "util/coding.h"
#include "util/crc32c.h"
#include "util/stop_watch.h"
namespace rocksdb {
namespace {
// a utility that helps writing block content to the file
// @offset will advance if @block_contents was successfully written.
// @block_handle the block handle this particular block.
Status WriteBlock(
const Slice& block_contents,
WritableFile* file,
uint64_t* offset,
BlockHandle* block_handle) {
block_handle->set_offset(*offset);
block_handle->set_size(block_contents.size());
Status s = file->Append(block_contents);
if (s.ok()) {
*offset += block_contents.size();
}
return s;
}
} // namespace
// kPlainTableMagicNumber was picked by running
// echo rocksdb.plain.table | sha1sum
// and taking the leading 64 bits.
extern const uint64_t kPlainTableMagicNumber = 0x4f3418eb7a8f13b8ull;
PlainTableBuilder::PlainTableBuilder(const Options& options,
WritableFile* file,
int user_key_size, int key_prefix_len) :
@@ -28,15 +56,18 @@ PlainTableBuilder::PlainTableBuilder(const Options& options,
PutFixed32(&version, 1 | 0x80000000);
file_->Append(Slice(version));
offset_ = 4;
// for plain table, we put all the data in a big chuck.
properties_.num_data_blocks = 1;
// emphasize that currently plain table doesn't have persistent index or
// filter block.
properties_.index_size = 0;
properties_.filter_size = 0;
}
PlainTableBuilder::~PlainTableBuilder() {
}
Status PlainTableBuilder::ChangeOptions(const Options& options) {
return Status::OK();
}
void PlainTableBuilder::Add(const Slice& key, const Slice& value) {
assert((int) key.size() == GetInternalKeyLength());
@@ -52,7 +83,17 @@ void PlainTableBuilder::Add(const Slice& key, const Slice& value) {
file_->Append(value);
offset_ += value_size + size.length();
num_entries_++;
properties_.num_entries++;
properties_.raw_key_size += key.size();
properties_.raw_value_size += value.size();
// notify property collectors
NotifyCollectTableCollectorsOnAdd(
key,
value,
options_.table_properties_collectors,
options_.info_log.get()
);
}
Status PlainTableBuilder::status() const {
@@ -62,7 +103,63 @@ Status PlainTableBuilder::status() const {
Status PlainTableBuilder::Finish() {
assert(!closed_);
closed_ = true;
return Status::OK();
properties_.data_size = offset_;
// Write the following blocks
// 1. [meta block: properties]
// 2. [metaindex block]
// 3. [footer]
MetaIndexBuilder meta_index_builer;
PropertyBlockBuilder property_block_builder;
// -- Add basic properties
property_block_builder.AddTableProperty(properties_);
// -- Add user collected properties
NotifyCollectTableCollectorsOnFinish(
options_.table_properties_collectors,
options_.info_log.get(),
&property_block_builder
);
// -- Write property block
BlockHandle property_block_handle;
auto s = WriteBlock(
property_block_builder.Finish(),
file_,
&offset_,
&property_block_handle
);
if (!s.ok()) {
return s;
}
meta_index_builer.Add(kPropertiesBlock, property_block_handle);
// -- write metaindex block
BlockHandle metaindex_block_handle;
s = WriteBlock(
meta_index_builer.Finish(),
file_,
&offset_,
&metaindex_block_handle
);
if (!s.ok()) {
return s;
}
// Write Footer
Footer footer(kPlainTableMagicNumber);
footer.set_metaindex_handle(metaindex_block_handle);
footer.set_index_handle(BlockHandle::NullBlockHandle());
std::string footer_encoding;
footer.EncodeTo(&footer_encoding);
s = file_->Append(footer_encoding);
if (s.ok()) {
offset_ += footer_encoding.size();
}
return s;
}
void PlainTableBuilder::Abandon() {
@@ -70,7 +167,7 @@ void PlainTableBuilder::Abandon() {
}
uint64_t PlainTableBuilder::NumEntries() const {
return num_entries_;
return properties_.num_entries;
}
uint64_t PlainTableBuilder::FileSize() const {