ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into columnfamilies

Browse Source
This commit is contained in:
Igor Canadi
2014-02-07 11:42:56 -08:00
parent 0143abdbb0 3ce8d9a988
commit 2b8c44639a
13 changed files with 299 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
table/format.cc
View File

@@ -9,6 +9,8 @@
#include "table/format.h"
#include <string>
#include "port/port.h"
#include "rocksdb/env.h"
#include "table/block.h"
@@ -43,8 +45,8 @@ void Footer::EncodeTo(std::string* dst) const {
metaindex_handle_.EncodeTo(dst);
index_handle_.EncodeTo(dst);
dst->resize(2 * BlockHandle::kMaxEncodedLength); // Padding
PutFixed32(dst, static_cast<uint32_t>(kTableMagicNumber & 0xffffffffu));
PutFixed32(dst, static_cast<uint32_t>(kTableMagicNumber >> 32));
PutFixed32(dst, static_cast<uint32_t>(table_magic_number() & 0xffffffffu));
PutFixed32(dst, static_cast<uint32_t>(table_magic_number() >> 32));
assert(dst->size() == original_size + kEncodedLength);
}
@@ -52,13 +54,21 @@ Status Footer::DecodeFrom(Slice* input) {
assert(input != nullptr);
assert(input->size() >= kEncodedLength);
const char* magic_ptr = input->data() + kEncodedLength - 8;
const char* magic_ptr =
input->data() + kEncodedLength - kMagicNumberLengthByte;
const uint32_t magic_lo = DecodeFixed32(magic_ptr);
const uint32_t magic_hi = DecodeFixed32(magic_ptr + 4);
const uint64_t magic = ((static_cast<uint64_t>(magic_hi) << 32) |
(static_cast<uint64_t>(magic_lo)));
if (magic != kTableMagicNumber) {
return Status::InvalidArgument("not an sstable (bad magic number)");
if (HasInitializedTableMagicNumber()) {
if (magic != table_magic_number()) {
char buffer[80];
snprintf(buffer, sizeof(buffer) - 1,
"not an sstable (bad magic number --- %lx)", magic);
return Status::InvalidArgument(buffer);
}
} else {
set_table_magic_number(magic);
}
Status result = metaindex_handle_.DecodeFrom(input);
@@ -221,7 +231,7 @@ Status UncompressBlockContents(const char* data, size_t n,
default:
return Status::Corruption("bad block type");
}
result->compression_type = kNoCompression; // not compressed any more
result->compression_type = kNoCompression; // not compressed any more
return Status::OK();
}

45
table/format.h
View File

@@ -21,6 +21,9 @@ class Block;
class RandomAccessFile;
struct ReadOptions;
// the length of the magic number in bytes.
const int kMagicNumberLengthByte = 8;
// BlockHandle is a pointer to the extent of a file that stores a data
// block or a meta block.
class BlockHandle {
@@ -63,12 +66,16 @@ class BlockHandle {
// end of every table file.
class Footer {
public:
// Constructs a footer without specifying its table magic number.
// In such case, the table magic number of such footer should be
// initialized via @ReadFooterFromFile().
Footer() : Footer(kInvalidTableMagicNumber) {}
// @table_magic_number serves two purposes:
// 1. Identify different types of the tables.
// 2. Help us to identify if a given file is a valid sst.
Footer(uint64_t table_magic_number) :
kTableMagicNumber(table_magic_number) {
}
explicit Footer(uint64_t table_magic_number)
: table_magic_number_(table_magic_number) {}
// The block handle for the metaindex block of the table
const BlockHandle& metaindex_handle() const { return metaindex_handle_; }
@@ -78,24 +85,52 @@ class Footer {
const BlockHandle& index_handle() const {
return index_handle_;
}
void set_index_handle(const BlockHandle& h) {
index_handle_ = h;
}
uint64_t table_magic_number() const { return table_magic_number_; }
void EncodeTo(std::string* dst) const;
// Set the current footer based on the input slice. If table_magic_number_
// is not set (i.e., HasInitializedTableMagicNumber() is true), then this
// function will also initialize table_magic_number_. Otherwise, this
// function will verify whether the magic number specified in the input
// slice matches table_magic_number_ and update the current footer only
// when the test passes.
Status DecodeFrom(Slice* input);
// Encoded length of a Footer. Note that the serialization of a
// Footer will always occupy exactly this many bytes. It consists
// of two block handles and a magic number.
enum {
kEncodedLength = 2*BlockHandle::kMaxEncodedLength + 8
kEncodedLength = 2 * BlockHandle::kMaxEncodedLength + 8
};
const uint64_t kInvalidTableMagicNumber = 0;
private:
// Set the table_magic_number only when it was not previously
// initialized. Return true on success.
bool set_table_magic_number(uint64_t magic_number) {
if (HasInitializedTableMagicNumber()) {
table_magic_number_ = magic_number;
return true;
}
return false;
}
// return true if @table_magic_number_ is set to a value different
// from @kInvalidTableMagicNumber.
bool HasInitializedTableMagicNumber() const {
return (table_magic_number_ != kInvalidTableMagicNumber);
}
BlockHandle metaindex_handle_;
BlockHandle index_handle_;
const uint64_t kTableMagicNumber;
uint64_t table_magic_number_;
};
// Read the footer from file

105
table/meta_blocks.cc
View File

@@ -2,12 +2,13 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#include "table/meta_blocks.h"
#include <map>
#include <string>
#include "rocksdb/table.h"
#include "rocksdb/table_properties.h"
#include "table/block.h"
#include "table/format.h"
#include "util/coding.h"
@@ -104,9 +105,8 @@ bool NotifyCollectTableCollectorsOnAdd(
Status s = collector->Add(key, value);
all_succeeded = all_succeeded && s.ok();
if (!s.ok()) {
LogPropertiesCollectionError(
info_log, "Add", /* method */ collector->Name()
);
LogPropertiesCollectionError(info_log, "Add" /* method */,
collector->Name());
}
}
return all_succeeded;
@@ -123,9 +123,8 @@ bool NotifyCollectTableCollectorsOnFinish(
all_succeeded = all_succeeded && s.ok();
if (!s.ok()) {
LogPropertiesCollectionError(
info_log, "Finish", /* method */ collector->Name()
);
LogPropertiesCollectionError(info_log, "Finish" /* method */,
collector->Name());
} else {
builder->Add(user_collected_properties);
}
@@ -151,14 +150,8 @@ Status ReadProperties(
BlockContents block_contents;
ReadOptions read_options;
read_options.verify_checksums = false;
Status s = ReadBlockContents(
file,
read_options,
handle,
&block_contents,
env,
false
);
Status s = ReadBlockContents(file, read_options, handle, &block_contents, env,
false);
if (!s.ok()) {
return s;
@@ -166,22 +159,20 @@ Status ReadProperties(
Block properties_block(block_contents);
std::unique_ptr<Iterator> iter(
properties_block.NewIterator(BytewiseComparator())
);
properties_block.NewIterator(BytewiseComparator()));
// All pre-defined properties of type uint64_t
std::unordered_map<std::string, uint64_t*> predefined_uint64_properties = {
{ TablePropertiesNames::kDataSize, &table_properties->data_size },
{ TablePropertiesNames::kIndexSize, &table_properties->index_size },
{ TablePropertiesNames::kFilterSize, &table_properties->filter_size },
{ TablePropertiesNames::kRawKeySize, &table_properties->raw_key_size },
{ TablePropertiesNames::kRawValueSize, &table_properties->raw_value_size },
{ TablePropertiesNames::kNumDataBlocks,
&table_properties->num_data_blocks },
{ TablePropertiesNames::kNumEntries, &table_properties->num_entries },
{ TablePropertiesNames::kFormatVersion, &table_properties->format_version },
{ TablePropertiesNames::kFixedKeyLen, &table_properties->fixed_key_len },
};
{TablePropertiesNames::kDataSize, &table_properties->data_size},
{TablePropertiesNames::kIndexSize, &table_properties->index_size},
{TablePropertiesNames::kFilterSize, &table_properties->filter_size},
{TablePropertiesNames::kRawKeySize, &table_properties->raw_key_size},
{TablePropertiesNames::kRawValueSize, &table_properties->raw_value_size},
{TablePropertiesNames::kNumDataBlocks,
&table_properties->num_data_blocks},
{TablePropertiesNames::kNumEntries, &table_properties->num_entries},
{TablePropertiesNames::kFormatVersion, &table_properties->format_version},
{TablePropertiesNames::kFixedKeyLen, &table_properties->fixed_key_len}};
std::string last_key;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
@@ -192,10 +183,8 @@ Status ReadProperties(
auto key = iter->key().ToString();
// properties block is strictly sorted with no duplicate key.
assert(
last_key.empty() ||
BytewiseComparator()->Compare(key, last_key) > 0
);
assert(last_key.empty() ||
BytewiseComparator()->Compare(key, last_key) > 0);
last_key = key;
auto raw_val = iter->value();
@@ -218,8 +207,7 @@ Status ReadProperties(
} else {
// handle user-collected properties
table_properties->user_collected_properties.insert(
std::make_pair(key, raw_val.ToString())
);
{key, raw_val.ToString()});
}
}
@@ -244,21 +232,14 @@ Status ReadTableProperties(
BlockContents metaindex_contents;
ReadOptions read_options;
read_options.verify_checksums = false;
s = ReadBlockContents(
file,
read_options,
metaindex_handle,
&metaindex_contents,
env,
false
);
s = ReadBlockContents(file, read_options, metaindex_handle,
&metaindex_contents, env, false);
if (!s.ok()) {
return s;
}
Block metaindex_block(metaindex_contents);
std::unique_ptr<Iterator> meta_iter(
metaindex_block.NewIterator(BytewiseComparator())
);
metaindex_block.NewIterator(BytewiseComparator()));
// -- Read property block
meta_iter->Seek(kPropertiesBlock);
@@ -266,21 +247,39 @@ Status ReadTableProperties(
if (meta_iter->Valid() &&
meta_iter->key() == kPropertiesBlock &&
meta_iter->status().ok()) {
s = ReadProperties(
meta_iter->value(),
file,
env,
info_log,
properties
);
s = ReadProperties(meta_iter->value(), file, env, info_log, properties);
} else {
s = Status::Corruption(
"Unable to read the property block from the plain table"
);
"Unable to read the property block from the plain table");
}
return s;
}
Status ReadTableMagicNumber(const std::string& file_path,
const Options& options,
const EnvOptions& env_options,
uint64_t* table_magic_number) {
unique_ptr<RandomAccessFile> file;
Status s = options.env->NewRandomAccessFile(file_path, &file, env_options);
if (!s.ok()) {
return s;
}
uint64_t file_size;
options.env->GetFileSize(file_path, &file_size);
if (file_size < Footer::kEncodedLength) {
return Status::InvalidArgument("file is too short to be an sstable");
}
Footer footer;
s = ReadFooterFromFile(file.get(), file_size, &footer);
if (!s.ok()) {
return s;
}
*table_magic_number = footer.table_magic_number();
return Status::OK();
}
} // namespace rocksdb

7
table/meta_blocks.h
View File

@@ -8,6 +8,7 @@
#include <memory>
#include <string>
#include "db/builder.h"
#include "rocksdb/comparator.h"
#include "rocksdb/options.h"
#include "rocksdb/slice.h"
@@ -118,4 +119,10 @@ Status ReadTableProperties(
Logger* info_log,
TableProperties* properties);
// Read the magic number of the specified file directly. The magic number
// of a valid sst table the last 8-byte of the file.
Status ReadTableMagicNumber(const std::string& file_path,
const Options& options,
const EnvOptions& env_options,
uint64_t* table_magic_number);
} // namespace rocksdb

51
table/table_properties.cc
View File

@@ -40,50 +40,31 @@ std::string TableProperties::ToString(
result.reserve(1024);
// Basic Info
AppendProperty(
result, "# data blocks", num_data_blocks, prop_delim, kv_delim
);
AppendProperty(result, "# data blocks", num_data_blocks, prop_delim,
kv_delim);
AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim);
AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim);
AppendProperty(
result,
"raw average key size",
num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0,
prop_delim,
kv_delim
);
AppendProperty(
result, "raw value size", raw_value_size, prop_delim, kv_delim
);
AppendProperty(
result,
"raw average value size",
num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0,
prop_delim,
kv_delim
);
AppendProperty(result, "raw average key size",
num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0,
prop_delim, kv_delim);
AppendProperty(result, "raw value size", raw_value_size, prop_delim,
kv_delim);
AppendProperty(result, "raw average value size",
num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0,
prop_delim, kv_delim);
AppendProperty(result, "data block size", data_size, prop_delim, kv_delim);
AppendProperty(result, "index block size", index_size, prop_delim, kv_delim);
AppendProperty(
result, "filter block size", filter_size, prop_delim, kv_delim
);
AppendProperty(
result,
"(estimated) table size",
data_size + index_size + filter_size,
prop_delim,
kv_delim
);
AppendProperty(result, "filter block size", filter_size, prop_delim,
kv_delim);
AppendProperty(result, "(estimated) table size",
data_size + index_size + filter_size, prop_delim, kv_delim);
AppendProperty(
result,
"filter policy name",
result, "filter policy name",
filter_policy_name.empty() ? std::string("N/A") : filter_policy_name,
prop_delim,
kv_delim
);
prop_delim, kv_delim);
return result;
}

117
table/table_test.cc
View File

@@ -6,6 +6,9 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <inttypes.h>
#include <stdio.h>
#include <algorithm>
#include <map>
#include <string>
@@ -25,6 +28,7 @@
#include "rocksdb/slice_transform.h"
#include "rocksdb/memtablerep.h"
#include "table/block.h"
#include "table/meta_blocks.h"
#include "table/block_based_table_builder.h"
#include "table/block_based_table_factory.h"
#include "table/block_based_table_reader.h"
@@ -946,10 +950,7 @@ TEST(BlockBasedTableTest, BasicBlockBasedTableProperties) {
block_builder.Add(item.first, item.second);
}
Slice content = block_builder.Finish();
ASSERT_EQ(
content.size() + kBlockTrailerSize,
props.data_size
);
ASSERT_EQ(content.size() + kBlockTrailerSize, props.data_size);
}
TEST(BlockBasedTableTest, FilterPolicyNameProperties) {
@@ -958,9 +959,7 @@ TEST(BlockBasedTableTest, FilterPolicyNameProperties) {
std::vector<std::string> keys;
KVMap kvmap;
Options options;
std::unique_ptr<const FilterPolicy> filter_policy(
NewBloomFilterPolicy(10)
);
std::unique_ptr<const FilterPolicy> filter_policy(NewBloomFilterPolicy(10));
options.filter_policy = filter_policy.get();
c.Finish(options, GetPlainInternalComparator(options.comparator), &keys,
@@ -1032,10 +1031,8 @@ TEST(BlockBasedTableTest, NumBlockStat) {
KVMap kvmap;
c.Finish(options, GetPlainInternalComparator(options.comparator), &ks,
&kvmap);
ASSERT_EQ(
kvmap.size(),
c.table_reader()->GetTableProperties().num_data_blocks
);
ASSERT_EQ(kvmap.size(),
c.table_reader()->GetTableProperties().num_data_blocks);
}
class BlockCacheProperties {
@@ -1050,32 +1047,26 @@ class BlockCacheProperties {
}
// Check if the fetched props matches the expected ones.
void AssertEqual(
long index_block_cache_miss,
long index_block_cache_hit,
long data_block_cache_miss,
long data_block_cache_hit) const {
void AssertEqual(int64_t index_block_cache_miss,
int64_t index_block_cache_hit, int64_t data_block_cache_miss,
int64_t data_block_cache_hit) const {
ASSERT_EQ(index_block_cache_miss, this->index_block_cache_miss);
ASSERT_EQ(index_block_cache_hit, this->index_block_cache_hit);
ASSERT_EQ(data_block_cache_miss, this->data_block_cache_miss);
ASSERT_EQ(data_block_cache_hit, this->data_block_cache_hit);
ASSERT_EQ(
index_block_cache_miss + data_block_cache_miss,
this->block_cache_miss
);
ASSERT_EQ(
index_block_cache_hit + data_block_cache_hit,
this->block_cache_hit
);
ASSERT_EQ(index_block_cache_miss + data_block_cache_miss,
this->block_cache_miss);
ASSERT_EQ(index_block_cache_hit + data_block_cache_hit,
this->block_cache_hit);
}
private:
long block_cache_miss = 0;
long block_cache_hit = 0;
long index_block_cache_miss = 0;
long index_block_cache_hit = 0;
long data_block_cache_miss = 0;
long data_block_cache_hit = 0;
int64_t block_cache_miss = 0;
int64_t block_cache_hit = 0;
int64_t index_block_cache_miss = 0;
int64_t index_block_cache_hit = 0;
int64_t data_block_cache_miss = 0;
int64_t data_block_cache_hit = 0;
};
TEST(BlockBasedTableTest, BlockCacheTest) {
@@ -1105,12 +1096,8 @@ TEST(BlockBasedTableTest, BlockCacheTest) {
{
BlockCacheProperties props(options.statistics.get());
// index will be added to block cache.
props.AssertEqual(
1, // index block miss
0,
0,
0
);
props.AssertEqual(1, // index block miss
0, 0, 0);
}
// Only index block will be accessed
@@ -1120,24 +1107,16 @@ TEST(BlockBasedTableTest, BlockCacheTest) {
// NOTE: to help better highlight the "detla" of each ticker, I use
// <last_value> + <added_value> to indicate the increment of changed
// value; other numbers remain the same.
props.AssertEqual(
1,
0 + 1, // index block hit
0,
0
);
props.AssertEqual(1, 0 + 1, // index block hit
0, 0);
}
// Only data block will be accessed
{
iter->SeekToFirst();
BlockCacheProperties props(options.statistics.get());
props.AssertEqual(
1,
1,
0 + 1, // data block miss
0
);
props.AssertEqual(1, 1, 0 + 1, // data block miss
0);
}
// Data block will be in cache
@@ -1145,12 +1124,8 @@ TEST(BlockBasedTableTest, BlockCacheTest) {
iter.reset(c.NewIterator());
iter->SeekToFirst();
BlockCacheProperties props(options.statistics.get());
props.AssertEqual(
1,
1 + 1, // index block hit
1,
0 + 1 // data block hit
);
props.AssertEqual(1, 1 + 1, /* index block hit */
1, 0 + 1 /* data block hit */);
}
// release the iterator so that the block cache can reset correctly.
iter.reset();
@@ -1176,12 +1151,8 @@ TEST(BlockBasedTableTest, BlockCacheTest) {
c.Reopen(options);
{
BlockCacheProperties props(options.statistics.get());
props.AssertEqual(
1, // index block miss
0,
0,
0
);
props.AssertEqual(1, // index block miss
0, 0, 0);
}
@@ -1191,12 +1162,9 @@ TEST(BlockBasedTableTest, BlockCacheTest) {
// is only 1, index block will be purged after data block is inserted.
iter.reset(c.NewIterator());
BlockCacheProperties props(options.statistics.get());
props.AssertEqual(
1 + 1, // index block miss
0,
0, // data block miss
0
);
props.AssertEqual(1 + 1, // index block miss
0, 0, // data block miss
0);
}
{
@@ -1204,12 +1172,8 @@ TEST(BlockBasedTableTest, BlockCacheTest) {
// block's cache miss.
iter->SeekToFirst();
BlockCacheProperties props(options.statistics.get());
props.AssertEqual(
2,
0,
0 + 1, // data block miss
0
);
props.AssertEqual(2, 0, 0 + 1, // data block miss
0);
}
}
@@ -1316,7 +1280,6 @@ TEST(GeneralTableTest, ApproximateOffsetOfPlain) {
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k06"), 510000, 511000));
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k07"), 510000, 511000));
ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 610000, 612000));
}
static void DoCompressionTest(CompressionType comp) {
@@ -1360,11 +1323,9 @@ TEST(GeneralTableTest, ApproximateOffsetOfCompressed) {
valid++;
}
for(int i =0; i < valid; i++)
{
for (int i = 0; i < valid; i++) {
DoCompressionTest(compression_state[i]);
}
}
TEST(Harness, Randomized) {
@@ -1375,8 +1336,8 @@ TEST(Harness, Randomized) {
for (int num_entries = 0; num_entries < 2000;
num_entries += (num_entries < 50 ? 1 : 200)) {
if ((num_entries % 10) == 0) {
fprintf(stderr, "case %d of %d: num_entries = %d\n",
(i + 1), int(args.size()), num_entries);
fprintf(stderr, "case %d of %d: num_entries = %d\n", (i + 1),
static_cast<int>(args.size()), num_entries);
}
for (int e = 0; e < num_entries; e++) {
std::string v;