mirror of
https://github.com/XRPLF/rippled.git
synced 2025-12-06 17:27:55 +00:00
Improve the "table stats"
Summary:
The primary motivation of the changes is to make it easier to figure out the inside of the tables.
* rename "table stats" to "table properties" since now we have more than "integers" to store in the property block.
* Add filter block size to the basic table properties.
* Whenever a table is built, we'll log the table properties (the sample output is in Test Plan).
* Make an api to expose deleted keys.
Test Plan:
Passed all existing test. and the sample output of table stats:
==================================================================
Basic Properties
------------------------------------------------------------------
# data blocks: 1
# entries: 1
raw key size: 9
raw average key size: 9
raw value size: 9
raw average value size: 0
data block size: 25
index block size: 27
filter block size: 18
(estimated) table size: 70
filter policy: rocksdb.BuiltinBloomFilter
==================================================================
User collected properties: InternalKeyPropertiesCollector
------------------------------------------------------------------
kDeletedKeys: 1
==================================================================
Reviewers: dhruba, haobo
Reviewed By: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14187
This commit is contained in:
@@ -10,7 +10,9 @@
|
||||
#include "table/block_based_table_builder.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <inttypes.h>
|
||||
#include <map>
|
||||
#include <stdio.h>
|
||||
|
||||
#include "rocksdb/flush_block_policy.h"
|
||||
#include "rocksdb/cache.h"
|
||||
@@ -45,13 +47,13 @@ struct BytewiseLessThan {
|
||||
// before writng to store.
|
||||
typedef std::map<std::string, std::string, BytewiseLessThan> BytewiseSortedMap;
|
||||
|
||||
void AddStats(BytewiseSortedMap& stats, std::string name, uint64_t val) {
|
||||
assert(stats.find(name) == stats.end());
|
||||
void AddProperties(BytewiseSortedMap& props, std::string name, uint64_t val) {
|
||||
assert(props.find(name) == props.end());
|
||||
|
||||
std::string dst;
|
||||
PutVarint64(&dst, val);
|
||||
|
||||
stats.insert(
|
||||
props.insert(
|
||||
std::make_pair(name, dst)
|
||||
);
|
||||
}
|
||||
@@ -63,12 +65,12 @@ static bool GoodCompressionRatio(size_t compressed_size, size_t raw_size) {
|
||||
|
||||
// Were we encounter any error occurs during user-defined statistics collection,
|
||||
// we'll write the warning message to info log.
|
||||
void LogStatsCollectionError(
|
||||
void LogPropertiesCollectionError(
|
||||
Logger* info_log, const std::string& method, const std::string& name) {
|
||||
assert(method == "Add" || method == "Finish");
|
||||
|
||||
std::string msg =
|
||||
"[Warning] encountered error when calling TableStatsCollector::" +
|
||||
"[Warning] encountered error when calling TablePropertiesCollector::" +
|
||||
method + "() with collector name: " + name;
|
||||
Log(info_log, "%s", msg.c_str());
|
||||
}
|
||||
@@ -84,12 +86,7 @@ struct BlockBasedTableBuilder::Rep {
|
||||
BlockBuilder index_block;
|
||||
std::string last_key;
|
||||
CompressionType compression_type;
|
||||
|
||||
uint64_t num_entries = 0;
|
||||
uint64_t num_data_blocks = 0;
|
||||
uint64_t raw_key_size = 0;
|
||||
uint64_t raw_value_size = 0;
|
||||
uint64_t data_size = 0;
|
||||
TableProperties props;
|
||||
|
||||
bool closed = false; // Either Finish() or Abandon() has been called.
|
||||
FilterBlockBuilder* filter_block;
|
||||
@@ -142,7 +139,7 @@ void BlockBasedTableBuilder::Add(const Slice& key, const Slice& value) {
|
||||
Rep* r = rep_;
|
||||
assert(!r->closed);
|
||||
if (!ok()) return;
|
||||
if (r->num_entries > 0) {
|
||||
if (r->props.num_entries > 0) {
|
||||
assert(r->options.comparator->Compare(key, Slice(r->last_key)) > 0);
|
||||
}
|
||||
|
||||
@@ -173,14 +170,14 @@ void BlockBasedTableBuilder::Add(const Slice& key, const Slice& value) {
|
||||
|
||||
r->last_key.assign(key.data(), key.size());
|
||||
r->data_block.Add(key, value);
|
||||
r->num_entries++;
|
||||
r->raw_key_size += key.size();
|
||||
r->raw_value_size += value.size();
|
||||
r->props.num_entries++;
|
||||
r->props.raw_key_size += key.size();
|
||||
r->props.raw_value_size += value.size();
|
||||
|
||||
for (auto collector : r->options.table_stats_collectors) {
|
||||
for (auto collector : r->options.table_properties_collectors) {
|
||||
Status s = collector->Add(key, value);
|
||||
if (!s.ok()) {
|
||||
LogStatsCollectionError(
|
||||
LogPropertiesCollectionError(
|
||||
r->options.info_log.get(),
|
||||
"Add", /* method */
|
||||
collector->Name()
|
||||
@@ -201,8 +198,8 @@ void BlockBasedTableBuilder::Flush() {
|
||||
if (r->filter_block != nullptr) {
|
||||
r->filter_block->StartBlock(r->offset);
|
||||
}
|
||||
r->data_size = r->offset;
|
||||
++r->num_data_blocks;
|
||||
r->props.data_size = r->offset;
|
||||
++r->props.num_data_blocks;
|
||||
}
|
||||
|
||||
void BlockBasedTableBuilder::WriteBlock(BlockBuilder* block,
|
||||
@@ -358,11 +355,12 @@ Status BlockBasedTableBuilder::Finish() {
|
||||
|
||||
// Write filter block
|
||||
if (ok() && r->filter_block != nullptr) {
|
||||
WriteRawBlock(r->filter_block->Finish(), kNoCompression,
|
||||
&filter_block_handle);
|
||||
auto filter_contents = r->filter_block->Finish();
|
||||
r->props.filter_size = filter_contents.size();
|
||||
WriteRawBlock(filter_contents, kNoCompression, &filter_block_handle);
|
||||
}
|
||||
|
||||
// To make sure stats block is able to keep the accurate size of index
|
||||
// To make sure properties block is able to keep the accurate size of index
|
||||
// block, we will finish writing all index entries here and flush them
|
||||
// to storage after metaindex block is written.
|
||||
if (ok() && !empty_data_block) {
|
||||
@@ -375,7 +373,7 @@ Status BlockBasedTableBuilder::Finish() {
|
||||
|
||||
// Write meta blocks and metaindex block with the following order.
|
||||
// 1. [meta block: filter]
|
||||
// 2. [meta block: stats]
|
||||
// 2. [meta block: properties]
|
||||
// 3. [metaindex block]
|
||||
if (ok()) {
|
||||
// We use `BytewiseComparator` as the comparator for meta block.
|
||||
@@ -400,67 +398,91 @@ Status BlockBasedTableBuilder::Finish() {
|
||||
);
|
||||
}
|
||||
|
||||
// Write stats block.
|
||||
// Write properties block.
|
||||
{
|
||||
BlockBuilder stats_block(
|
||||
BlockBuilder properties_block(
|
||||
r->options.block_restart_interval,
|
||||
BytewiseComparator()
|
||||
);
|
||||
|
||||
BytewiseSortedMap stats;
|
||||
BytewiseSortedMap properties;
|
||||
|
||||
// Add basic stats
|
||||
AddStats(stats, BlockBasedTableStatsNames::kRawKeySize, r->raw_key_size);
|
||||
AddStats(stats, BlockBasedTableStatsNames::kRawValueSize,
|
||||
r->raw_value_size);
|
||||
AddStats(stats, BlockBasedTableStatsNames::kDataSize, r->data_size);
|
||||
AddStats(
|
||||
stats,
|
||||
BlockBasedTableStatsNames::kIndexSize,
|
||||
r->index_block.CurrentSizeEstimate() + kBlockTrailerSize
|
||||
// Add basic properties
|
||||
AddProperties(
|
||||
properties,
|
||||
BlockBasedTablePropertiesNames::kRawKeySize,
|
||||
r->props.raw_key_size
|
||||
);
|
||||
AddStats(stats, BlockBasedTableStatsNames::kNumEntries, r->num_entries);
|
||||
AddStats(stats, BlockBasedTableStatsNames::kNumDataBlocks,
|
||||
r->num_data_blocks);
|
||||
AddProperties(
|
||||
properties,
|
||||
BlockBasedTablePropertiesNames::kRawValueSize,
|
||||
r->props.raw_value_size
|
||||
);
|
||||
AddProperties(
|
||||
properties,
|
||||
BlockBasedTablePropertiesNames::kDataSize,
|
||||
r->props.data_size
|
||||
);
|
||||
r->props.index_size =
|
||||
r->index_block.CurrentSizeEstimate() + kBlockTrailerSize;
|
||||
AddProperties(
|
||||
properties,
|
||||
BlockBasedTablePropertiesNames::kIndexSize,
|
||||
r->props.index_size
|
||||
);
|
||||
AddProperties(
|
||||
properties,
|
||||
BlockBasedTablePropertiesNames::kNumEntries,
|
||||
r->props.num_entries
|
||||
);
|
||||
AddProperties(
|
||||
properties,
|
||||
BlockBasedTablePropertiesNames::kNumDataBlocks,
|
||||
r->props.num_data_blocks);
|
||||
if (r->filter_block != nullptr) {
|
||||
stats.insert(std::make_pair(
|
||||
BlockBasedTableStatsNames::kFilterPolicy,
|
||||
properties.insert({
|
||||
BlockBasedTablePropertiesNames::kFilterPolicy,
|
||||
r->options.filter_policy->Name()
|
||||
));
|
||||
});
|
||||
}
|
||||
AddProperties(
|
||||
properties,
|
||||
BlockBasedTablePropertiesNames::kFilterSize,
|
||||
r->props.filter_size
|
||||
);
|
||||
|
||||
for (auto collector : r->options.table_stats_collectors) {
|
||||
TableStats::UserCollectedStats user_collected_stats;
|
||||
for (auto collector : r->options.table_properties_collectors) {
|
||||
TableProperties::UserCollectedProperties user_collected_properties;
|
||||
Status s =
|
||||
collector->Finish(&user_collected_stats);
|
||||
collector->Finish(&user_collected_properties);
|
||||
|
||||
if (!s.ok()) {
|
||||
LogStatsCollectionError(
|
||||
LogPropertiesCollectionError(
|
||||
r->options.info_log.get(),
|
||||
"Finish", /* method */
|
||||
collector->Name()
|
||||
);
|
||||
} else {
|
||||
stats.insert(
|
||||
user_collected_stats.begin(),
|
||||
user_collected_stats.end()
|
||||
properties.insert(
|
||||
user_collected_properties.begin(),
|
||||
user_collected_properties.end()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto& stat : stats) {
|
||||
stats_block.Add(stat.first, stat.second);
|
||||
for (const auto& stat : properties) {
|
||||
properties_block.Add(stat.first, stat.second);
|
||||
}
|
||||
|
||||
BlockHandle stats_block_handle;
|
||||
WriteBlock(&stats_block, &stats_block_handle);
|
||||
BlockHandle properties_block_handle;
|
||||
WriteBlock(&properties_block, &properties_block_handle);
|
||||
|
||||
std::string handle_encoding;
|
||||
stats_block_handle.EncodeTo(&handle_encoding);
|
||||
properties_block_handle.EncodeTo(&handle_encoding);
|
||||
meta_block_handles.insert(
|
||||
std::make_pair(BlockBasedTable::kStatsBlock, handle_encoding)
|
||||
{ BlockBasedTable::kPropertiesBlock, handle_encoding }
|
||||
);
|
||||
} // end of stats block writing
|
||||
} // end of properties block writing
|
||||
|
||||
for (const auto& metablock : meta_block_handles) {
|
||||
meta_index_block.Add(metablock.first, metablock.second);
|
||||
@@ -486,6 +508,31 @@ Status BlockBasedTableBuilder::Finish() {
|
||||
r->offset += footer_encoding.size();
|
||||
}
|
||||
}
|
||||
|
||||
// Print out the table stats
|
||||
if (ok()) {
|
||||
// user collected properties
|
||||
std::string user_collected;
|
||||
user_collected.reserve(1024);
|
||||
for (auto collector : r->options.table_properties_collectors) {
|
||||
for (const auto& prop : collector->GetReadableProperties()) {
|
||||
user_collected.append(prop.first);
|
||||
user_collected.append("=");
|
||||
user_collected.append(prop.second);
|
||||
user_collected.append("; ");
|
||||
}
|
||||
}
|
||||
|
||||
Log(
|
||||
r->options.info_log,
|
||||
"Table was constructed:\n"
|
||||
" basic properties: %s\n"
|
||||
" user collected properties: %s",
|
||||
r->props.ToString().c_str(),
|
||||
user_collected.c_str()
|
||||
);
|
||||
}
|
||||
|
||||
return r->status;
|
||||
}
|
||||
|
||||
@@ -496,7 +543,7 @@ void BlockBasedTableBuilder::Abandon() {
|
||||
}
|
||||
|
||||
uint64_t BlockBasedTableBuilder::NumEntries() const {
|
||||
return rep_->num_entries;
|
||||
return rep_->props.num_entries;
|
||||
}
|
||||
|
||||
uint64_t BlockBasedTableBuilder::FileSize() const {
|
||||
|
||||
@@ -58,7 +58,7 @@ struct BlockBasedTable::Rep {
|
||||
unique_ptr<Block> index_block;
|
||||
unique_ptr<FilterBlockReader> filter;
|
||||
|
||||
TableStats table_stats;
|
||||
TableProperties table_properties;
|
||||
};
|
||||
|
||||
BlockBasedTable::~BlockBasedTable() {
|
||||
@@ -261,18 +261,18 @@ Status BlockBasedTable::Open(const Options& options,
|
||||
std::unique_ptr<Iterator> meta_iter;
|
||||
s = ReadMetaBlock(rep, &meta, &meta_iter);
|
||||
|
||||
// Read the stats
|
||||
meta_iter->Seek(kStatsBlock);
|
||||
if (meta_iter->Valid() && meta_iter->key() == Slice(kStatsBlock)) {
|
||||
// Read the properties
|
||||
meta_iter->Seek(kPropertiesBlock);
|
||||
if (meta_iter->Valid() && meta_iter->key() == Slice(kPropertiesBlock)) {
|
||||
s = meta_iter->status();
|
||||
if (s.ok()) {
|
||||
s = ReadStats(meta_iter->value(), rep, &rep->table_stats);
|
||||
s = ReadProperties(meta_iter->value(), rep, &rep->table_properties);
|
||||
}
|
||||
|
||||
if (!s.ok()) {
|
||||
auto err_msg =
|
||||
"[Warning] Encountered error while reading data from stats block " +
|
||||
s.ToString();
|
||||
"[Warning] Encountered error while reading data from properties "
|
||||
"block " + s.ToString();
|
||||
Log(rep->options.info_log, "%s", err_msg.c_str());
|
||||
}
|
||||
}
|
||||
@@ -349,8 +349,8 @@ void BlockBasedTable::SetupForCompaction() {
|
||||
compaction_optimized_ = true;
|
||||
}
|
||||
|
||||
TableStats& BlockBasedTable::GetTableStats() {
|
||||
return rep_->table_stats;
|
||||
TableProperties& BlockBasedTable::GetTableProperties() {
|
||||
return rep_->table_properties;
|
||||
}
|
||||
|
||||
// Load the meta-block from the file. On success, return the loaded meta block
|
||||
@@ -372,8 +372,8 @@ Status BlockBasedTable::ReadMetaBlock(
|
||||
|
||||
if (!s.ok()) {
|
||||
auto err_msg =
|
||||
"[Warning] Encountered error while reading data from stats block " +
|
||||
s.ToString();
|
||||
"[Warning] Encountered error while reading data from properties"
|
||||
"block " + s.ToString();
|
||||
Log(rep->options.info_log, "%s", err_msg.c_str());
|
||||
}
|
||||
if (!s.ok()) {
|
||||
@@ -414,14 +414,14 @@ FilterBlockReader* BlockBasedTable::ReadFilter (
|
||||
rep->options, block.data, block.heap_allocated);
|
||||
}
|
||||
|
||||
Status BlockBasedTable::ReadStats(
|
||||
const Slice& handle_value, Rep* rep, TableStats* table_stats) {
|
||||
assert(table_stats);
|
||||
Status BlockBasedTable::ReadProperties(
|
||||
const Slice& handle_value, Rep* rep, TableProperties* table_properties) {
|
||||
assert(table_properties);
|
||||
|
||||
Slice v = handle_value;
|
||||
BlockHandle handle;
|
||||
if (!handle.DecodeFrom(&v).ok()) {
|
||||
return Status::InvalidArgument("Failed to decode stats block handle");
|
||||
return Status::InvalidArgument("Failed to decode properties block handle");
|
||||
}
|
||||
|
||||
BlockContents block_contents;
|
||||
@@ -438,20 +438,27 @@ Status BlockBasedTable::ReadStats(
|
||||
return s;
|
||||
}
|
||||
|
||||
Block stats_block(block_contents);
|
||||
Block properties_block(block_contents);
|
||||
std::unique_ptr<Iterator> iter(
|
||||
stats_block.NewIterator(BytewiseComparator())
|
||||
properties_block.NewIterator(BytewiseComparator())
|
||||
);
|
||||
|
||||
// All pre-defined stats of type uint64_t
|
||||
std::unordered_map<std::string, uint64_t*> predefined_uint64_stats = {
|
||||
{ BlockBasedTableStatsNames::kDataSize, &table_stats->data_size },
|
||||
{ BlockBasedTableStatsNames::kIndexSize, &table_stats->index_size },
|
||||
{ BlockBasedTableStatsNames::kRawKeySize, &table_stats->raw_key_size },
|
||||
{ BlockBasedTableStatsNames::kRawValueSize, &table_stats->raw_value_size },
|
||||
{ BlockBasedTableStatsNames::kNumDataBlocks,
|
||||
&table_stats->num_data_blocks },
|
||||
{ BlockBasedTableStatsNames::kNumEntries, &table_stats->num_entries },
|
||||
// All pre-defined properties of type uint64_t
|
||||
std::unordered_map<std::string, uint64_t*> predefined_uint64_properties = {
|
||||
{ BlockBasedTablePropertiesNames::kDataSize,
|
||||
&table_properties->data_size },
|
||||
{ BlockBasedTablePropertiesNames::kIndexSize,
|
||||
&table_properties->index_size },
|
||||
{ BlockBasedTablePropertiesNames::kFilterSize,
|
||||
&table_properties->filter_size },
|
||||
{ BlockBasedTablePropertiesNames::kRawKeySize,
|
||||
&table_properties->raw_key_size },
|
||||
{ BlockBasedTablePropertiesNames::kRawValueSize,
|
||||
&table_properties->raw_value_size },
|
||||
{ BlockBasedTablePropertiesNames::kNumDataBlocks,
|
||||
&table_properties->num_data_blocks },
|
||||
{ BlockBasedTablePropertiesNames::kNumEntries,
|
||||
&table_properties->num_entries },
|
||||
};
|
||||
|
||||
std::string last_key;
|
||||
@@ -462,7 +469,7 @@ Status BlockBasedTable::ReadStats(
|
||||
}
|
||||
|
||||
auto key = iter->key().ToString();
|
||||
// stats block is strictly sorted with no duplicate key.
|
||||
// properties block is strictly sorted with no duplicate key.
|
||||
assert(
|
||||
last_key.empty() ||
|
||||
BytewiseComparator()->Compare(key, last_key) > 0
|
||||
@@ -470,25 +477,25 @@ Status BlockBasedTable::ReadStats(
|
||||
last_key = key;
|
||||
|
||||
auto raw_val = iter->value();
|
||||
auto pos = predefined_uint64_stats.find(key);
|
||||
auto pos = predefined_uint64_properties.find(key);
|
||||
|
||||
if (pos != predefined_uint64_stats.end()) {
|
||||
// handle predefined rocksdb stats
|
||||
if (pos != predefined_uint64_properties.end()) {
|
||||
// handle predefined rocksdb properties
|
||||
uint64_t val;
|
||||
if (!GetVarint64(&raw_val, &val)) {
|
||||
// skip malformed value
|
||||
auto error_msg =
|
||||
"[Warning] detect malformed value in stats meta-block:"
|
||||
"[Warning] detect malformed value in properties meta-block:"
|
||||
"\tkey: " + key + "\tval: " + raw_val.ToString();
|
||||
Log(rep->options.info_log, "%s", error_msg.c_str());
|
||||
continue;
|
||||
}
|
||||
*(pos->second) = val;
|
||||
} else if (key == BlockBasedTableStatsNames::kFilterPolicy) {
|
||||
table_stats->filter_policy_name = raw_val.ToString();
|
||||
} else if (key == BlockBasedTablePropertiesNames::kFilterPolicy) {
|
||||
table_properties->filter_policy_name = raw_val.ToString();
|
||||
} else {
|
||||
// handle user-collected
|
||||
table_stats->user_collected_stats.insert(
|
||||
table_properties->user_collected_properties.insert(
|
||||
std::make_pair(key, raw_val.ToString())
|
||||
);
|
||||
}
|
||||
@@ -1066,20 +1073,25 @@ uint64_t BlockBasedTable::ApproximateOffsetOf(const Slice& key) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const std::string BlockBasedTable::kFilterBlockPrefix = "filter.";
|
||||
const std::string BlockBasedTable::kStatsBlock = "rocksdb.stats";
|
||||
|
||||
const std::string BlockBasedTableStatsNames::kDataSize = "rocksdb.data.size";
|
||||
const std::string BlockBasedTableStatsNames::kIndexSize = "rocksdb.index.size";
|
||||
const std::string BlockBasedTableStatsNames::kRawKeySize =
|
||||
const std::string BlockBasedTable::kFilterBlockPrefix =
|
||||
"filter.";
|
||||
const std::string BlockBasedTable::kPropertiesBlock =
|
||||
"rocksdb.properties";
|
||||
const std::string BlockBasedTablePropertiesNames::kDataSize =
|
||||
"rocksdb.data.size";
|
||||
const std::string BlockBasedTablePropertiesNames::kIndexSize =
|
||||
"rocksdb.index.size";
|
||||
const std::string BlockBasedTablePropertiesNames::kFilterSize =
|
||||
"rocksdb.filter.size";
|
||||
const std::string BlockBasedTablePropertiesNames::kRawKeySize =
|
||||
"rocksdb.raw.key.size";
|
||||
const std::string BlockBasedTableStatsNames::kRawValueSize =
|
||||
const std::string BlockBasedTablePropertiesNames::kRawValueSize =
|
||||
"rocksdb.raw.value.size";
|
||||
const std::string BlockBasedTableStatsNames::kNumDataBlocks =
|
||||
const std::string BlockBasedTablePropertiesNames::kNumDataBlocks =
|
||||
"rocksdb.num.data.blocks";
|
||||
const std::string BlockBasedTableStatsNames::kNumEntries =
|
||||
const std::string BlockBasedTablePropertiesNames::kNumEntries =
|
||||
"rocksdb.num.entries";
|
||||
const std::string BlockBasedTableStatsNames::kFilterPolicy =
|
||||
const std::string BlockBasedTablePropertiesNames::kFilterPolicy =
|
||||
"rocksdb.filter.policy";
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/iterator.h"
|
||||
#include "rocksdb/statistics.h"
|
||||
#include "rocksdb/table_stats.h"
|
||||
#include "rocksdb/table_properties.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
@@ -38,7 +38,7 @@ using std::unique_ptr;
|
||||
class BlockBasedTable : public TableReader {
|
||||
public:
|
||||
static const std::string kFilterBlockPrefix;
|
||||
static const std::string kStatsBlock;
|
||||
static const std::string kPropertiesBlock;
|
||||
|
||||
// Attempt to open the table that is stored in bytes [0..file_size)
|
||||
// of "file", and read the metadata entries necessary to allow
|
||||
@@ -88,7 +88,7 @@ class BlockBasedTable : public TableReader {
|
||||
// posix_fadvise
|
||||
void SetupForCompaction() override;
|
||||
|
||||
TableStats& GetTableStats() override;
|
||||
TableProperties& GetTableProperties() override;
|
||||
|
||||
~BlockBasedTable();
|
||||
|
||||
@@ -142,7 +142,7 @@ class BlockBasedTable : public TableReader {
|
||||
|
||||
void ReadMeta(const Footer& footer);
|
||||
void ReadFilter(const Slice& filter_handle_value);
|
||||
static Status ReadStats(const Slice& handle_value, Rep* rep);
|
||||
static Status ReadProperties(const Slice& handle_value, Rep* rep);
|
||||
|
||||
// Read the meta block from sst.
|
||||
static Status ReadMetaBlock(
|
||||
@@ -156,9 +156,9 @@ class BlockBasedTable : public TableReader {
|
||||
Rep* rep,
|
||||
size_t* filter_size = nullptr);
|
||||
|
||||
// Read the table stats from stats block.
|
||||
static Status ReadStats(
|
||||
const Slice& handle_value, Rep* rep, TableStats* stats);
|
||||
// Read the table properties from properties block.
|
||||
static Status ReadProperties(
|
||||
const Slice& handle_value, Rep* rep, TableProperties* properties);
|
||||
|
||||
static void SetupCacheKeyPrefix(Rep* rep);
|
||||
|
||||
@@ -181,9 +181,10 @@ class BlockBasedTable : public TableReader {
|
||||
void operator=(const TableReader&) = delete;
|
||||
};
|
||||
|
||||
struct BlockBasedTableStatsNames {
|
||||
struct BlockBasedTablePropertiesNames {
|
||||
static const std::string kDataSize;
|
||||
static const std::string kIndexSize;
|
||||
static const std::string kFilterSize;
|
||||
static const std::string kRawKeySize;
|
||||
static const std::string kRawValueSize;
|
||||
static const std::string kNumDataBlocks;
|
||||
|
||||
@@ -799,7 +799,7 @@ class TableTest { };
|
||||
|
||||
// This test include all the basic checks except those for index size and block
|
||||
// size, which will be conducted in separated unit tests.
|
||||
TEST(TableTest, BasicTableStats) {
|
||||
TEST(TableTest, BasicTableProperties) {
|
||||
BlockBasedTableConstructor c(BytewiseComparator());
|
||||
|
||||
c.Add("a1", "val1");
|
||||
@@ -820,16 +820,16 @@ TEST(TableTest, BasicTableStats) {
|
||||
|
||||
c.Finish(options, &keys, &kvmap);
|
||||
|
||||
auto& stats = c.table_reader()->GetTableStats();
|
||||
ASSERT_EQ(kvmap.size(), stats.num_entries);
|
||||
auto& props = c.table_reader()->GetTableProperties();
|
||||
ASSERT_EQ(kvmap.size(), props.num_entries);
|
||||
|
||||
auto raw_key_size = kvmap.size() * 2ul;
|
||||
auto raw_value_size = kvmap.size() * 4ul;
|
||||
|
||||
ASSERT_EQ(raw_key_size, stats.raw_key_size);
|
||||
ASSERT_EQ(raw_value_size, stats.raw_value_size);
|
||||
ASSERT_EQ(1ul, stats.num_data_blocks);
|
||||
ASSERT_EQ("", stats.filter_policy_name); // no filter policy is used
|
||||
ASSERT_EQ(raw_key_size, props.raw_key_size);
|
||||
ASSERT_EQ(raw_value_size, props.raw_value_size);
|
||||
ASSERT_EQ(1ul, props.num_data_blocks);
|
||||
ASSERT_EQ("", props.filter_policy_name); // no filter policy is used
|
||||
|
||||
// Verify data size.
|
||||
BlockBuilder block_builder(options);
|
||||
@@ -839,11 +839,11 @@ TEST(TableTest, BasicTableStats) {
|
||||
Slice content = block_builder.Finish();
|
||||
ASSERT_EQ(
|
||||
content.size() + kBlockTrailerSize,
|
||||
stats.data_size
|
||||
props.data_size
|
||||
);
|
||||
}
|
||||
|
||||
TEST(TableTest, FilterPolicyNameStats) {
|
||||
TEST(TableTest, FilterPolicyNameProperties) {
|
||||
BlockBasedTableConstructor c(BytewiseComparator());
|
||||
c.Add("a1", "val1");
|
||||
std::vector<std::string> keys;
|
||||
@@ -855,8 +855,8 @@ TEST(TableTest, FilterPolicyNameStats) {
|
||||
options.filter_policy = filter_policy.get();
|
||||
|
||||
c.Finish(options, &keys, &kvmap);
|
||||
auto& stats = c.table_reader()->GetTableStats();
|
||||
ASSERT_EQ("rocksdb.BuiltinBloomFilter", stats.filter_policy_name);
|
||||
auto& props = c.table_reader()->GetTableProperties();
|
||||
ASSERT_EQ("rocksdb.BuiltinBloomFilter", props.filter_policy_name);
|
||||
}
|
||||
|
||||
static std::string RandomString(Random* rnd, int len) {
|
||||
@@ -897,7 +897,7 @@ TEST(TableTest, IndexSizeStat) {
|
||||
|
||||
c.Finish(options, &ks, &kvmap);
|
||||
auto index_size =
|
||||
c.table_reader()->GetTableStats().index_size;
|
||||
c.table_reader()->GetTableProperties().index_size;
|
||||
ASSERT_GT(index_size, last_index_size);
|
||||
last_index_size = index_size;
|
||||
}
|
||||
@@ -927,13 +927,13 @@ TEST(TableTest, NumBlockStat) {
|
||||
c.Finish(options, &ks, &kvmap);
|
||||
ASSERT_EQ(
|
||||
kvmap.size(),
|
||||
c.table_reader()->GetTableStats().num_data_blocks
|
||||
c.table_reader()->GetTableProperties().num_data_blocks
|
||||
);
|
||||
}
|
||||
|
||||
class BlockCacheStats {
|
||||
class BlockCacheProperties {
|
||||
public:
|
||||
explicit BlockCacheStats(std::shared_ptr<Statistics> statistics) {
|
||||
explicit BlockCacheProperties(std::shared_ptr<Statistics> statistics) {
|
||||
block_cache_miss =
|
||||
statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
|
||||
block_cache_hit =
|
||||
@@ -948,7 +948,7 @@ class BlockCacheStats {
|
||||
statistics.get()->getTickerCount(BLOCK_CACHE_DATA_HIT);
|
||||
}
|
||||
|
||||
// Check if the fetched stats matches the expected ones.
|
||||
// Check if the fetched props matches the expected ones.
|
||||
void AssertEqual(
|
||||
long index_block_cache_miss,
|
||||
long index_block_cache_hit,
|
||||
@@ -996,9 +996,9 @@ TEST(TableTest, BlockCacheTest) {
|
||||
|
||||
// At first, no block will be accessed.
|
||||
{
|
||||
BlockCacheStats stats(options.statistics);
|
||||
BlockCacheProperties props(options.statistics);
|
||||
// index will be added to block cache.
|
||||
stats.AssertEqual(
|
||||
props.AssertEqual(
|
||||
1, // index block miss
|
||||
0,
|
||||
0,
|
||||
@@ -1009,11 +1009,11 @@ TEST(TableTest, BlockCacheTest) {
|
||||
// Only index block will be accessed
|
||||
{
|
||||
iter.reset(c.NewIterator());
|
||||
BlockCacheStats stats(options.statistics);
|
||||
BlockCacheProperties props(options.statistics);
|
||||
// NOTE: to help better highlight the "detla" of each ticker, I use
|
||||
// <last_value> + <added_value> to indicate the increment of changed
|
||||
// value; other numbers remain the same.
|
||||
stats.AssertEqual(
|
||||
props.AssertEqual(
|
||||
1,
|
||||
0 + 1, // index block hit
|
||||
0,
|
||||
@@ -1024,8 +1024,8 @@ TEST(TableTest, BlockCacheTest) {
|
||||
// Only data block will be accessed
|
||||
{
|
||||
iter->SeekToFirst();
|
||||
BlockCacheStats stats(options.statistics);
|
||||
stats.AssertEqual(
|
||||
BlockCacheProperties props(options.statistics);
|
||||
props.AssertEqual(
|
||||
1,
|
||||
1,
|
||||
0 + 1, // data block miss
|
||||
@@ -1037,8 +1037,8 @@ TEST(TableTest, BlockCacheTest) {
|
||||
{
|
||||
iter.reset(c.NewIterator());
|
||||
iter->SeekToFirst();
|
||||
BlockCacheStats stats(options.statistics);
|
||||
stats.AssertEqual(
|
||||
BlockCacheProperties props(options.statistics);
|
||||
props.AssertEqual(
|
||||
1,
|
||||
1 + 1, // index block hit
|
||||
1,
|
||||
@@ -1050,16 +1050,16 @@ TEST(TableTest, BlockCacheTest) {
|
||||
|
||||
// -- PART 2: Open without block cache
|
||||
options.block_cache.reset();
|
||||
options.statistics = CreateDBStatistics(); // reset the stats
|
||||
options.statistics = CreateDBStatistics(); // reset the props
|
||||
c.Reopen(options);
|
||||
|
||||
{
|
||||
iter.reset(c.NewIterator());
|
||||
iter->SeekToFirst();
|
||||
ASSERT_EQ("key", iter->key().ToString());
|
||||
BlockCacheStats stats(options.statistics);
|
||||
BlockCacheProperties props(options.statistics);
|
||||
// Nothing is affected at all
|
||||
stats.AssertEqual(0, 0, 0, 0);
|
||||
props.AssertEqual(0, 0, 0, 0);
|
||||
}
|
||||
|
||||
// -- PART 3: Open with very small block cache
|
||||
@@ -1068,8 +1068,8 @@ TEST(TableTest, BlockCacheTest) {
|
||||
options.block_cache = NewLRUCache(1);
|
||||
c.Reopen(options);
|
||||
{
|
||||
BlockCacheStats stats(options.statistics);
|
||||
stats.AssertEqual(
|
||||
BlockCacheProperties props(options.statistics);
|
||||
props.AssertEqual(
|
||||
1, // index block miss
|
||||
0,
|
||||
0,
|
||||
@@ -1083,8 +1083,8 @@ TEST(TableTest, BlockCacheTest) {
|
||||
// It first cache index block then data block. But since the cache size
|
||||
// is only 1, index block will be purged after data block is inserted.
|
||||
iter.reset(c.NewIterator());
|
||||
BlockCacheStats stats(options.statistics);
|
||||
stats.AssertEqual(
|
||||
BlockCacheProperties props(options.statistics);
|
||||
props.AssertEqual(
|
||||
1 + 1, // index block miss
|
||||
0,
|
||||
0, // data block miss
|
||||
@@ -1096,8 +1096,8 @@ TEST(TableTest, BlockCacheTest) {
|
||||
// SeekToFirst() accesses data block. With similar reason, we expect data
|
||||
// block's cache miss.
|
||||
iter->SeekToFirst();
|
||||
BlockCacheStats stats(options.statistics);
|
||||
stats.AssertEqual(
|
||||
BlockCacheProperties props(options.statistics);
|
||||
props.AssertEqual(
|
||||
2,
|
||||
0,
|
||||
0 + 1, // data block miss
|
||||
|
||||
Reference in New Issue
Block a user