From 566f18e6ad0499310bca0c960e2c12e2bb9dd400 Mon Sep 17 00:00:00 2001 From: Kai Liu Date: Thu, 6 Mar 2014 17:30:46 -0800 Subject: [PATCH] More precise calculation of sub_index_size Summary: Previous we did rough estimation of subindex size, which in worst case may result in array reallocation. This patch aims to get the exact size and avoid any reallocation. Test Plan: make all check Reviewers: sdong, dhruba, haobo Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D16125 --- table/plain_table_reader.cc | 69 ++++++++++++++----------------------- table/plain_table_reader.h | 16 ++++----- 2 files changed, 33 insertions(+), 52 deletions(-) diff --git a/table/plain_table_reader.cc b/table/plain_table_reader.cc index 8b675af113..46886291e9 100644 --- a/table/plain_table_reader.cc +++ b/table/plain_table_reader.cc @@ -287,8 +287,7 @@ void PlainTableReader::AllocateIndexAndBloom(int num_prefixes) { size_t PlainTableReader::BucketizeIndexesAndFillBloom( IndexRecordList* record_list, std::vector* hash_to_offsets, - std::vector* bucket_count) { - size_t sub_index_size_needed = 0; + std::vector* entries_per_bucket) { bool first = true; uint32_t prev_hash = 0; size_t num_records = record_list->GetNumRecords(); @@ -306,36 +305,31 @@ size_t PlainTableReader::BucketizeIndexesAndFillBloom( IndexRecord* prev_bucket_head = (*hash_to_offsets)[bucket]; index_record->next = prev_bucket_head; (*hash_to_offsets)[bucket] = index_record; - auto& item_count = (*bucket_count)[bucket]; - if (item_count > 0) { - if (item_count == 1) { - sub_index_size_needed += kOffsetLen + 1; - } - if (item_count == 127) { - // Need more than one byte for length - sub_index_size_needed++; - } - sub_index_size_needed += kOffsetLen; - } - item_count++; + (*entries_per_bucket)[bucket]++; } - return sub_index_size_needed; + size_t sub_index_size = 0; + for (auto entry_count : *entries_per_bucket) { + if (entry_count <= 1) { + continue; + } + // Only buckets with more than 1 entry will have subindex. + sub_index_size += VarintLength(entry_count); + // total bytes needed to store these entries' in-file offsets. + sub_index_size += entry_count * kOffsetLen; + } + return sub_index_size; } void PlainTableReader::FillIndexes( - size_t sub_index_size_needed, + const size_t kSubIndexSize, const std::vector& hash_to_offsets, - const std::vector& bucket_count) { - Log(options_.info_log, "Reserving %zu bytes for sub index", - sub_index_size_needed); - // 8 bytes buffer for variable length size - size_t buffer_size = 8 * 8; - size_t buffer_used = 0; - sub_index_size_needed += buffer_size; - sub_index_.reset(new char[sub_index_size_needed]); + const std::vector& entries_per_bucket) { + Log(options_.info_log, "Reserving %zu bytes for plain table's sub_index", + kSubIndexSize); + sub_index_.reset(new char[kSubIndexSize]); size_t sub_index_offset = 0; for (int i = 0; i < index_size_; i++) { - uint32_t num_keys_for_bucket = bucket_count[i]; + uint32_t num_keys_for_bucket = entries_per_bucket[i]; switch (num_keys_for_bucket) { case 0: // No key for bucket @@ -351,21 +345,6 @@ void PlainTableReader::FillIndexes( char* prev_ptr = &sub_index_[sub_index_offset]; char* cur_ptr = EncodeVarint32(prev_ptr, num_keys_for_bucket); sub_index_offset += (cur_ptr - prev_ptr); - if (cur_ptr - prev_ptr > 2 - || (cur_ptr - prev_ptr == 2 && num_keys_for_bucket <= 127)) { - // Need to resize sub_index. Exponentially grow buffer. - buffer_used += cur_ptr - prev_ptr - 1; - if (buffer_used + 4 > buffer_size) { - Log(options_.info_log, "Recalculate suffix_map length to %zu", - sub_index_size_needed); - - sub_index_size_needed += buffer_size; - buffer_size *= 2; - char* new_sub_index = new char[sub_index_size_needed]; - memcpy(new_sub_index, sub_index_.get(), sub_index_offset); - sub_index_.reset(new_sub_index); - } - } char* sub_index_pos = &sub_index_[sub_index_offset]; IndexRecord* record = hash_to_offsets[i]; int j; @@ -375,12 +354,14 @@ void PlainTableReader::FillIndexes( } assert(j == -1 && record == nullptr); sub_index_offset += kOffsetLen * num_keys_for_bucket; + assert(sub_index_offset <= kSubIndexSize); break; } } + assert(sub_index_offset == kSubIndexSize); Log(options_.info_log, "hash table size: %d, suffix_map length %zu", - index_size_, sub_index_size_needed); + index_size_, kSubIndexSize); } Status PlainTableReader::PopulateIndex() { @@ -422,11 +403,11 @@ Status PlainTableReader::PopulateIndex() { // Bucketize all the index records to a temp data structure, in which for // each bucket, we generate a linked list of IndexRecord, in reversed order. std::vector hash_to_offsets(index_size_, nullptr); - std::vector bucket_count(index_size_, 0); + std::vector entries_per_bucket(index_size_, 0); size_t sub_index_size_needed = BucketizeIndexesAndFillBloom( - &record_list, &hash_to_offsets, &bucket_count); + &record_list, &hash_to_offsets, &entries_per_bucket); // From the temp data structure, populate indexes. - FillIndexes(sub_index_size_needed, hash_to_offsets, bucket_count); + FillIndexes(sub_index_size_needed, hash_to_offsets, entries_per_bucket); return Status::OK(); } diff --git a/table/plain_table_reader.h b/table/plain_table_reader.h index 16bbc8ba5c..a93b3dd35f 100644 --- a/table/plain_table_reader.h +++ b/table/plain_table_reader.h @@ -196,18 +196,18 @@ class PlainTableReader: public TableReader { // containing a linklist of IndexRecord hashed to the same bucket, in reverse // order. // of offsets for the hash, in reversed order. - // bucket_count is sized of index_size_. The value is how many index + // entries_per_bucket is sized of index_size_. The value is how many index // records are there in bucket_headers for the same bucket. - size_t BucketizeIndexesAndFillBloom(IndexRecordList* record_list, - std::vector* bucket_headers, - std::vector* bucket_count); + size_t BucketizeIndexesAndFillBloom( + IndexRecordList* record_list, std::vector* bucket_headers, + std::vector* entries_per_bucket); // Internal helper class to fill the indexes and bloom filters to internal - // data structures. bucket_headers and bucket_count are bucketized indexes - // and counts generated by BucketizeIndexesAndFillBloom(). - void FillIndexes(size_t sub_index_size_needed, + // data structures. bucket_headers and entries_per_bucket are bucketized + // indexes and counts generated by BucketizeIndexesAndFillBloom(). + void FillIndexes(const size_t kSubIndexSize, const std::vector& bucket_headers, - const std::vector& bucket_count); + const std::vector& entries_per_bucket); // Read a plain table key from the position `start`. The read content // will be written to `key` and the size of read bytes will be populated