ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into columnfamilies

Browse Source 
Conflicts:
	db/db_impl.cc
	db/db_impl_readonly.cc
	db/db_test.cc
	db/version_edit.cc
	db/version_edit.h
	db/version_set.cc
	db/version_set.h
	db/version_set_reduce_num_levels.cc
This commit is contained in:
Igor Canadi
2014-01-21 17:01:52 -08:00
parent 00065d0d5d 4e8321bfea
commit 23f6791c9e
49 changed files with 2550 additions and 2317 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
util/hash_skiplist_rep.cc
View File

@@ -31,17 +31,15 @@ class HashSkipListRep : public MemTableRep {
virtual ~HashSkipListRep();
virtual std::shared_ptr<MemTableRep::Iterator> GetIterator() override;
virtual MemTableRep::Iterator* GetIterator() override;
virtual std::shared_ptr<MemTableRep::Iterator> GetIterator(
const Slice& slice) override;
virtual MemTableRep::Iterator* GetIterator(const Slice& slice) override;
virtual std::shared_ptr<MemTableRep::Iterator> GetPrefixIterator(
const Slice& prefix) override;
virtual std::shared_ptr<MemTableRep::Iterator> GetDynamicPrefixIterator()
virtual MemTableRep::Iterator* GetPrefixIterator(const Slice& prefix)
override;
virtual MemTableRep::Iterator* GetDynamicPrefixIterator() override;
private:
friend class DynamicIterator;
typedef SkipList<const char*, MemTableRep::KeyComparator&> Bucket;
@@ -208,18 +206,15 @@ class HashSkipListRep : public MemTableRep {
virtual void SeekToLast() { }
private:
};
std::shared_ptr<EmptyIterator> empty_iterator_;
};
HashSkipListRep::HashSkipListRep(MemTableRep::KeyComparator& compare,
Arena* arena, const SliceTransform* transform, size_t bucket_size)
: bucket_size_(bucket_size),
transform_(transform),
compare_(compare),
arena_(arena),
empty_iterator_(std::make_shared<EmptyIterator>()) {
Arena* arena, const SliceTransform* transform,
size_t bucket_size)
: bucket_size_(bucket_size),
transform_(transform),
compare_(compare),
arena_(arena) {
buckets_ = new port::AtomicPointer[bucket_size];
for (size_t i = 0; i < bucket_size_; ++i) {
@@ -263,7 +258,7 @@ size_t HashSkipListRep::ApproximateMemoryUsage() {
return sizeof(buckets_);
}
std::shared_ptr<MemTableRep::Iterator> HashSkipListRep::GetIterator() {
MemTableRep::Iterator* HashSkipListRep::GetIterator() {
auto list = new Bucket(compare_, arena_);
for (size_t i = 0; i < bucket_size_; ++i) {
auto bucket = GetBucket(i);
@@ -274,35 +269,30 @@ std::shared_ptr<MemTableRep::Iterator> HashSkipListRep::GetIterator() {
}
}
}
return std::make_shared<Iterator>(list);
return new Iterator(list);
}
std::shared_ptr<MemTableRep::Iterator> HashSkipListRep::GetPrefixIterator(
const Slice& prefix) {
MemTableRep::Iterator* HashSkipListRep::GetPrefixIterator(const Slice& prefix) {
auto bucket = GetBucket(prefix);
if (bucket == nullptr) {
return empty_iterator_;
return new EmptyIterator();
}
return std::make_shared<Iterator>(bucket, false);
return new Iterator(bucket, false);
}
std::shared_ptr<MemTableRep::Iterator> HashSkipListRep::GetIterator(
const Slice& slice) {
MemTableRep::Iterator* HashSkipListRep::GetIterator(const Slice& slice) {
return GetPrefixIterator(transform_->Transform(slice));
}
std::shared_ptr<MemTableRep::Iterator>
HashSkipListRep::GetDynamicPrefixIterator() {
return std::make_shared<DynamicIterator>(*this);
MemTableRep::Iterator* HashSkipListRep::GetDynamicPrefixIterator() {
return new DynamicIterator(*this);
}
} // anon namespace
std::shared_ptr<MemTableRep>
HashSkipListRepFactory::CreateMemTableRep(MemTableRep::KeyComparator &compare,
Arena *arena) {
return std::make_shared<HashSkipListRep>(compare, arena, transform_,
bucket_count_);
MemTableRep* HashSkipListRepFactory::CreateMemTableRep(
MemTableRep::KeyComparator& compare, Arena* arena) {
return new HashSkipListRep(compare, arena, transform_, bucket_count_);
}
MemTableRepFactory* NewHashSkipListRepFactory(

4
util/hash_skiplist_rep.h
View File

@@ -21,8 +21,8 @@ class HashSkipListRepFactory : public MemTableRepFactory {
virtual ~HashSkipListRepFactory() { delete transform_; }
virtual std::shared_ptr<MemTableRep> CreateMemTableRep(
MemTableRep::KeyComparator& compare, Arena* arena) override;
virtual MemTableRep* CreateMemTableRep(MemTableRep::KeyComparator& compare,
Arena* arena) override;
virtual const char* Name() const override {
return "HashSkipListRepFactory";

66
util/histogram.cc
View File

@@ -16,27 +16,38 @@
namespace rocksdb {
HistogramBucketMapper::HistogramBucketMapper() :
// Add newer bucket index here.
// Should be alwyas added in sorted order.
bucketValues_({
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20, 25, 30, 35, 40, 45,
50, 60, 70, 80, 90, 100, 120, 140, 160, 180, 200, 250, 300, 350, 400, 450,
500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000, 2500, 3000,
3500, 4000, 4500, 5000, 6000, 7000, 8000, 9000, 10000, 12000, 14000,
16000, 18000, 20000, 25000, 30000, 35000, 40000, 45000, 50000, 60000,
70000, 80000, 90000, 100000, 120000, 140000, 160000, 180000, 200000,
250000, 300000, 350000, 400000, 450000, 500000, 600000, 700000, 800000,
900000, 1000000, 1200000, 1400000, 1600000, 1800000, 2000000, 2500000,
3000000, 3500000, 4000000, 4500000, 5000000, 6000000, 7000000, 8000000,
9000000, 10000000, 12000000, 14000000, 16000000, 18000000, 20000000,
25000000, 30000000, 35000000, 40000000, 45000000, 50000000, 60000000,
70000000, 80000000, 90000000, 100000000, 120000000, 140000000, 160000000,
180000000, 200000000, 250000000, 300000000, 350000000, 400000000,
450000000, 500000000, 600000000, 700000000, 800000000, 900000000,
1000000000}),
maxBucketValue_(bucketValues_.back()),
minBucketValue_(bucketValues_.front()) {
HistogramBucketMapper::HistogramBucketMapper()
:
// Add newer bucket index here.
// Should be alwyas added in sorted order.
// If you change this, you also need to change
// size of array buckets_ in HistogramImpl
bucketValues_(
{1, 2, 3, 4, 5, 6,
7, 8, 9, 10, 12, 14,
16, 18, 20, 25, 30, 35,
40, 45, 50, 60, 70, 80,
90, 100, 120, 140, 160, 180,
200, 250, 300, 350, 400, 450,
500, 600, 700, 800, 900, 1000,
1200, 1400, 1600, 1800, 2000, 2500,
3000, 3500, 4000, 4500, 5000, 6000,
7000, 8000, 9000, 10000, 12000, 14000,
16000, 18000, 20000, 25000, 30000, 35000,
40000, 45000, 50000, 60000, 70000, 80000,
90000, 100000, 120000, 140000, 160000, 180000,
200000, 250000, 300000, 350000, 400000, 450000,
500000, 600000, 700000, 800000, 900000, 1000000,
1200000, 1400000, 1600000, 1800000, 2000000, 2500000,
3000000, 3500000, 4000000, 4500000, 5000000, 6000000,
7000000, 8000000, 9000000, 10000000, 12000000, 14000000,
16000000, 18000000, 20000000, 25000000, 30000000, 35000000,
40000000, 45000000, 50000000, 60000000, 70000000, 80000000,
90000000, 100000000, 120000000, 140000000, 160000000, 180000000,
200000000, 250000000, 300000000, 350000000, 400000000, 450000000,
500000000, 600000000, 700000000, 800000000, 900000000, 1000000000}),
maxBucketValue_(bucketValues_.back()),
minBucketValue_(bucketValues_.front()) {
for (size_t i =0; i < bucketValues_.size(); ++i) {
valueIndexMap_[bucketValues_[i]] = i;
}
@@ -62,24 +73,17 @@ namespace {
const HistogramBucketMapper bucketMapper;
}
HistogramImpl::HistogramImpl() :
min_(bucketMapper.LastValue()),
max_(0),
num_(0),
sum_(0),
sum_squares_(0),
buckets_(std::vector<uint64_t>(bucketMapper.BucketCount(), 0)) {}
void HistogramImpl::Clear() {
min_ = bucketMapper.LastValue();
max_ = 0;
num_ = 0;
sum_ = 0;
sum_squares_ = 0;
buckets_.resize(bucketMapper.BucketCount(), 0);
memset(buckets_, 0, sizeof buckets_);
}
bool HistogramImpl::Empty() { return sum_squares_ == 0; }
void HistogramImpl::Add(uint64_t value) {
const size_t index = bucketMapper.IndexForValue(value);
buckets_[index] += 1;

18
util/histogram.h
View File

@@ -52,9 +52,8 @@ class HistogramBucketMapper {
class HistogramImpl {
public:
HistogramImpl();
virtual ~HistogramImpl() {}
virtual void Clear();
virtual bool Empty();
virtual void Add(uint64_t value);
void Merge(const HistogramImpl& other);
@@ -67,13 +66,14 @@ class HistogramImpl {
virtual void Data(HistogramData * const data) const;
private:
double min_;
double max_;
double num_;
double sum_;
double sum_squares_;
std::vector<uint64_t> buckets_;
// To be able to use HistogramImpl as thread local variable, its constructor
// has to be static. That's why we're using manually values from BucketMapper
double min_ = 1000000000; // this is BucketMapper:LastValue()
double max_ = 0;
double num_ = 0;
double sum_ = 0;
double sum_squares_ = 0;
uint64_t buckets_[138] = {0}; // this is BucketMapper::BucketCount()
};
} // namespace rocksdb

2
util/ldb_cmd.cc
View File

@@ -1024,7 +1024,7 @@ Status ReduceDBLevelsCommand::GetOldNumOfLevels(Options& opt,
}
int max = -1;
for (int i = 0; i < versions.NumberLevels(); i++) {
if (versions.NumLevelFiles(i)) {
if (versions.current()->NumLevelFiles(i)) {
max = i;
}
}

75
util/manual_compaction_test.cc
View File

@@ -9,9 +9,13 @@
#include <cstdlib>
#include "rocksdb/db.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/slice.h"
#include "rocksdb/write_batch.h"
#include "util/testharness.h"
using namespace rocksdb;
namespace {
const int kNumKeys = 1100000;
@@ -26,12 +30,71 @@ std::string Key2(int i) {
return Key1(i) + "_xxx";
}
class ManualCompactionTest { };
class ManualCompactionTest {
public:
ManualCompactionTest() {
// Get rid of any state from an old run.
dbname_ = rocksdb::test::TmpDir() + "/rocksdb_cbug_test";
DestroyDB(dbname_, rocksdb::Options());
}
std::string dbname_;
};
class DestroyAllCompactionFilter : public CompactionFilter {
public:
DestroyAllCompactionFilter() {}
virtual bool Filter(int level,
const Slice& key,
const Slice& existing_value,
std::string* new_value,
bool* value_changed) const {
return existing_value.ToString() == "destroy";
}
virtual const char* Name() const {
return "DestroyAllCompactionFilter";
}
};
TEST(ManualCompactionTest, CompactTouchesAllKeys) {
for (int iter = 0; iter < 2; ++iter) {
DB* db;
Options options;
if (iter == 0) { // level compaction
options.num_levels = 3;
options.compaction_style = kCompactionStyleLevel;
} else { // universal compaction
options.compaction_style = kCompactionStyleUniversal;
}
options.create_if_missing = true;
options.compression = rocksdb::kNoCompression;
options.compaction_filter = new DestroyAllCompactionFilter();
ASSERT_OK(DB::Open(options, dbname_, &db));
db->Put(WriteOptions(), Slice("key1"), Slice("destroy"));
db->Put(WriteOptions(), Slice("key2"), Slice("destroy"));
db->Put(WriteOptions(), Slice("key3"), Slice("value3"));
db->Put(WriteOptions(), Slice("key4"), Slice("destroy"));
Slice key4("key4");
db->CompactRange(nullptr, &key4);
Iterator* itr = db->NewIterator(ReadOptions());
itr->SeekToFirst();
ASSERT_TRUE(itr->Valid());
ASSERT_EQ("key3", itr->key().ToString());
itr->Next();
ASSERT_TRUE(!itr->Valid());
delete itr;
delete options.compaction_filter;
delete db;
DestroyDB(dbname_, options);
}
}
TEST(ManualCompactionTest, Test) {
// Get rid of any state from an old run.
std::string dbpath = rocksdb::test::TmpDir() + "/rocksdb_cbug_test";
DestroyDB(dbpath, rocksdb::Options());
// Open database. Disable compression since it affects the creation
// of layers and the code below is trying to test against a very
@@ -40,7 +103,7 @@ TEST(ManualCompactionTest, Test) {
rocksdb::Options db_options;
db_options.create_if_missing = true;
db_options.compression = rocksdb::kNoCompression;
ASSERT_OK(rocksdb::DB::Open(db_options, dbpath, &db));
ASSERT_OK(rocksdb::DB::Open(db_options, dbname_, &db));
// create first key range
rocksdb::WriteBatch batch;
@@ -83,7 +146,7 @@ TEST(ManualCompactionTest, Test) {
// close database
delete db;
DestroyDB(dbpath, rocksdb::Options());
DestroyDB(dbname_, rocksdb::Options());
}
} // anonymous namespace

10
util/skiplistrep.cc
View File

@@ -90,15 +90,15 @@ public:
// Unhide default implementations of GetIterator
using MemTableRep::GetIterator;
virtual std::shared_ptr<MemTableRep::Iterator> GetIterator() override {
return std::make_shared<SkipListRep::Iterator>(&skip_list_);
virtual MemTableRep::Iterator* GetIterator() override {
return new SkipListRep::Iterator(&skip_list_);
}
};
}
std::shared_ptr<MemTableRep> SkipListFactory::CreateMemTableRep (
MemTableRep::KeyComparator& compare, Arena* arena) {
return std::shared_ptr<MemTableRep>(new SkipListRep(compare, arena));
MemTableRep* SkipListFactory::CreateMemTableRep(
MemTableRep::KeyComparator& compare, Arena* arena) {
return new SkipListRep(compare, arena);
}
} // namespace rocksdb

43
util/statistics.cc
View File

@@ -3,12 +3,48 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#include "util/statistics.h"
#include "rocksdb/statistics.h"
#include <cstdio>
namespace rocksdb {
std::shared_ptr<Statistics> CreateDBStatistics() {
return std::make_shared<StatisticsImpl>();
}
StatisticsImpl::StatisticsImpl() {}
StatisticsImpl::~StatisticsImpl() {}
long StatisticsImpl::getTickerCount(Tickers tickerType) {
assert(tickerType < TICKER_ENUM_MAX);
return tickers_[tickerType];
}
void StatisticsImpl::setTickerCount(Tickers tickerType, uint64_t count) {
assert(tickerType < TICKER_ENUM_MAX);
tickers_[tickerType] = count;
}
void StatisticsImpl::recordTick(Tickers tickerType, uint64_t count) {
assert(tickerType < TICKER_ENUM_MAX);
tickers_[tickerType] += count;
}
void StatisticsImpl::measureTime(Histograms histogramType, uint64_t value) {
assert(histogramType < HISTOGRAM_ENUM_MAX);
histograms_[histogramType].Add(value);
}
void StatisticsImpl::histogramData(Histograms histogramType,
HistogramData* const data) {
assert(histogramType < HISTOGRAM_ENUM_MAX);
histograms_[histogramType].Data(data);
}
namespace {
// a buffer size used for temp string buffers
const int kBufferSize = 200;
@@ -32,11 +68,8 @@ std::string HistogramToString (
return std::string(buffer);
};
std::string TickerToString (
Statistics* dbstats,
const Tickers& ticker,
const std::string& name) {
std::string TickerToString(Statistics* dbstats, const Tickers& ticker,
const std::string& name) {
char buffer[kBufferSize];
snprintf(buffer, kBufferSize, "%s COUNT : %ld\n",
name.c_str(), dbstats->getTickerCount(ticker));

53
util/statistics.h Normal file
View File

@@ -0,0 +1,53 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#pragma once
#include "rocksdb/statistics.h"
#include "util/histogram.h"
#include "util/mutexlock.h"
#define UNLIKELY(val) (__builtin_expect((val), 0))
namespace rocksdb {
class StatisticsImpl : public Statistics {
public:
StatisticsImpl();
virtual ~StatisticsImpl();
virtual long getTickerCount(Tickers tickerType);
virtual void setTickerCount(Tickers tickerType, uint64_t count);
virtual void recordTick(Tickers tickerType, uint64_t count);
virtual void measureTime(Histograms histogramType, uint64_t value);
virtual void histogramData(Histograms histogramType,
HistogramData* const data);
private:
std::atomic_uint_fast64_t tickers_[TICKER_ENUM_MAX];
HistogramImpl histograms_[HISTOGRAM_ENUM_MAX];
};
// Utility functions
inline void MeasureTime(Statistics* statistics, Histograms histogramType,
uint64_t value) {
if (statistics) {
statistics->measureTime(histogramType, value);
}
}
inline void RecordTick(Statistics* statistics, Tickers ticker,
uint64_t count = 1) {
if (statistics) {
statistics->recordTick(ticker, count);
}
}
inline void SetTickerCount(Statistics* statistics, Tickers ticker,
uint64_t count) {
if (statistics) {
statistics->setTickerCount(ticker, count);
}
}
}

32
util/statistics_imp.h
View File

@@ -1,32 +0,0 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#pragma once
#include "rocksdb/statistics.h"
namespace rocksdb {
// Utility functions
inline void RecordTick(Statistics* statistics,
Tickers ticker,
uint64_t count = 1) {
assert(HistogramsNameMap.size() == HISTOGRAM_ENUM_MAX);
assert(TickersNameMap.size() == TICKER_ENUM_MAX);
if (statistics) {
statistics->recordTick(ticker, count);
}
}
inline void SetTickerCount(Statistics* statistics,
Tickers ticker,
uint64_t count) {
assert(HistogramsNameMap.size() == HISTOGRAM_ENUM_MAX);
assert(TickersNameMap.size() == TICKER_ENUM_MAX);
if (statistics) {
statistics->setTickerCount(ticker, count);
}
}
}

8
util/stop_watch.h
View File

@@ -5,7 +5,7 @@
//
#pragma once
#include "rocksdb/env.h"
#include "util/statistics_imp.h"
#include "util/statistics.h"
namespace rocksdb {
// Auto-scoped.
@@ -28,11 +28,7 @@ class StopWatch {
return env_->NowMicros() - start_time_;
}
~StopWatch() {
if (statistics_) {
statistics_->measureTime(histogram_name_, ElapsedMicros());
}
}
~StopWatch() { MeasureTime(statistics_, histogram_name_, ElapsedMicros()); }
private:
Env* const env_;

14
util/vectorrep.cc
View File

@@ -88,7 +88,7 @@ class VectorRep : public MemTableRep {
using MemTableRep::GetIterator;
// Return an iterator over the keys in this representation.
virtual std::shared_ptr<MemTableRep::Iterator> GetIterator() override;
virtual MemTableRep::Iterator* GetIterator() override;
private:
friend class Iterator;
@@ -228,22 +228,22 @@ void VectorRep::Iterator::SeekToLast() {
}
}
std::shared_ptr<MemTableRep::Iterator> VectorRep::GetIterator() {
MemTableRep::Iterator* VectorRep::GetIterator() {
ReadLock l(&rwlock_);
// Do not sort here. The sorting would be done the first time
// a Seek is performed on the iterator.
if (immutable_) {
return std::make_shared<Iterator>(this, bucket_, compare_);
return new Iterator(this, bucket_, compare_);
} else {
std::shared_ptr<Bucket> tmp;
tmp.reset(new Bucket(*bucket_)); // make a copy
return std::make_shared<Iterator>(nullptr, tmp, compare_);
return new Iterator(nullptr, tmp, compare_);
}
}
} // anon namespace
std::shared_ptr<MemTableRep> VectorRepFactory::CreateMemTableRep(
MemTableRep::KeyComparator& compare, Arena* arena) {
return std::make_shared<VectorRep>(compare, arena, count_);
MemTableRep* VectorRepFactory::CreateMemTableRep(
MemTableRep::KeyComparator& compare, Arena* arena) {
return new VectorRep(compare, arena, count_);
}
} // namespace rocksdb