ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

ldb: file_size and write_buffer_size options.

Browse Source 
Summary:
This allows ldb to control the write_buffer_size (which reflects to L0 file size) and file_size (which reflects to L1 file size).  Since the target_file_size_ratio is 1 by default, all other levels will also have the same file size as L1.

As part of the diff, I also cleaned up some unused code and help messages.

Test Plan: ./ldb load --db=/data/users/zshao/test_leveldb --file_size=64000000 --write_buffer_size=32000000 --create_if_missing --input_hex --disable_wal

Reviewers: dhruba, sheki, emayanke

Reviewed By: dhruba

CC: leveldb

Differential Revision: https://reviews.facebook.net/D7569
This commit is contained in:
Zheng Shao
2012-12-20 22:56:58 -08:00
parent 1aae609b92
commit 70f0f50731
2 changed files with 33 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
util/ldb_cmd.cc
View File

@@ -15,6 +15,8 @@ const char* LDBCommand::BLOOM_ARG = "--bloom_bits=";
const char* LDBCommand::COMPRESSION_TYPE_ARG = "--compression_type=";
const char* LDBCommand::BLOCK_SIZE = "--block_size=";
const char* LDBCommand::AUTO_COMPACTION = "--auto_compaction=";
const char* LDBCommand::WRITE_BUFFER_SIZE_ARG = "--write_buffer_size=";
const char* LDBCommand::FILE_SIZE_ARG = "--file_size=";
void LDBCommand::parse_open_args(std::vector<std::string>& args) {
std::vector<std::string> rest_of_args;
@@ -23,7 +25,9 @@ void LDBCommand::parse_open_args(std::vector<std::string>& args) {
if (arg.find(BLOOM_ARG) == 0
|| arg.find(COMPRESSION_TYPE_ARG) == 0
|| arg.find(BLOCK_SIZE) == 0
|| arg.find(AUTO_COMPACTION) == 0) {
|| arg.find(AUTO_COMPACTION) == 0
|| arg.find(WRITE_BUFFER_SIZE_ARG) == 0
|| arg.find(FILE_SIZE_ARG) == 0) {
open_args_.push_back(arg);
} else {
rest_of_args.push_back(arg);
@@ -81,6 +85,25 @@ leveldb::Options LDBCommand::PrepareOptionsForOpenDB() {
exec_state_ = LDBCommandExecuteResult::FAILED(
"Unknown compression level: " + comp);
}
} else if (arg.find(WRITE_BUFFER_SIZE_ARG) == 0) {
std::string write_buffer_str = arg.substr(strlen(WRITE_BUFFER_SIZE_ARG));
int write_buffer_size = atoi(write_buffer_str.c_str());
if (write_buffer_size == 0) {
exec_state_ = LDBCommandExecuteResult::FAILED(
std::string("Badly-formatted buffer size: ") + write_buffer_str);
}
opt.write_buffer_size = write_buffer_size;
} else if (arg.find(FILE_SIZE_ARG) == 0) {
std::string file_size_str = arg.substr(strlen(FILE_SIZE_ARG));
int file_size = atoi(file_size_str.c_str());
if (file_size == 0) {
exec_state_ = LDBCommandExecuteResult::FAILED(
std::string("Badly-formatted file size: ") + file_size_str);
}
opt.target_file_size_base = file_size;
} else {
exec_state_ = LDBCommandExecuteResult::FAILED(
"Unknown option: " + arg);
}
}
@@ -347,8 +370,6 @@ void DBDumper::DoCommand() {
const char* ReduceDBLevels::NEW_LEVLES_ARG = "--new_levels=";
const char* ReduceDBLevels::PRINT_OLD_LEVELS_ARG = "--print_old_levels";
const char* ReduceDBLevels::COMPRESSION_TYPE_ARG = "--compression=";
const char* ReduceDBLevels::FILE_SIZE_ARG = "--file_size=";
ReduceDBLevels::ReduceDBLevels(std::string& db_name,
std::vector<std::string>& args)
@@ -356,8 +377,6 @@ ReduceDBLevels::ReduceDBLevels(std::string& db_name,
old_levels_(1 << 16),
new_levels_(-1),
print_old_levels_(false) {
file_size_ = leveldb::Options().target_file_size_base;
compression_ = leveldb::Options().compression;
for (unsigned int i = 0; i < args.size(); i++) {
std::string& arg = args.at(i);
@@ -365,21 +384,6 @@ ReduceDBLevels::ReduceDBLevels(std::string& db_name,
new_levels_ = atoi(arg.substr(strlen(NEW_LEVLES_ARG)).c_str());
} else if (arg.find(PRINT_OLD_LEVELS_ARG) == 0) {
print_old_levels_ = true;
} else if (arg.find(COMPRESSION_TYPE_ARG) == 0) {
const char* type = arg.substr(strlen(COMPRESSION_TYPE_ARG)).c_str();
if (!strcasecmp(type, "none"))
compression_ = leveldb::kNoCompression;
else if (!strcasecmp(type, "snappy"))
compression_ = leveldb::kSnappyCompression;
else if (!strcasecmp(type, "zlib"))
compression_ = leveldb::kZlibCompression;
else if (!strcasecmp(type, "bzip2"))
compression_ = leveldb::kBZip2Compression;
else
exec_state_ = LDBCommandExecuteResult::FAILED(
"Invalid compression arg : " + arg);
} else if (arg.find(FILE_SIZE_ARG) == 0) {
file_size_ = atoi(arg.substr(strlen(FILE_SIZE_ARG)).c_str());
} else {
exec_state_ = LDBCommandExecuteResult::FAILED(
"Unknown argument." + arg);

16
util/ldb_cmd.h
View File

@@ -123,11 +123,15 @@ public:
ret.append(LDBCommand::BLOOM_ARG);
ret.append("<int,e.g.:14>] [");
ret.append(LDBCommand::COMPRESSION_TYPE_ARG);
ret.append("<no|snappy|zlib|bzip2> ");
ret.append("<no|snappy|zlib|bzip2>] [");
ret.append(LDBCommand::BLOCK_SIZE);
ret.append("=<block_size_in_bytes> ");
ret.append("<block_size_in_bytes>] [");
ret.append(LDBCommand::AUTO_COMPACTION);
ret.append("=<true|false>]");
ret.append("<true|false>] [");
ret.append(LDBCommand::WRITE_BUFFER_SIZE_ARG);
ret.append("<int,e.g.:4194304>] [");
ret.append(LDBCommand::FILE_SIZE_ARG);
ret.append("<int,e.g.:2097152>] ");
}
/* Run the command, and return the execute result. */
@@ -209,6 +213,8 @@ private:
static const char* COMPRESSION_TYPE_ARG;
static const char* BLOCK_SIZE;
static const char* AUTO_COMPACTION;
static const char* WRITE_BUFFER_SIZE_ARG;
static const char* FILE_SIZE_ARG;
std::vector<std::string> open_args_;
void parse_open_args(std::vector<std::string>& args);
};
@@ -296,13 +302,9 @@ private:
int old_levels_;
int new_levels_;
bool print_old_levels_;
int file_size_;
enum leveldb::CompressionType compression_;
static const char* NEW_LEVLES_ARG;
static const char* PRINT_OLD_LEVELS_ARG;
static const char* COMPRESSION_TYPE_ARG;
static const char* FILE_SIZE_ARG;
Status GetOldNumOfLevels(leveldb::Options& opt, int* levels);
};