ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2026-06-03 08:46:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update src/rocksdb2 to rocksdb-3.5.1:

Browse Source 
Merge commit 'c168d54495d7d7b84639514f6443ad99b89ce996' into develop
This commit is contained in:
Vinnie Falco
2014-10-27 11:36:32 -07:00
parent 2cce22052b c168d54495
commit 6fbce4c2f7
195 changed files with 4433 additions and 9442 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
src/rocksdb2/java/org/rocksdb/BlockBasedTableConfig.java
View File

@@ -14,14 +14,11 @@ public class BlockBasedTableConfig extends TableFormatConfig {
public BlockBasedTableConfig() {
noBlockCache_ = false;
blockCacheSize_ = 8 * 1024 * 1024;
blockSize_ = 4 * 1024;
blockSizeDeviation_ = 10;
blockRestartInterval_ = 16;
blockSize_ = 4 * 1024;
blockSizeDeviation_ =10;
blockRestartInterval_ =16;
wholeKeyFiltering_ = true;
bitsPerKey_ = 10;
cacheIndexAndFilterBlocks_ = false;
hashIndexAllowCollision_ = true;
blockCacheCompressedSize_ = 0;
bitsPerKey_ = 0;
}
/**
@@ -74,8 +71,8 @@ public class BlockBasedTableConfig extends TableFormatConfig {
* number means use default settings."
* @return the reference to the current option.
*/
public BlockBasedTableConfig setCacheNumShardBits(int blockCacheNumShardBits) {
blockCacheNumShardBits_ = blockCacheNumShardBits;
public BlockBasedTableConfig setCacheNumShardBits(int numShardBits) {
numShardBits_ = numShardBits;
return this;
}
@@ -87,7 +84,7 @@ public class BlockBasedTableConfig extends TableFormatConfig {
* @return the number of shard bits used in the block cache.
*/
public int cacheNumShardBits() {
return blockCacheNumShardBits_;
return numShardBits_;
}
/**
@@ -189,135 +186,25 @@ public class BlockBasedTableConfig extends TableFormatConfig {
bitsPerKey_ = bitsPerKey;
return this;
}
/**
* Indicating if we'd put index/filter blocks to the block cache.
If not specified, each "table reader" object will pre-load index/filter
block during table initialization.
*
* @return if index and filter blocks should be put in block cache.
*/
public boolean cacheIndexAndFilterBlocks() {
return cacheIndexAndFilterBlocks_;
}
/**
* Indicating if we'd put index/filter blocks to the block cache.
If not specified, each "table reader" object will pre-load index/filter
block during table initialization.
*
* @param index and filter blocks should be put in block cache.
* @return the reference to the current config.
*/
public BlockBasedTableConfig setCacheIndexAndFilterBlocks(
boolean cacheIndexAndFilterBlocks) {
cacheIndexAndFilterBlocks_ = cacheIndexAndFilterBlocks;
return this;
}
/**
* Influence the behavior when kHashSearch is used.
if false, stores a precise prefix to block range mapping
if true, does not store prefix and allows prefix hash collision
(less memory consumption)
*
* @return if hash collisions should be allowed.
*/
public boolean hashIndexAllowCollision() {
return hashIndexAllowCollision_;
}
/**
* Influence the behavior when kHashSearch is used.
if false, stores a precise prefix to block range mapping
if true, does not store prefix and allows prefix hash collision
(less memory consumption)
*
* @param if hash collisions should be allowed.
* @return the reference to the current config.
*/
public BlockBasedTableConfig setHashIndexAllowCollision(
boolean hashIndexAllowCollision) {
hashIndexAllowCollision_ = hashIndexAllowCollision;
return this;
}
/**
* Size of compressed block cache. If 0, then block_cache_compressed is set
* to null.
*
* @return size of compressed block cache.
*/
public long blockCacheCompressedSize() {
return blockCacheCompressedSize_;
}
/**
* Size of compressed block cache. If 0, then block_cache_compressed is set
* to null.
*
* @param size of compressed block cache.
* @return the reference to the current config.
*/
public BlockBasedTableConfig setBlockCacheCompressedSize(
long blockCacheCompressedSize) {
blockCacheCompressedSize_ = blockCacheCompressedSize;
return this;
}
/**
* Controls the number of shards for the block compressed cache.
* This is applied only if blockCompressedCacheSize is set to non-negative.
*
* @return numShardBits the number of shard bits. The resulting
* number of shards would be 2 ^ numShardBits. Any negative
* number means use default settings.
*/
public int blockCacheCompressedNumShardBits() {
return blockCacheCompressedNumShardBits_;
}
/**
* Controls the number of shards for the block compressed cache.
* This is applied only if blockCompressedCacheSize is set to non-negative.
*
* @param numShardBits the number of shard bits. The resulting
* number of shards would be 2 ^ numShardBits. Any negative
* number means use default settings."
* @return the reference to the current option.
*/
public BlockBasedTableConfig setBlockCacheCompressedNumShardBits(
int blockCacheCompressedNumShardBits) {
blockCacheCompressedNumShardBits_ = blockCacheCompressedNumShardBits;
return this;
}
@Override protected long newTableFactoryHandle() {
return newTableFactoryHandle(noBlockCache_, blockCacheSize_,
blockCacheNumShardBits_, blockSize_, blockSizeDeviation_,
blockRestartInterval_, wholeKeyFiltering_, bitsPerKey_,
cacheIndexAndFilterBlocks_, hashIndexAllowCollision_,
blockCacheCompressedSize_, blockCacheCompressedNumShardBits_);
return newTableFactoryHandle(noBlockCache_, blockCacheSize_, numShardBits_,
blockSize_, blockSizeDeviation_, blockRestartInterval_,
wholeKeyFiltering_, bitsPerKey_);
}
private native long newTableFactoryHandle(
boolean noBlockCache, long blockCacheSize, int blockCacheNumShardBits,
boolean noBlockCache, long blockCacheSize, int numShardbits,
long blockSize, int blockSizeDeviation, int blockRestartInterval,
boolean wholeKeyFiltering, int bitsPerKey,
boolean cacheIndexAndFilterBlocks, boolean hashIndexAllowCollision,
long blockCacheCompressedSize, int blockCacheCompressedNumShardBits);
boolean wholeKeyFiltering, int bitsPerKey);
private boolean noBlockCache_;
private long blockCacheSize_;
private int blockCacheNumShardBits_;
private int numShardBits_;
private long shard;
private long blockSize_;
private int blockSizeDeviation_;
private int blockRestartInterval_;
private boolean wholeKeyFiltering_;
private int bitsPerKey_;
private boolean cacheIndexAndFilterBlocks_;
private boolean hashIndexAllowCollision_;
private long blockCacheCompressedSize_;
private int blockCacheCompressedNumShardBits_;
}

36
src/rocksdb2/java/org/rocksdb/GenericRateLimiterConfig.java
View File

@@ -1,36 +0,0 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb;
/**
* Config for rate limiter, which is used to control write rate of flush and
* compaction.
*/
public class GenericRateLimiterConfig extends RateLimiterConfig {
private static final long DEFAULT_REFILL_PERIOD_MICROS = (100 * 1000);
private static final int DEFAULT_FAIRNESS = 10;
public GenericRateLimiterConfig(long rateBytesPerSecond,
long refillPeriodMicros, int fairness) {
rateBytesPerSecond_ = rateBytesPerSecond;
refillPeriodMicros_ = refillPeriodMicros;
fairness_ = fairness;
}
public GenericRateLimiterConfig(long rateBytesPerSecond) {
this(rateBytesPerSecond, DEFAULT_REFILL_PERIOD_MICROS, DEFAULT_FAIRNESS);
}
@Override protected long newRateLimiterHandle() {
return newRateLimiterHandle(rateBytesPerSecond_, refillPeriodMicros_,
fairness_);
}
private native long newRateLimiterHandle(long rateBytesPerSecond,
long refillPeriodMicros, int fairness);
private final long rateBytesPerSecond_;
private final long refillPeriodMicros_;
private final int fairness_;
}

58
src/rocksdb2/java/org/rocksdb/NativeLibraryLoader.java
View File

@@ -1,58 +0,0 @@
package org.rocksdb;
import java.io.*;
/**
* This class is used to load the RocksDB shared library from within the jar.
* The shared library is extracted to a temp folder and loaded from there.
*/
public class NativeLibraryLoader {
private static String sharedLibraryName = "librocksdbjni.so";
private static String tempFilePrefix = "librocksdbjni";
private static String tempFileSuffix = ".so";
public static void loadLibraryFromJar(String tmpDir)
throws IOException {
File temp;
if(tmpDir == null || tmpDir.equals(""))
temp = File.createTempFile(tempFilePrefix, tempFileSuffix);
else
temp = new File(tmpDir + "/" + sharedLibraryName);
temp.deleteOnExit();
if (!temp.exists()) {
throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist.");
}
byte[] buffer = new byte[102400];
int readBytes;
InputStream is = ClassLoader.getSystemClassLoader().getResourceAsStream(sharedLibraryName);
if (is == null) {
throw new RuntimeException(sharedLibraryName + " was not found inside JAR.");
}
OutputStream os = null;
try {
os = new FileOutputStream(temp);
while ((readBytes = is.read(buffer)) != -1) {
os.write(buffer, 0, readBytes);
}
} finally {
if(os != null)
os.close();
if(is != null)
is.close();
}
System.load(temp.getAbsolutePath());
}
/**
* Private constructor to disallow instantiation
*/
private NativeLibraryLoader() {
}
}

102
src/rocksdb2/java/org/rocksdb/Options.java
View File

@@ -13,19 +13,8 @@ package org.rocksdb;
* native resources will be released as part of the process.
*/
public class Options extends RocksObject {
static {
RocksDB.loadLibrary();
}
static final long DEFAULT_CACHE_SIZE = 8 << 20;
static final int DEFAULT_NUM_SHARD_BITS = -1;
/**
* Builtin RocksDB comparators
*/
public enum BuiltinComparator {
BYTEWISE_COMPARATOR, REVERSE_BYTEWISE_COMPARATOR;
}
/**
* Construct options for opening a RocksDB.
*
@@ -86,21 +75,6 @@ public class Options extends RocksObject {
return createIfMissing(nativeHandle_);
}
/**
* Set BuiltinComparator to be used with RocksDB.
*
* Note: Comparator can be set once upon database creation.
*
* Default: BytewiseComparator.
* @param builtinComparator a BuiltinComparator type.
*/
public void setBuiltinComparator(BuiltinComparator builtinComparator) {
assert(isInitialized());
setBuiltinComparator(nativeHandle_, builtinComparator.ordinal());
}
private native void setBuiltinComparator(long handle, int builtinComparator);
/**
* Amount of data to build up in memory (backed by an unsorted log
* on disk) before converting to a sorted on-disk file.
@@ -331,6 +305,40 @@ public class Options extends RocksObject {
}
private native void setUseFsync(long handle, boolean useFsync);
/**
* The time interval in seconds between each two consecutive stats logs.
* This number controls how often a new scribe log about
* db deploy stats is written out.
* -1 indicates no logging at all.
*
* @return the time interval in seconds between each two consecutive
* stats logs.
*/
public int dbStatsLogInterval() {
assert(isInitialized());
return dbStatsLogInterval(nativeHandle_);
}
private native int dbStatsLogInterval(long handle);
/**
* The time interval in seconds between each two consecutive stats logs.
* This number controls how often a new scribe log about
* db deploy stats is written out.
* -1 indicates no logging at all.
* Default value is 1800 (half an hour).
*
* @param dbStatsLogInterval the time interval in seconds between each
* two consecutive stats logs.
* @return the reference to the current option.
*/
public Options setDbStatsLogInterval(int dbStatsLogInterval) {
assert(isInitialized());
setDbStatsLogInterval(nativeHandle_, dbStatsLogInterval);
return this;
}
private native void setDbStatsLogInterval(
long handle, int dbStatsLogInterval);
/**
* Returns the directory of info log.
*
@@ -1093,6 +1101,33 @@ public class Options extends RocksObject {
private native void setBytesPerSync(
long handle, long bytesPerSync);
/**
* Allow RocksDB to use thread local storage to optimize performance.
* Default: true
*
* @return true if thread-local storage is allowed
*/
public boolean allowThreadLocal() {
assert(isInitialized());
return allowThreadLocal(nativeHandle_);
}
private native boolean allowThreadLocal(long handle);
/**
* Allow RocksDB to use thread local storage to optimize performance.
* Default: true
*
* @param allowThreadLocal true if thread-local storage is allowed.
* @return the reference to the current option.
*/
public Options setAllowThreadLocal(boolean allowThreadLocal) {
assert(isInitialized());
setAllowThreadLocal(nativeHandle_, allowThreadLocal);
return this;
}
private native void setAllowThreadLocal(
long handle, boolean allowThreadLocal);
/**
* Set the config for mem-table.
*
@@ -1103,19 +1138,6 @@ public class Options extends RocksObject {
setMemTableFactory(nativeHandle_, config.newMemTableFactoryHandle());
return this;
}
/**
* Use to control write rate of flush and compaction. Flush has higher
* priority than compaction. Rate limiting is disabled if nullptr.
* Default: nullptr
*
* @param config rate limiter config.
* @return the instance of the current Options.
*/
public Options setRateLimiterConfig(RateLimiterConfig config) {
setRateLimiter(nativeHandle_, config.newRateLimiterHandle());
return this;
}
/**
* Returns the name of the current mem table representation.
@@ -2204,8 +2226,6 @@ public class Options extends RocksObject {
private native long statisticsPtr(long optHandle);
private native void setMemTableFactory(long handle, long factoryHandle);
private native void setRateLimiter(long handle,
long rateLimiterHandle);
private native String memTableFactoryName(long handle);
private native void setTableFactory(long handle, long factoryHandle);

20
src/rocksdb2/java/org/rocksdb/RateLimiterConfig.java
View File

@@ -1,20 +0,0 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb;
/**
* Config for rate limiter, which is used to control write rate of flush and
* compaction.
*/
public abstract class RateLimiterConfig {
/**
* This function should only be called by Options.setRateLimiter(),
* which will create a c++ shared-pointer to the c++ RateLimiter
* that is associated with the Java RateLimtierConifg.
*
* @see Options.setRateLimiter()
*/
abstract protected long newRateLimiterHandle();
}

41
src/rocksdb2/java/org/rocksdb/RocksDB.java
View File

@@ -11,7 +11,6 @@ import java.util.HashMap;
import java.io.Closeable;
import java.io.IOException;
import org.rocksdb.util.Environment;
import org.rocksdb.NativeLibraryLoader;
/**
* A RocksDB is a persistent ordered map from keys to values. It is safe for
@@ -24,19 +23,11 @@ public class RocksDB extends RocksObject {
private static final String[] compressionLibs_ = {
"snappy", "z", "bzip2", "lz4", "lz4hc"};
static {
RocksDB.loadLibrary();
}
/**
* Loads the necessary library files.
* Calling this method twice will have no effect.
* By default the method extracts the shared library for loading at
* java.io.tmpdir, however, you can override this temporary location by
* setting the environment variable ROCKSDB_SHAREDLIB_DIR.
*/
public static synchronized void loadLibrary() {
String tmpDir = System.getenv("ROCKSDB_SHAREDLIB_DIR");
// loading possibly necessary libraries.
for (String lib : compressionLibs_) {
try {
@@ -45,14 +36,8 @@ public class RocksDB extends RocksObject {
// since it may be optional, we ignore its loading failure here.
}
}
try
{
NativeLibraryLoader.loadLibraryFromJar(tmpDir);
}
catch (IOException e)
{
throw new RuntimeException("Unable to load the RocksDB shared library" + e);
}
// However, if any of them is required. We will see error here.
System.loadLibrary("rocksdbjni");
}
/**
@@ -324,26 +309,6 @@ public class RocksDB extends RocksObject {
throws RocksDBException {
remove(nativeHandle_, writeOpt.nativeHandle_, key, key.length);
}
/**
* DB implementations can export properties about their state
via this method. If "property" is a valid property understood by this
DB implementation, fills "*value" with its current value and returns
true. Otherwise returns false.
Valid property names include:
"rocksdb.num-files-at-level<N>" - return the number of files at level <N>,
where <N> is an ASCII representation of a level number (e.g. "0").
"rocksdb.stats" - returns a multi-line string that describes statistics
about the internal operation of the DB.
"rocksdb.sstables" - returns a multi-line string that describes all
of the sstables that make up the db contents.
*/
public String getProperty(String property) throws RocksDBException {
return getProperty0(nativeHandle_, property, property.length());
}
/**
* Return a heap-allocated iterator over the contents of the database.
@@ -398,8 +363,6 @@ public class RocksDB extends RocksObject {
protected native void remove(
long handle, long writeOptHandle,
byte[] key, int keyLen) throws RocksDBException;
protected native String getProperty0(long nativeHandle,
String property, int propertyLength) throws RocksDBException;
protected native long iterator0(long optHandle);
private native void disposeInternal(long handle);

2
src/rocksdb2/java/org/rocksdb/benchmark/DbBenchmark.java
View File

@@ -255,7 +255,7 @@ public class DbBenchmark {
for (long j = 0; j < entriesPerBatch_; j++) {
getKey(key, i + j, keyRange_);
DbBenchmark.this.gen_.generate(value);
batch.put(key, value);
db_.put(writeOpt_, key, value);
stats_.finishedSingleOp(keySize_ + valueSize_);
}
db_.write(writeOpt_, batch);

12
src/rocksdb2/java/org/rocksdb/test/OptionsTest.java
View File

@@ -52,6 +52,12 @@ public class OptionsTest {
assert(opt.useFsync() == boolValue);
}
{ // DbStatsLogInterval test
int intValue = rand.nextInt();
opt.setDbStatsLogInterval(intValue);
assert(opt.dbStatsLogInterval() == intValue);
}
{ // DbLogDir test
String str = "path/to/DbLogDir";
opt.setDbLogDir(str);
@@ -184,6 +190,12 @@ public class OptionsTest {
assert(opt.bytesPerSync() == longValue);
}
{ // AllowThreadLocal test
boolean boolValue = rand.nextBoolean();
opt.setAllowThreadLocal(boolValue);
assert(opt.allowThreadLocal() == boolValue);
}
{ // WriteBufferSize test
long longValue = rand.nextLong();
opt.setWriteBufferSize(longValue);