Skip to content

Commit ef8b8a8

Browse files
yhchiangyhchiang
committed
[Java] Add Java bindings for memtables and sst format.
Summary: Add Java bindings for memtables and sst format. Specifically, add two abstract Java classses --- MemTableConfig and SstFormatConfig. Each MemTable / SST implementation should has its own config class extends MemTableConfig / SstFormatConfig respectively and pass it to Options via setMemTableConfig / setSstConfig. Test Plan: make rocksdbjava make jdb_test make jdb_bench java/jdb_bench.sh \ --benchmarks=fillseq,readrandom,readwhilewriting \ --memtablerep=hash_skiplist \ --use_plain_table=1 \ --key_size=20 \ --prefix_size=12 \ --value_size=100 \ --cache_size=17179869184 \ --disable_wal=0 \ --sync=0 \ Reviewers: haobo, ankgup87, sdong Reviewed By: haobo CC: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D17997
1 parent 8dc3436 commit ef8b8a8

15 files changed

+755
-47
lines changed

‎java/Makefile

+1-25
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.Iterator
1+
NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.Iterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig
22
NATIVE_INCLUDE = ./include
33
ROCKSDB_JAR = rocksdbjni.jar
44

@@ -28,27 +28,3 @@ test: java
2828

2929
db_bench: java
3030
javac org/rocksdb/benchmark/*.java
31-
rm -rf /tmp/rocksdbjni-bench
32-
java -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.benchmark.DbBenchmark --threads=1 --benchmarks=fillseq,readrandom,readwhilewriting
33-
rm -rf /tmp/rocksdbjni-bench
34-
java -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.benchmark.DbBenchmark --threads=1 --benchmarks=fillseq,readrandom,readwhilewriting --cache_size=200000000
35-
rm -rf /tmp/rocksdbjni-bench
36-
java -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.benchmark.DbBenchmark --threads=2 --benchmarks=fillseq,readrandom,readwhilewriting
37-
rm -rf /tmp/rocksdbjni-bench
38-
java -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.benchmark.DbBenchmark --threads=2 --benchmarks=fillseq,readrandom,readwhilewriting --cache_size=200000000
39-
rm -rf /tmp/rocksdbjni-bench
40-
java -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.benchmark.DbBenchmark --threads=4 --benchmarks=fillseq,readrandom,readwhilewriting
41-
rm -rf /tmp/rocksdbjni-bench
42-
java -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.benchmark.DbBenchmark --threads=4 --benchmarks=fillseq,readrandom,readwhilewriting --cache_size=200000000
43-
rm -rf /tmp/rocksdbjni-bench
44-
java -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.benchmark.DbBenchmark --threads=8 --benchmarks=fillseq,readrandom,readwhilewriting
45-
rm -rf /tmp/rocksdbjni-bench
46-
java -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.benchmark.DbBenchmark --threads=8 --benchmarks=fillseq,readrandom,readwhilewriting --cache_size=200000000
47-
rm -rf /tmp/rocksdbjni-bench
48-
java -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.benchmark.DbBenchmark --threads=16 --benchmarks=fillseq,readrandom,readwhilewriting
49-
rm -rf /tmp/rocksdbjni-bench
50-
java -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.benchmark.DbBenchmark --threads=16 --benchmarks=fillseq,readrandom,readwhilewriting --cache_size=200000000
51-
rm -rf /tmp/rocksdbjni-bench
52-
java -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.benchmark.DbBenchmark --threads=32 --benchmarks=fillseq,readrandom,readwhilewriting
53-
rm -rf /tmp/rocksdbjni-bench
54-
java -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.benchmark.DbBenchmark --threads=32 --benchmarks=fillseq,readrandom,readwhilewriting --cache_size=200000000

‎java/RocksDBSample.java

+23
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,29 @@ public static void main(String[] args) {
4848
assert(options.blockSize() == 64 * SizeUnit.KB);
4949
assert(options.maxBackgroundCompactions() == 10);
5050

51+
assert(options.memTableFactoryName().equals("SkipListFactory"));
52+
options.setMemTableConfig(
53+
new HashSkipListMemTableConfig()
54+
.setHeight(4)
55+
.setBranchingFactor(4)
56+
.setBucketCount(2000000));
57+
assert(options.memTableFactoryName().equals("HashSkipListRepFactory"));
58+
59+
options.setMemTableConfig(
60+
new HashLinkedListMemTableConfig()
61+
.setBucketCount(100000));
62+
assert(options.memTableFactoryName().equals("HashLinkedListRepFactory"));
63+
64+
options.setMemTableConfig(
65+
new VectorMemTableConfig().setReservedSize(10000));
66+
assert(options.memTableFactoryName().equals("VectorRepFactory"));
67+
68+
options.setMemTableConfig(new SkipListMemTableConfig());
69+
assert(options.memTableFactoryName().equals("SkipListFactory"));
70+
71+
options.setTableFormatConfig(new PlainTableConfig());
72+
assert(options.tableFactoryName().equals("PlainTable"));
73+
5174
try {
5275
db = RocksDB.open(options, db_path_not_found);
5376
db.put("hello".getBytes(), "world".getBytes());

‎java/jdb_bench.sh

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
java -Djava.library.path=.:../ -cp "rocksdbjni.jar:.:./*" org.rocksdb.benchmark.DbBenchmark $@

‎java/org/rocksdb/HashLinkedListMemTableConfig.java

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package org.rocksdb;
2+
3+
/**
4+
* The config for hash linked list memtable representation
5+
* Such memtable contains a fix-sized array of buckets, where
6+
* each bucket points to a sorted singly-linked
7+
* list (or null if the bucket is empty).
8+
*
9+
* Note that since this mem-table representation relies on the
10+
* key prefix, it is required to invoke one of the usePrefixExtractor
11+
* functions to specify how to extract key prefix given a key.
12+
* If proper prefix-extractor is not set, then RocksDB will
13+
* use the default memtable representation (SkipList) instead
14+
* and post a warning in the LOG.
15+
*/
16+
public class HashLinkedListMemTableConfig extends MemTableConfig {
17+
public static final long DEFAULT_BUCKET_COUNT = 50000;
18+
19+
public HashLinkedListMemTableConfig() {
20+
bucketCount_ = DEFAULT_BUCKET_COUNT;
21+
}
22+
23+
/**
24+
* Set the number of buckets in the fixed-size array used
25+
* in the hash linked-list mem-table.
26+
*
27+
* @param count the number of hash buckets.
28+
* @return the reference to the current HashLinkedListMemTableConfig.
29+
*/
30+
public HashLinkedListMemTableConfig setBucketCount(long count) {
31+
bucketCount_ = count;
32+
return this;
33+
}
34+
35+
/**
36+
* Returns the number of buckets that will be used in the memtable
37+
* created based on this config.
38+
*
39+
* @return the number of buckets
40+
*/
41+
public long bucketCount() {
42+
return bucketCount_;
43+
}
44+
45+
@Override protected long newMemTableFactoryHandle() {
46+
return newMemTableFactoryHandle(bucketCount_);
47+
}
48+
49+
private native long newMemTableFactoryHandle(long bucketCount);
50+
51+
private long bucketCount_;
52+
}

‎java/org/rocksdb/HashSkipListMemTableConfig.java

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
package org.rocksdb;
2+
3+
/**
4+
* The config for hash skip-list mem-table representation.
5+
* Such mem-table representation contains a fix-sized array of
6+
* buckets, where each bucket points to a skiplist (or null if the
7+
* bucket is empty).
8+
*
9+
* Note that since this mem-table representation relies on the
10+
* key prefix, it is required to invoke one of the usePrefixExtractor
11+
* functions to specify how to extract key prefix given a key.
12+
* If proper prefix-extractor is not set, then RocksDB will
13+
* use the default memtable representation (SkipList) instead
14+
* and post a warning in the LOG.
15+
*/
16+
public class HashSkipListMemTableConfig extends MemTableConfig {
17+
public static final int DEFAULT_BUCKET_COUNT = 1000000;
18+
public static final int DEFAULT_BRANCHING_FACTOR = 4;
19+
public static final int DEFAULT_HEIGHT = 4;
20+
21+
public HashSkipListMemTableConfig() {
22+
bucketCount_ = DEFAULT_BUCKET_COUNT;
23+
branchingFactor_ = DEFAULT_BRANCHING_FACTOR;
24+
height_ = DEFAULT_HEIGHT;
25+
}
26+
27+
/**
28+
* Set the number of hash buckets used in the hash skiplist memtable.
29+
* Default = 1000000.
30+
*
31+
* @param count the number of hash buckets used in the hash
32+
* skiplist memtable.
33+
* @return the reference to the current HashSkipListMemTableConfig.
34+
*/
35+
public HashSkipListMemTableConfig setBucketCount(long count) {
36+
bucketCount_ = count;
37+
return this;
38+
}
39+
40+
/**
41+
* @return the number of hash buckets
42+
*/
43+
public long bucketCount() {
44+
return bucketCount_;
45+
}
46+
47+
/**
48+
* Set the height of the skip list. Default = 4.
49+
*
50+
* @return the reference to the current HashSkipListMemTableConfig.
51+
*/
52+
public HashSkipListMemTableConfig setHeight(int height) {
53+
height_ = height;
54+
return this;
55+
}
56+
57+
/**
58+
* @return the height of the skip list.
59+
*/
60+
public int height() {
61+
return height_;
62+
}
63+
64+
/**
65+
* Set the branching factor used in the hash skip-list memtable.
66+
* This factor controls the probabilistic size ratio between adjacent
67+
* links in the skip list.
68+
*
69+
* @param bf the probabilistic size ratio between adjacent link
70+
* lists in the skip list.
71+
* @return the reference to the current HashSkipListMemTableConfig.
72+
*/
73+
public HashSkipListMemTableConfig setBranchingFactor(int bf) {
74+
branchingFactor_ = bf;
75+
return this;
76+
}
77+
78+
/**
79+
* @return branching factor, the probabilistic size ratio between
80+
* adjacent links in the skip list.
81+
*/
82+
public int branchingFactor() {
83+
return branchingFactor_;
84+
}
85+
86+
@Override protected long newMemTableFactoryHandle() {
87+
return newMemTableFactoryHandle(
88+
bucketCount_, height_, branchingFactor_);
89+
}
90+
91+
private native long newMemTableFactoryHandle(
92+
long bucketCount, int height, int branchingFactor);
93+
94+
private long bucketCount_;
95+
private int branchingFactor_;
96+
private int height_;
97+
}

‎java/org/rocksdb/MemTableConfig.java

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
2+
// This source code is licensed under the BSD-style license found in the
3+
// LICENSE file in the root directory of this source tree. An additional grant
4+
// of patent rights can be found in the PATENTS file in the same directory.
5+
package org.rocksdb;
6+
7+
/**
8+
* MemTableConfig is used to config the internal mem-table of a RocksDB.
9+
* It is required for each memtable to have one such sub-class to allow
10+
* Java developers to use it.
11+
*
12+
* To make a RocksDB to use a specific MemTable format, its associated
13+
* MemTableConfig should be properly set and passed into Options
14+
* via Options.setMemTableFactory() and open the db using that Options.
15+
*
16+
* @see Options
17+
*/
18+
public abstract class MemTableConfig {
19+
/**
20+
* This function should only be called by Options.setMemTableConfig(),
21+
* which will create a c++ shared-pointer to the c++ MemTableRepFactory
22+
* that associated with the Java MemTableConfig.
23+
*
24+
* @see Options.setMemTableFactory()
25+
*/
26+
abstract protected long newMemTableFactoryHandle();
27+
}

‎java/org/rocksdb/Options.java

+71
Original file line numberDiff line numberDiff line change
@@ -1129,6 +1129,64 @@ public Options setAllowThreadLocal(boolean allowThreadLocal) {
11291129
private native void setAllowThreadLocal(
11301130
long handle, boolean allowThreadLocal);
11311131

1132+
/**
1133+
* Set the config for mem-table.
1134+
*
1135+
* @param config the mem-table config.
1136+
* @return the instance of the current Options.
1137+
*/
1138+
public Options setMemTableConfig(MemTableConfig config) {
1139+
setMemTableFactory(nativeHandle_, config.newMemTableFactoryHandle());
1140+
return this;
1141+
}
1142+
1143+
/**
1144+
* Returns the name of the current mem table representation.
1145+
* Memtable format can be set using setTableFormatConfig.
1146+
*
1147+
* @return the name of the currently-used memtable factory.
1148+
* @see setTableFormatConfig()
1149+
*/
1150+
public String memTableFactoryName() {
1151+
assert(isInitialized());
1152+
return memTableFactoryName(nativeHandle_);
1153+
}
1154+
1155+
/**
1156+
* Set the config for table format.
1157+
*
1158+
* @param config the table format config.
1159+
* @return the reference of the current Options.
1160+
*/
1161+
public Options setTableFormatConfig(TableFormatConfig config) {
1162+
setTableFactory(nativeHandle_, config.newTableFactoryHandle());
1163+
return this;
1164+
}
1165+
1166+
/**
1167+
* @return the name of the currently used table factory.
1168+
*/
1169+
public String tableFactoryName() {
1170+
assert(isInitialized());
1171+
return tableFactoryName(nativeHandle_);
1172+
}
1173+
1174+
/**
1175+
* This prefix-extractor uses the first n bytes of a key as its prefix.
1176+
*
1177+
* In some hash-based memtable representation such as HashLinkedList
1178+
* and HashSkipList, prefixes are used to partition the keys into
1179+
* several buckets. Prefix extractor is used to specify how to
1180+
* extract the prefix given a key.
1181+
*
1182+
* @param n use the first n bytes of a key as its prefix.
1183+
*/
1184+
public Options useFixedLengthPrefixExtractor(int n) {
1185+
assert(isInitialized());
1186+
useFixedLengthPrefixExtractor(nativeHandle_, n);
1187+
return this;
1188+
}
1189+
11321190
/**
11331191
* Release the memory allocated for the current instance
11341192
* in the c++ side.
@@ -1147,6 +1205,10 @@ private boolean isInitialized() {
11471205
return (nativeHandle_ != 0);
11481206
}
11491207

1208+
static final int DEFAULT_PLAIN_TABLE_BLOOM_BITS_PER_KEY = 10;
1209+
static final double DEFAULT_PLAIN_TABLE_HASH_TABLE_RATIO = 0.75;
1210+
static final int DEFAULT_PLAIN_TABLE_INDEX_SPARSENESS = 16;
1211+
11501212
private native void newOptions();
11511213
private native void dispose0();
11521214
private native void setCreateIfMissing(long handle, boolean flag);
@@ -1167,6 +1229,15 @@ private native void setMaxBackgroundCompactions(
11671229
private native void createStatistics(long optHandle);
11681230
private native long statisticsPtr(long optHandle);
11691231

1232+
private native void setMemTableFactory(long handle, long factoryHandle);
1233+
private native String memTableFactoryName(long handle);
1234+
1235+
private native void setTableFactory(long handle, long factoryHandle);
1236+
private native String tableFactoryName(long handle);
1237+
1238+
private native void useFixedLengthPrefixExtractor(
1239+
long handle, int prefixLength);
1240+
11701241
long nativeHandle_;
11711242
long cacheSize_;
11721243
}

0 commit comments

Comments
 (0)