Skip to content

Commit 51af7c3

Browse files
author
Lei Jin
committed
CuckooTable: add one option to allow identity function for the first hash function
Summary: MurmurHash becomes expensive when we do millions Get() a second in one thread. Add this option to allow the first hash function to use identity function as hash function. It results in QPS increase from 3.7M/s to ~4.3M/s. I did not observe improvement for end to end RocksDB performance. This may be caused by other bottlenecks that I will address in a separate diff. Test Plan: ``` [ljin@dev1964 rocksdb] ./cuckoo_table_reader_test --enable_perf --file_dir=/dev/shm --write --identity_as_first_hash=0 ==== Test CuckooReaderTest.WhenKeyExists ==== Test CuckooReaderTest.WhenKeyExistsWithUint64Comparator ==== Test CuckooReaderTest.CheckIterator ==== Test CuckooReaderTest.CheckIteratorUint64 ==== Test CuckooReaderTest.WhenKeyNotFound ==== Test CuckooReaderTest.TestReadPerformance With 125829120 items, utilization is 93.75%, number of hash functions: 2. Time taken per op is 0.272us (3.7 Mqps) with batch size of 0, # of found keys 125829120 With 125829120 items, utilization is 93.75%, number of hash functions: 2. Time taken per op is 0.138us (7.2 Mqps) with batch size of 10, # of found keys 125829120 With 125829120 items, utilization is 93.75%, number of hash functions: 2. Time taken per op is 0.142us (7.1 Mqps) with batch size of 25, # of found keys 125829120 With 125829120 items, utilization is 93.75%, number of hash functions: 2. Time taken per op is 0.142us (7.0 Mqps) with batch size of 50, # of found keys 125829120 With 125829120 items, utilization is 93.75%, number of hash functions: 2. Time taken per op is 0.144us (6.9 Mqps) with batch size of 100, # of found keys 125829120 With 104857600 items, utilization is 78.12%, number of hash functions: 2. Time taken per op is 0.201us (5.0 Mqps) with batch size of 0, # of found keys 104857600 With 104857600 items, utilization is 78.12%, number of hash functions: 2. Time taken per op is 0.121us (8.3 Mqps) with batch size of 10, # of found keys 104857600 With 104857600 items, utilization is 78.12%, number of hash functions: 2. Time taken per op is 0.123us (8.1 Mqps) with batch size of 25, # of found keys 104857600 With 104857600 items, utilization is 78.12%, number of hash functions: 2. Time taken per op is 0.121us (8.3 Mqps) with batch size of 50, # of found keys 104857600 With 104857600 items, utilization is 78.12%, number of hash functions: 2. Time taken per op is 0.112us (8.9 Mqps) with batch size of 100, # of found keys 104857600 With 83886080 items, utilization is 62.50%, number of hash functions: 2. Time taken per op is 0.251us (4.0 Mqps) with batch size of 0, # of found keys 83886080 With 83886080 items, utilization is 62.50%, number of hash functions: 2. Time taken per op is 0.107us (9.4 Mqps) with batch size of 10, # of found keys 83886080 With 83886080 items, utilization is 62.50%, number of hash functions: 2. Time taken per op is 0.099us (10.1 Mqps) with batch size of 25, # of found keys 83886080 With 83886080 items, utilization is 62.50%, number of hash functions: 2. Time taken per op is 0.100us (10.0 Mqps) with batch size of 50, # of found keys 83886080 With 83886080 items, utilization is 62.50%, number of hash functions: 2. Time taken per op is 0.116us (8.6 Mqps) with batch size of 100, # of found keys 83886080 With 73400320 items, utilization is 54.69%, number of hash functions: 2. Time taken per op is 0.189us (5.3 Mqps) with batch size of 0, # of found keys 73400320 With 73400320 items, utilization is 54.69%, number of hash functions: 2. Time taken per op is 0.095us (10.5 Mqps) with batch size of 10, # of found keys 73400320 With 73400320 items, utilization is 54.69%, number of hash functions: 2. Time taken per op is 0.096us (10.4 Mqps) with batch size of 25, # of found keys 73400320 With 73400320 items, utilization is 54.69%, number of hash functions: 2. Time taken per op is 0.098us (10.2 Mqps) with batch size of 50, # of found keys 73400320 With 73400320 items, utilization is 54.69%, number of hash functions: 2. Time taken per op is 0.105us (9.5 Mqps) with batch size of 100, # of found keys 73400320 [ljin@dev1964 rocksdb] ./cuckoo_table_reader_test --enable_perf --file_dir=/dev/shm --write --identity_as_first_hash=1 ==== Test CuckooReaderTest.WhenKeyExists ==== Test CuckooReaderTest.WhenKeyExistsWithUint64Comparator ==== Test CuckooReaderTest.CheckIterator ==== Test CuckooReaderTest.CheckIteratorUint64 ==== Test CuckooReaderTest.WhenKeyNotFound ==== Test CuckooReaderTest.TestReadPerformance With 125829120 items, utilization is 93.75%, number of hash functions: 2. Time taken per op is 0.230us (4.3 Mqps) with batch size of 0, # of found keys 125829120 With 125829120 items, utilization is 93.75%, number of hash functions: 2. Time taken per op is 0.086us (11.7 Mqps) with batch size of 10, # of found keys 125829120 With 125829120 items, utilization is 93.75%, number of hash functions: 2. Time taken per op is 0.088us (11.3 Mqps) with batch size of 25, # of found keys 125829120 With 125829120 items, utilization is 93.75%, number of hash functions: 2. Time taken per op is 0.083us (12.1 Mqps) with batch size of 50, # of found keys 125829120 With 125829120 items, utilization is 93.75%, number of hash functions: 2. Time taken per op is 0.083us (12.1 Mqps) with batch size of 100, # of found keys 125829120 With 104857600 items, utilization is 78.12%, number of hash functions: 2. Time taken per op is 0.159us (6.3 Mqps) with batch size of 0, # of found keys 104857600 With 104857600 items, utilization is 78.12%, number of hash functions: 2. Time taken per op is 0.078us (12.8 Mqps) with batch size of 10, # of found keys 104857600 With 104857600 items, utilization is 78.12%, number of hash functions: 2. Time taken per op is 0.080us (12.6 Mqps) with batch size of 25, # of found keys 104857600 With 104857600 items, utilization is 78.12%, number of hash functions: 2. Time taken per op is 0.080us (12.5 Mqps) with batch size of 50, # of found keys 104857600 With 104857600 items, utilization is 78.12%, number of hash functions: 2. Time taken per op is 0.082us (12.2 Mqps) with batch size of 100, # of found keys 104857600 With 83886080 items, utilization is 62.50%, number of hash functions: 2. Time taken per op is 0.154us (6.5 Mqps) with batch size of 0, # of found keys 83886080 With 83886080 items, utilization is 62.50%, number of hash functions: 2. Time taken per op is 0.077us (13.0 Mqps) with batch size of 10, # of found keys 83886080 With 83886080 items, utilization is 62.50%, number of hash functions: 2. Time taken per op is 0.077us (12.9 Mqps) with batch size of 25, # of found keys 83886080 With 83886080 items, utilization is 62.50%, number of hash functions: 2. Time taken per op is 0.078us (12.8 Mqps) with batch size of 50, # of found keys 83886080 With 83886080 items, utilization is 62.50%, number of hash functions: 2. Time taken per op is 0.079us (12.6 Mqps) with batch size of 100, # of found keys 83886080 With 73400320 items, utilization is 54.69%, number of hash functions: 2. Time taken per op is 0.218us (4.6 Mqps) with batch size of 0, # of found keys 73400320 With 73400320 items, utilization is 54.69%, number of hash functions: 2. Time taken per op is 0.083us (12.0 Mqps) with batch size of 10, # of found keys 73400320 With 73400320 items, utilization is 54.69%, number of hash functions: 2. Time taken per op is 0.085us (11.7 Mqps) with batch size of 25, # of found keys 73400320 With 73400320 items, utilization is 54.69%, number of hash functions: 2. Time taken per op is 0.086us (11.6 Mqps) with batch size of 50, # of found keys 73400320 With 73400320 items, utilization is 54.69%, number of hash functions: 2. Time taken per op is 0.078us (12.8 Mqps) with batch size of 100, # of found keys 73400320 ``` Reviewers: sdong, igor, yhchiang Reviewed By: igor Subscribers: leveldb Differential Revision: https://reviews.facebook.net/D23451
1 parent 0350435 commit 51af7c3

12 files changed

+141
-72
lines changed

HISTORY.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,12 @@
22

33
## Unreleased (will be released with 3.6)
44
### Disk format changes
5-
* If you're using RocksDB on ARM platforms and you're using default bloom filter, there is a disk format change you need to be aware of. There are three steps you need to do when you convert to new release: 1. turn off filter policy, 2. compact the whole database, 3. turn on filter policy
5+
* If you're using RocksDB on ARM platforms and you're using default bloom filter, there is a disk format change you need to be aware of. There are three steps you need to do when you convert to new release: 1. turn off filter policy, 2. compact the whole database, 3. turn on filter policy
66

77
### Behavior changes
88
* We have refactored our system of stalling writes. Any stall-related statistics' meanings are changed. Instead of per-write stall counts, we now count stalls per-epoch, where epochs are periods between flushes and compactions. You'll find more information in our Tuning Perf Guide once we release RocksDB 3.6.
99
* When disableDataSync=true, we no longer sync the MANIFEST file.
10+
* Add identity_as_first_hash property to CuckooTable. SST file needs to be rebuilt to be opened by reader properly.
1011

1112
----- Past Releases -----
1213

db/db_bench.cc

+7-1
Original file line numberDiff line numberDiff line change
@@ -514,6 +514,9 @@ DEFINE_int64(keys_per_prefix, 0, "control average number of keys generated "
514514
"i.e. use the prefix comes with the generated random number.");
515515
DEFINE_bool(enable_io_prio, false, "Lower the background flush/compaction "
516516
"threads' IO priority");
517+
DEFINE_bool(identity_as_first_hash, false, "the first hash function of cuckoo "
518+
"table becomes an identity function. This is only valid when key "
519+
"is 8 bytes");
517520

518521
enum RepFactory {
519522
kSkipList,
@@ -1739,8 +1742,11 @@ class Benchmark {
17391742
fprintf(stderr, "Invalid cuckoo_hash_ratio\n");
17401743
exit(1);
17411744
}
1745+
rocksdb::CuckooTableOptions table_options;
1746+
table_options.hash_table_ratio = FLAGS_cuckoo_hash_ratio;
1747+
table_options.identity_as_first_hash = FLAGS_identity_as_first_hash;
17421748
options.table_factory = std::shared_ptr<TableFactory>(
1743-
NewCuckooTableFactory(FLAGS_cuckoo_hash_ratio));
1749+
NewCuckooTableFactory(table_options));
17441750
} else {
17451751
BlockBasedTableOptions block_based_options;
17461752
if (FLAGS_use_hash_search) {

include/rocksdb/table.h

+27-14
Original file line numberDiff line numberDiff line change
@@ -251,23 +251,36 @@ struct CuckooTablePropertyNames {
251251
// Denotes if the key sorted in the file is Internal Key (if false)
252252
// or User Key only (if true).
253253
static const std::string kIsLastLevel;
254+
// Indicate if using identity function for the first hash function.
255+
static const std::string kIdentityAsFirstHash;
256+
};
257+
258+
struct CuckooTableOptions {
259+
// Determines the utilization of hash tables. Smaller values
260+
// result in larger hash tables with fewer collisions.
261+
double hash_table_ratio = 0.9;
262+
// A property used by builder to determine the depth to go to
263+
// to search for a path to displace elements in case of
264+
// collision. See Builder.MakeSpaceForKey method. Higher
265+
// values result in more efficient hash tables with fewer
266+
// lookups but take more time to build.
267+
uint32_t max_search_depth = 100;
268+
// In case of collision while inserting, the builder
269+
// attempts to insert in the next cuckoo_block_size
270+
// locations before skipping over to the next Cuckoo hash
271+
// function. This makes lookups more cache friendly in case
272+
// of collisions.
273+
uint32_t cuckoo_block_size = 5;
274+
// If this options is enabled, user key is treated as uint64_t and its value
275+
// is used as hash value directly. This option changes builder's behavior.
276+
// Reader ignore this option and behave according to what specified in table
277+
// property.
278+
bool identity_as_first_hash = false;
254279
};
255280

256281
// Cuckoo Table Factory for SST table format using Cache Friendly Cuckoo Hashing
257-
// @hash_table_ratio: Determines the utilization of hash tables. Smaller values
258-
// result in larger hash tables with fewer collisions.
259-
// @max_search_depth: A property used by builder to determine the depth to go to
260-
// to search for a path to displace elements in case of
261-
// collision. See Builder.MakeSpaceForKey method. Higher
262-
// values result in more efficient hash tables with fewer
263-
// lookups but take more time to build.
264-
// @cuckoo_block_size: In case of collision while inserting, the builder
265-
// attempts to insert in the next cuckoo_block_size
266-
// locations before skipping over to the next Cuckoo hash
267-
// function. This makes lookups more cache friendly in case
268-
// of collisions.
269-
extern TableFactory* NewCuckooTableFactory(double hash_table_ratio = 0.9,
270-
uint32_t max_search_depth = 100, uint32_t cuckoo_block_size = 5);
282+
extern TableFactory* NewCuckooTableFactory(
283+
const CuckooTableOptions& table_options = CuckooTableOptions());
271284

272285
#endif // ROCKSDB_LITE
273286

table/cuckoo_table_builder.cc

+12-3
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ const std::string CuckooTablePropertyNames::kIsLastLevel =
3535
"rocksdb.cuckoo.file.islastlevel";
3636
const std::string CuckooTablePropertyNames::kCuckooBlockSize =
3737
"rocksdb.cuckoo.hash.cuckooblocksize";
38+
const std::string CuckooTablePropertyNames::kIdentityAsFirstHash =
39+
"rocksdb.cuckoo.hash.identityfirst";
3840

3941
// Obtained by running echo rocksdb.table.cuckoo | sha1sum
4042
extern const uint64_t kCuckooTableMagicNumber = 0x926789d0c5f17873ull;
@@ -43,6 +45,7 @@ CuckooTableBuilder::CuckooTableBuilder(
4345
WritableFile* file, double max_hash_table_ratio,
4446
uint32_t max_num_hash_table, uint32_t max_search_depth,
4547
const Comparator* user_comparator, uint32_t cuckoo_block_size,
48+
bool identity_as_first_hash,
4649
uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t))
4750
: num_hash_func_(2),
4851
file_(file),
@@ -54,6 +57,7 @@ CuckooTableBuilder::CuckooTableBuilder(
5457
is_last_level_file_(false),
5558
has_seen_first_key_(false),
5659
ucomp_(user_comparator),
60+
identity_as_first_hash_(identity_as_first_hash),
5761
get_slice_hash_(get_slice_hash),
5862
closed_(false) {
5963
// Data is in a huge block.
@@ -119,7 +123,7 @@ Status CuckooTableBuilder::MakeHashTable(std::vector<CuckooBucket>* buckets) {
119123
for (uint32_t hash_cnt = 0; hash_cnt < num_hash_func_ && !bucket_found;
120124
++hash_cnt) {
121125
uint64_t hash_val = CuckooHash(user_key, hash_cnt,
122-
hash_table_size_minus_one, get_slice_hash_);
126+
hash_table_size_minus_one, identity_as_first_hash_, get_slice_hash_);
123127
// If there is a collision, check next cuckoo_block_size_ locations for
124128
// empty locations. While checking, if we reach end of the hash table,
125129
// stop searching and proceed for next hash function.
@@ -149,7 +153,7 @@ Status CuckooTableBuilder::MakeHashTable(std::vector<CuckooBucket>* buckets) {
149153
// We don't really need to rehash the entire table because old hashes are
150154
// still valid and we only increased the number of hash functions.
151155
uint64_t hash_val = CuckooHash(user_key, num_hash_func_,
152-
hash_table_size_minus_one, get_slice_hash_);
156+
hash_table_size_minus_one, identity_as_first_hash_, get_slice_hash_);
153157
++num_hash_func_;
154158
for (uint32_t block_idx = 0; block_idx < cuckoo_block_size_;
155159
++block_idx, ++hash_val) {
@@ -261,6 +265,10 @@ Status CuckooTableBuilder::Finish() {
261265
CuckooTablePropertyNames::kCuckooBlockSize].assign(
262266
reinterpret_cast<const char*>(&cuckoo_block_size_),
263267
sizeof(cuckoo_block_size_));
268+
properties_.user_collected_properties[
269+
CuckooTablePropertyNames::kIdentityAsFirstHash].assign(
270+
reinterpret_cast<const char*>(&identity_as_first_hash_),
271+
sizeof(identity_as_first_hash_));
264272

265273
// Write meta blocks.
266274
MetaIndexBuilder meta_index_builder;
@@ -380,7 +388,8 @@ bool CuckooTableBuilder::MakeSpaceForKey(
380388
uint64_t child_bucket_id = CuckooHash(
381389
(is_last_level_file_ ? kvs_[curr_bucket.vector_idx].first :
382390
ExtractUserKey(Slice(kvs_[curr_bucket.vector_idx].first))),
383-
hash_cnt, hash_table_size_minus_one, get_slice_hash_);
391+
hash_cnt, hash_table_size_minus_one, identity_as_first_hash_,
392+
get_slice_hash_);
384393
// Iterate inside Cuckoo Block.
385394
for (uint32_t block_idx = 0; block_idx < cuckoo_block_size_;
386395
++block_idx, ++child_bucket_id) {

table/cuckoo_table_builder.h

+2
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ class CuckooTableBuilder: public TableBuilder {
2424
WritableFile* file, double max_hash_table_ratio,
2525
uint32_t max_num_hash_func, uint32_t max_search_depth,
2626
const Comparator* user_comparator, uint32_t cuckoo_block_size,
27+
bool identity_as_first_hash,
2728
uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t));
2829

2930
// REQUIRES: Either Finish() or Abandon() has been called.
@@ -87,6 +88,7 @@ class CuckooTableBuilder: public TableBuilder {
8788
TableProperties properties_;
8889
bool has_seen_first_key_;
8990
const Comparator* ucomp_;
91+
bool identity_as_first_hash_;
9092
uint64_t (*get_slice_hash_)(const Slice& s, uint32_t index,
9193
uint64_t max_num_buckets);
9294
std::string largest_user_key_ = "";

table/cuckoo_table_builder_test.cc

+12-11
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ TEST(CuckooBuilderTest, SuccessWithEmptyFile) {
133133
fname = test::TmpDir() + "/EmptyFile";
134134
ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_));
135135
CuckooTableBuilder builder(writable_file.get(), kHashTableRatio,
136-
4, 100, BytewiseComparator(), 1, GetSliceHash);
136+
4, 100, BytewiseComparator(), 1, false, GetSliceHash);
137137
ASSERT_OK(builder.status());
138138
ASSERT_EQ(0UL, builder.FileSize());
139139
ASSERT_OK(builder.Finish());
@@ -162,7 +162,7 @@ TEST(CuckooBuilderTest, WriteSuccessNoCollisionFullKey) {
162162
fname = test::TmpDir() + "/NoCollisionFullKey";
163163
ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_));
164164
CuckooTableBuilder builder(writable_file.get(), kHashTableRatio,
165-
num_hash_fun, 100, BytewiseComparator(), 1, GetSliceHash);
165+
num_hash_fun, 100, BytewiseComparator(), 1, false, GetSliceHash);
166166
ASSERT_OK(builder.status());
167167
for (uint32_t i = 0; i < user_keys.size(); i++) {
168168
builder.Add(Slice(keys[i]), Slice(values[i]));
@@ -202,7 +202,7 @@ TEST(CuckooBuilderTest, WriteSuccessWithCollisionFullKey) {
202202
fname = test::TmpDir() + "/WithCollisionFullKey";
203203
ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_));
204204
CuckooTableBuilder builder(writable_file.get(), kHashTableRatio,
205-
num_hash_fun, 100, BytewiseComparator(), 1, GetSliceHash);
205+
num_hash_fun, 100, BytewiseComparator(), 1, false, GetSliceHash);
206206
ASSERT_OK(builder.status());
207207
for (uint32_t i = 0; i < user_keys.size(); i++) {
208208
builder.Add(Slice(keys[i]), Slice(values[i]));
@@ -243,7 +243,8 @@ TEST(CuckooBuilderTest, WriteSuccessWithCollisionAndCuckooBlock) {
243243
fname = test::TmpDir() + "/WithCollisionFullKey2";
244244
ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_));
245245
CuckooTableBuilder builder(writable_file.get(), kHashTableRatio,
246-
num_hash_fun, 100, BytewiseComparator(), cuckoo_block_size, GetSliceHash);
246+
num_hash_fun, 100, BytewiseComparator(), cuckoo_block_size, false,
247+
GetSliceHash);
247248
ASSERT_OK(builder.status());
248249
for (uint32_t i = 0; i < user_keys.size(); i++) {
249250
builder.Add(Slice(keys[i]), Slice(values[i]));
@@ -288,7 +289,7 @@ TEST(CuckooBuilderTest, WithCollisionPathFullKey) {
288289
fname = test::TmpDir() + "/WithCollisionPathFullKey";
289290
ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_));
290291
CuckooTableBuilder builder(writable_file.get(), kHashTableRatio,
291-
num_hash_fun, 100, BytewiseComparator(), 1, GetSliceHash);
292+
num_hash_fun, 100, BytewiseComparator(), 1, false, GetSliceHash);
292293
ASSERT_OK(builder.status());
293294
for (uint32_t i = 0; i < user_keys.size(); i++) {
294295
builder.Add(Slice(keys[i]), Slice(values[i]));
@@ -330,7 +331,7 @@ TEST(CuckooBuilderTest, WithCollisionPathFullKeyAndCuckooBlock) {
330331
fname = test::TmpDir() + "/WithCollisionPathFullKeyAndCuckooBlock";
331332
ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_));
332333
CuckooTableBuilder builder(writable_file.get(), kHashTableRatio,
333-
num_hash_fun, 100, BytewiseComparator(), 2, GetSliceHash);
334+
num_hash_fun, 100, BytewiseComparator(), 2, false, GetSliceHash);
334335
ASSERT_OK(builder.status());
335336
for (uint32_t i = 0; i < user_keys.size(); i++) {
336337
builder.Add(Slice(keys[i]), Slice(values[i]));
@@ -366,7 +367,7 @@ TEST(CuckooBuilderTest, WriteSuccessNoCollisionUserKey) {
366367
fname = test::TmpDir() + "/NoCollisionUserKey";
367368
ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_));
368369
CuckooTableBuilder builder(writable_file.get(), kHashTableRatio,
369-
num_hash_fun, 100, BytewiseComparator(), 1, GetSliceHash);
370+
num_hash_fun, 100, BytewiseComparator(), 1, false, GetSliceHash);
370371
ASSERT_OK(builder.status());
371372
for (uint32_t i = 0; i < user_keys.size(); i++) {
372373
builder.Add(Slice(GetInternalKey(user_keys[i], true)), Slice(values[i]));
@@ -402,7 +403,7 @@ TEST(CuckooBuilderTest, WriteSuccessWithCollisionUserKey) {
402403
fname = test::TmpDir() + "/WithCollisionUserKey";
403404
ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_));
404405
CuckooTableBuilder builder(writable_file.get(), kHashTableRatio,
405-
num_hash_fun, 100, BytewiseComparator(), 1, GetSliceHash);
406+
num_hash_fun, 100, BytewiseComparator(), 1, false, GetSliceHash);
406407
ASSERT_OK(builder.status());
407408
for (uint32_t i = 0; i < user_keys.size(); i++) {
408409
builder.Add(Slice(GetInternalKey(user_keys[i], true)), Slice(values[i]));
@@ -440,7 +441,7 @@ TEST(CuckooBuilderTest, WithCollisionPathUserKey) {
440441
fname = test::TmpDir() + "/WithCollisionPathUserKey";
441442
ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_));
442443
CuckooTableBuilder builder(writable_file.get(), kHashTableRatio,
443-
num_hash_fun, 2, BytewiseComparator(), 1, GetSliceHash);
444+
num_hash_fun, 2, BytewiseComparator(), 1, false, GetSliceHash);
444445
ASSERT_OK(builder.status());
445446
for (uint32_t i = 0; i < user_keys.size(); i++) {
446447
builder.Add(Slice(GetInternalKey(user_keys[i], true)), Slice(values[i]));
@@ -478,7 +479,7 @@ TEST(CuckooBuilderTest, FailWhenCollisionPathTooLong) {
478479
fname = test::TmpDir() + "/WithCollisionPathUserKey";
479480
ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_));
480481
CuckooTableBuilder builder(writable_file.get(), kHashTableRatio,
481-
num_hash_fun, 2, BytewiseComparator(), 1, GetSliceHash);
482+
num_hash_fun, 2, BytewiseComparator(), 1, false, GetSliceHash);
482483
ASSERT_OK(builder.status());
483484
for (uint32_t i = 0; i < user_keys.size(); i++) {
484485
builder.Add(Slice(GetInternalKey(user_keys[i], false)), Slice("value"));
@@ -498,7 +499,7 @@ TEST(CuckooBuilderTest, FailWhenSameKeyInserted) {
498499
fname = test::TmpDir() + "/FailWhenSameKeyInserted";
499500
ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_));
500501
CuckooTableBuilder builder(writable_file.get(), kHashTableRatio,
501-
num_hash_fun, 100, BytewiseComparator(), 1, GetSliceHash);
502+
num_hash_fun, 100, BytewiseComparator(), 1, false, GetSliceHash);
502503
ASSERT_OK(builder.status());
503504

504505
builder.Add(Slice(GetInternalKey(user_key, false)), Slice("value1"));

table/cuckoo_table_factory.cc

+12-10
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,10 @@ TableBuilder* CuckooTableFactory::NewTableBuilder(
3030
const InternalKeyComparator& internal_comparator,
3131
WritableFile* file, const CompressionType,
3232
const CompressionOptions&) const {
33-
return new CuckooTableBuilder(file, hash_table_ratio_, 64,
34-
max_search_depth_, internal_comparator.user_comparator(),
35-
cuckoo_block_size_, nullptr);
33+
return new CuckooTableBuilder(file, table_options_.hash_table_ratio, 64,
34+
table_options_.max_search_depth, internal_comparator.user_comparator(),
35+
table_options_.cuckoo_block_size, table_options_.identity_as_first_hash,
36+
nullptr);
3637
}
3738

3839
std::string CuckooTableFactory::GetPrintableTableOptions() const {
@@ -42,21 +43,22 @@ std::string CuckooTableFactory::GetPrintableTableOptions() const {
4243
char buffer[kBufferSize];
4344

4445
snprintf(buffer, kBufferSize, " hash_table_ratio: %lf\n",
45-
hash_table_ratio_);
46+
table_options_.hash_table_ratio);
4647
ret.append(buffer);
4748
snprintf(buffer, kBufferSize, " max_search_depth: %u\n",
48-
max_search_depth_);
49+
table_options_.max_search_depth);
4950
ret.append(buffer);
5051
snprintf(buffer, kBufferSize, " cuckoo_block_size: %u\n",
51-
cuckoo_block_size_);
52+
table_options_.cuckoo_block_size);
53+
ret.append(buffer);
54+
snprintf(buffer, kBufferSize, " identity_as_first_hash: %d\n",
55+
table_options_.identity_as_first_hash);
5256
ret.append(buffer);
5357
return ret;
5458
}
5559

56-
TableFactory* NewCuckooTableFactory(double hash_table_ratio,
57-
uint32_t max_search_depth, uint32_t cuckoo_block_size) {
58-
return new CuckooTableFactory(
59-
hash_table_ratio, max_search_depth, cuckoo_block_size);
60+
TableFactory* NewCuckooTableFactory(const CuckooTableOptions& table_options) {
61+
return new CuckooTableFactory(table_options);
6062
}
6163

6264
} // namespace rocksdb

table/cuckoo_table_factory.h

+8-8
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,18 @@ namespace rocksdb {
1616
const uint32_t kCuckooMurmurSeedMultiplier = 816922183;
1717
static inline uint64_t CuckooHash(
1818
const Slice& user_key, uint32_t hash_cnt, uint64_t table_size_minus_one,
19+
bool identity_as_first_hash,
1920
uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t)) {
2021
#ifndef NDEBUG
2122
// This part is used only in unit tests.
2223
if (get_slice_hash != nullptr) {
2324
return get_slice_hash(user_key, hash_cnt, table_size_minus_one + 1);
2425
}
2526
#endif
27+
if (hash_cnt == 0 && identity_as_first_hash) {
28+
return (*reinterpret_cast<const int64_t*>(user_key.data())) &
29+
table_size_minus_one;
30+
}
2631
return MurmurHash(user_key.data(), user_key.size(),
2732
kCuckooMurmurSeedMultiplier * hash_cnt) & table_size_minus_one;
2833
}
@@ -36,11 +41,8 @@ static inline uint64_t CuckooHash(
3641
// - Does not support Merge operations.
3742
class CuckooTableFactory : public TableFactory {
3843
public:
39-
CuckooTableFactory(double hash_table_ratio, uint32_t max_search_depth,
40-
uint32_t cuckoo_block_size)
41-
: hash_table_ratio_(hash_table_ratio),
42-
max_search_depth_(max_search_depth),
43-
cuckoo_block_size_(cuckoo_block_size) {}
44+
explicit CuckooTableFactory(const CuckooTableOptions& table_options)
45+
: table_options_(table_options) {}
4446
~CuckooTableFactory() {}
4547

4648
const char* Name() const override { return "CuckooTable"; }
@@ -63,9 +65,7 @@ class CuckooTableFactory : public TableFactory {
6365
std::string GetPrintableTableOptions() const override;
6466

6567
private:
66-
const double hash_table_ratio_;
67-
const uint32_t max_search_depth_;
68-
const uint32_t cuckoo_block_size_;
68+
const CuckooTableOptions table_options_;
6969
};
7070

7171
} // namespace rocksdb

0 commit comments

Comments
 (0)