Skip to content

Commit 0f2d768

Browse files
author
Lei Jin
committed
hints for narrowing down FindFile range and avoiding checking unrelevant L0 files
Summary: The file tree structure in Version is prebuilt and the range of each file is known. On the Get() code path, we do binary search in FindFile() by comparing target key with each file's largest key and also check the range for each L0 file. With some pre-calculated knowledge, each key comparision that has been done can serve as a hint to narrow down further searches: (1) If a key falls within a L0 file's range, we can safely skip the next file if its range does not overlap with the current one. (2) If a key falls within a file's range in level L0 - Ln-1, we should only need to binary search in the next level for files that overlap with the current one. (1) will be able to skip some files depending one the key distribution. (2) can greatly reduce the range of binary search, especially for bottom levels, given that one file most likely only overlaps with N files from the level below (where N is max_bytes_for_level_multiplier). So on level L, we will only look at ~N files instead of N^L files. Some inital results: measured with 500M key DB, when write is light (10k/s = 1.2M/s), this improves QPS ~7% on top of blocked bloom. When write is heavier (80k/s = 9.6M/s), it gives us ~13% improvement. Test Plan: make all check Reviewers: haobo, igor, dhruba, sdong, yhchiang Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D17205
1 parent 27d3bc1 commit 0f2d768

7 files changed

+771
-34
lines changed

‎Makefile

+4
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ TESTS = \
106106
backupable_db_test \
107107
version_edit_test \
108108
version_set_test \
109+
file_indexer_test \
109110
write_batch_test\
110111
deletefile_test \
111112
table_test \
@@ -376,6 +377,9 @@ version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
376377
version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS)
377378
$(CXX) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
378379

380+
file_indexer_test : db/file_indexer_test.o $(LIBOBJECTS) $(TESTHARNESS)
381+
$(CXX) db/file_indexer_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
382+
379383
reduce_levels_test: tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS)
380384
$(CXX) tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
381385

‎db/file_indexer.cc

+202
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
2+
// This source code is licensed under the BSD-style license found in the
3+
// LICENSE file in the root directory of this source tree. An additional grant
4+
// of patent rights can be found in the PATENTS file in the same directory.
5+
//
6+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7+
// Use of this source code is governed by a BSD-style license that can be
8+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
9+
10+
#include "db/file_indexer.h"
11+
#include <algorithm>
12+
#include "rocksdb/comparator.h"
13+
#include "db/version_edit.h"
14+
15+
namespace rocksdb {
16+
17+
FileIndexer::FileIndexer(const uint32_t num_levels,
18+
const Comparator* ucmp)
19+
: num_levels_(num_levels),
20+
ucmp_(ucmp),
21+
next_level_index_(num_levels),
22+
level_rb_(num_levels, -1) {
23+
}
24+
25+
26+
uint32_t FileIndexer::NumLevelIndex() {
27+
return next_level_index_.size();
28+
}
29+
30+
uint32_t FileIndexer::LevelIndexSize(uint32_t level) {
31+
return next_level_index_[level].size();
32+
}
33+
34+
void FileIndexer::GetNextLevelIndex(
35+
const uint32_t level, const uint32_t file_index, const int cmp_smallest,
36+
const int cmp_largest, int32_t* left_bound, int32_t* right_bound) {
37+
assert(level > 0);
38+
39+
// Last level, no hint
40+
if (level == num_levels_ - 1) {
41+
*left_bound = 0;
42+
*right_bound = -1;
43+
return;
44+
}
45+
46+
assert(level < num_levels_ - 1);
47+
assert(static_cast<int32_t>(file_index) <= level_rb_[level]);
48+
49+
const auto& index = next_level_index_[level][file_index];
50+
51+
if (cmp_smallest < 0) {
52+
*left_bound = (level > 0 && file_index > 0) ?
53+
next_level_index_[level][file_index - 1].largest_lb : 0;
54+
*right_bound = index.smallest_rb;
55+
} else if (cmp_smallest == 0) {
56+
*left_bound = index.smallest_lb;
57+
*right_bound = index.smallest_rb;
58+
} else if (cmp_smallest > 0 && cmp_largest < 0) {
59+
*left_bound = index.smallest_lb;
60+
*right_bound = index.largest_rb;
61+
} else if (cmp_largest == 0) {
62+
*left_bound = index.largest_lb;
63+
*right_bound = index.largest_rb;
64+
} else if (cmp_largest > 0) {
65+
*left_bound = index.largest_lb;
66+
*right_bound = level_rb_[level + 1];
67+
} else {
68+
assert(false);
69+
}
70+
71+
assert(*left_bound >= 0);
72+
assert(*left_bound <= *right_bound + 1);
73+
assert(*right_bound <= level_rb_[level + 1]);
74+
}
75+
76+
void FileIndexer::ClearIndex() {
77+
for (uint32_t level = 1; level < num_levels_; ++level) {
78+
next_level_index_[level].clear();
79+
}
80+
}
81+
82+
void FileIndexer::UpdateIndex(std::vector<FileMetaData*>* const files) {
83+
if (files == nullptr) {
84+
return;
85+
}
86+
87+
// L1 - Ln-1
88+
for (uint32_t level = 1; level < num_levels_ - 1; ++level) {
89+
const auto& upper_files = files[level];
90+
const int32_t upper_size = upper_files.size();
91+
const auto& lower_files = files[level + 1];
92+
level_rb_[level] = upper_files.size() - 1;
93+
if (upper_size == 0) {
94+
continue;
95+
}
96+
auto& index = next_level_index_[level];
97+
index.resize(upper_size);
98+
99+
CalculateLB(upper_files, lower_files, &index,
100+
[this](const FileMetaData* a, const FileMetaData* b) -> int {
101+
return ucmp_->Compare(a->smallest.user_key(), b->largest.user_key());
102+
},
103+
[](IndexUnit* index, int32_t f_idx) {
104+
index->smallest_lb = f_idx;
105+
});
106+
CalculateLB(upper_files, lower_files, &index,
107+
[this](const FileMetaData* a, const FileMetaData* b) -> int {
108+
return ucmp_->Compare(a->largest.user_key(), b->largest.user_key());
109+
},
110+
[](IndexUnit* index, int32_t f_idx) {
111+
index->largest_lb = f_idx;
112+
});
113+
CalculateRB(upper_files, lower_files, &index,
114+
[this](const FileMetaData* a, const FileMetaData* b) -> int {
115+
return ucmp_->Compare(a->smallest.user_key(), b->smallest.user_key());
116+
},
117+
[](IndexUnit* index, int32_t f_idx) {
118+
index->smallest_rb = f_idx;
119+
});
120+
CalculateRB(upper_files, lower_files, &index,
121+
[this](const FileMetaData* a, const FileMetaData* b) -> int {
122+
return ucmp_->Compare(a->largest.user_key(), b->smallest.user_key());
123+
},
124+
[](IndexUnit* index, int32_t f_idx) {
125+
index->largest_rb = f_idx;
126+
});
127+
}
128+
level_rb_[num_levels_ - 1] = files[num_levels_ - 1].size() - 1;
129+
}
130+
131+
void FileIndexer::CalculateLB(const std::vector<FileMetaData*>& upper_files,
132+
const std::vector<FileMetaData*>& lower_files,
133+
std::vector<IndexUnit>* index,
134+
std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
135+
std::function<void(IndexUnit*, int32_t)> set_index) {
136+
const int32_t upper_size = upper_files.size();
137+
const int32_t lower_size = lower_files.size();
138+
int32_t upper_idx = 0;
139+
int32_t lower_idx = 0;
140+
while (upper_idx < upper_size && lower_idx < lower_size) {
141+
int cmp = cmp_op(upper_files[upper_idx], lower_files[lower_idx]);
142+
143+
if (cmp == 0) {
144+
set_index(&(*index)[upper_idx], lower_idx);
145+
++upper_idx;
146+
++lower_idx;
147+
} else if (cmp > 0) {
148+
// Lower level's file (largest) is smaller, a key won't hit in that
149+
// file. Move to next lower file
150+
++lower_idx;
151+
} else {
152+
// Lower level's file becomes larger, update the index, and
153+
// move to the next upper file
154+
set_index(&(*index)[upper_idx], lower_idx);
155+
++upper_idx;
156+
}
157+
}
158+
159+
while (upper_idx < upper_size) {
160+
// Lower files are exhausted, that means the remaining upper files are
161+
// greater than any lower files. Set the index to be the lower level size.
162+
set_index(&(*index)[upper_idx], lower_size);
163+
++upper_idx;
164+
}
165+
}
166+
167+
void FileIndexer::CalculateRB(const std::vector<FileMetaData*>& upper_files,
168+
const std::vector<FileMetaData*>& lower_files,
169+
std::vector<IndexUnit>* index,
170+
std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
171+
std::function<void(IndexUnit*, int32_t)> set_index) {
172+
const int32_t upper_size = upper_files.size();
173+
const int32_t lower_size = lower_files.size();
174+
int32_t upper_idx = upper_size - 1;
175+
int32_t lower_idx = lower_size - 1;
176+
while (upper_idx >= 0 && lower_idx >= 0) {
177+
int cmp = cmp_op(upper_files[upper_idx], lower_files[lower_idx]);
178+
179+
if (cmp == 0) {
180+
set_index(&(*index)[upper_idx], lower_idx);
181+
--upper_idx;
182+
--lower_idx;
183+
} else if (cmp < 0) {
184+
// Lower level's file (smallest) is larger, a key won't hit in that
185+
// file. Move to next lower file.
186+
--lower_idx;
187+
} else {
188+
// Lower level's file becomes smaller, update the index, and move to
189+
// the next the upper file
190+
set_index(&(*index)[upper_idx], lower_idx);
191+
--upper_idx;
192+
}
193+
}
194+
while (upper_idx >= 0) {
195+
// Lower files are exhausted, that means the remaining upper files are
196+
// smaller than any lower files. Set it to -1.
197+
set_index(&(*index)[upper_idx], -1);
198+
--upper_idx;
199+
}
200+
}
201+
202+
} // namespace rocksdb

‎db/file_indexer.h

+129
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
2+
// This source code is licensed under the BSD-style license found in the
3+
// LICENSE file in the root directory of this source tree. An additional grant
4+
// of patent rights can be found in the PATENTS file in the same directory.
5+
//
6+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7+
// Use of this source code is governed by a BSD-style license that can be
8+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
9+
10+
#pragma once
11+
#include <cstdint>
12+
#include <functional>
13+
#include <limits>
14+
#include <vector>
15+
16+
namespace rocksdb {
17+
18+
class Comparator;
19+
class FileMetaData;
20+
21+
// The file tree structure in Version is prebuilt and the range of each file
22+
// is known. On Version::Get(), it uses binary search to find a potential file
23+
// and then check if a target key can be found in the file by comparing the key
24+
// to each file's smallest and largest key. The results of these comparisions
25+
// can be reused beyond checking if a key falls into a file's range.
26+
// With some pre-calculated knowledge, each key comparision that has been done
27+
// can serve as a hint to narrow down further searches: if a key compared to
28+
// be smaller than a file's smallest or largest, that comparison can be used
29+
// to find out the right bound of next binary search. Similarly, if a key
30+
// compared to be larger than a file's smallest or largest, it can be utilized
31+
// to find out the left bound of next binary search.
32+
// With these hints: it can greatly reduce the range of binary search,
33+
// especially for bottom levels, given that one file most likely overlaps with
34+
// only N files from level below (where N is max_bytes_for_level_multiplier).
35+
// So on level L, we will only look at ~N files instead of N^L files on the
36+
// naive approach.
37+
class FileIndexer {
38+
public:
39+
FileIndexer(const uint32_t num_levels, const Comparator* ucmp);
40+
41+
uint32_t NumLevelIndex();
42+
43+
uint32_t LevelIndexSize(uint32_t level);
44+
45+
// Return a file index range in the next level to search for a key based on
46+
// smallest and largest key comparision for the current file specified by
47+
// level and file_index. When *left_index < *right_index, both index should
48+
// be valid and fit in the vector size.
49+
void GetNextLevelIndex(
50+
const uint32_t level, const uint32_t file_index, const int cmp_smallest,
51+
const int cmp_largest, int32_t* left_bound, int32_t* right_bound);
52+
53+
void ClearIndex();
54+
55+
void UpdateIndex(std::vector<FileMetaData*>* const files);
56+
57+
enum {
58+
kLevelMaxIndex = std::numeric_limits<int32_t>::max()
59+
};
60+
61+
private:
62+
const uint32_t num_levels_;
63+
const Comparator* ucmp_;
64+
65+
struct IndexUnit {
66+
IndexUnit()
67+
: smallest_lb(0), largest_lb(0), smallest_rb(-1), largest_rb(-1) {}
68+
// During file search, a key is compared against smallest and largest
69+
// from a FileMetaData. It can have 3 possible outcomes:
70+
// (1) key is smaller than smallest, implying it is also smaller than
71+
// larger. Precalculated index based on "smallest < smallest" can
72+
// be used to provide right bound.
73+
// (2) key is in between smallest and largest.
74+
// Precalculated index based on "smallest > greatest" can be used to
75+
// provide left bound.
76+
// Precalculated index based on "largest < smallest" can be used to
77+
// provide right bound.
78+
// (3) key is larger than largest, implying it is also larger than smallest.
79+
// Precalculated index based on "largest > largest" can be used to
80+
// provide left bound.
81+
//
82+
// As a result, we will need to do:
83+
// Compare smallest (<=) and largest keys from upper level file with
84+
// smallest key from lower level to get a right bound.
85+
// Compare smallest (>=) and largest keys from upper level file with
86+
// largest key from lower level to get a left bound.
87+
//
88+
// Example:
89+
// level 1: [50 - 60]
90+
// level 2: [1 - 40], [45 - 55], [58 - 80]
91+
// A key 35, compared to be less than 50, 3rd file on level 2 can be
92+
// skipped according to rule (1). LB = 0, RB = 1.
93+
// A key 53, sits in the middle 50 and 60. 1st file on level 2 can be
94+
// skipped according to rule (2)-a, but the 3rd file cannot be skipped
95+
// because 60 is greater than 58. LB = 1, RB = 2.
96+
// A key 70, compared to be larger than 60. 1st and 2nd file can be skipped
97+
// according to rule (3). LB = 2, RB = 2.
98+
//
99+
// Point to a left most file in a lower level that may contain a key,
100+
// which compares greater than smallest of a FileMetaData (upper level)
101+
int32_t smallest_lb;
102+
// Point to a left most file in a lower level that may contain a key,
103+
// which compares greater than largest of a FileMetaData (upper level)
104+
int32_t largest_lb;
105+
// Point to a right most file in a lower level that may contain a key,
106+
// which compares smaller than smallest of a FileMetaData (upper level)
107+
int32_t smallest_rb;
108+
// Point to a right most file in a lower level that may contain a key,
109+
// which compares smaller than largest of a FileMetaData (upper level)
110+
int32_t largest_rb;
111+
};
112+
113+
void CalculateLB(const std::vector<FileMetaData*>& upper_files,
114+
const std::vector<FileMetaData*>& lower_files,
115+
std::vector<IndexUnit>* index,
116+
std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
117+
std::function<void(IndexUnit*, int32_t)> set_index);
118+
119+
void CalculateRB(const std::vector<FileMetaData*>& upper_files,
120+
const std::vector<FileMetaData*>& lower_files,
121+
std::vector<IndexUnit>* index,
122+
std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
123+
std::function<void(IndexUnit*, int32_t)> set_index);
124+
125+
std::vector<std::vector<IndexUnit>> next_level_index_;
126+
std::vector<int32_t> level_rb_;
127+
};
128+
129+
} // namespace rocksdb

0 commit comments

Comments
 (0)