Add BloomFilter to PlainTableIterator::Seek()

siying · siying · commit 718488abc5ff · 2013-11-21T22:26:39.000-08:00
Summary:
This patch adds a simple bloom filter in PlainTableIterator::Seek()

Test Plan: N/A

Reviewers:

CC:

Task ID: #

Blame Rev:
diff --git a/db/plain_table_db_test.cc b/db/plain_table_db_test.cc
@@ -59,7 +59,7 @@ class PlainTableDBTest {
   // Return the current option configuration.
   Options CurrentOptions() {
     Options options;
-    options.table_factory.reset(new PlainTableFactory(16, 8));
+    options.table_factory.reset(new PlainTableFactory(16, 8, 2, 0.8));
     options.allow_mmap_reads = true;
     return options;
   }
diff --git a/include/rocksdb/plain_table_factory.h b/include/rocksdb/plain_table_factory.h
@@ -47,8 +47,15 @@ class PlainTableFactory: public TableFactory {
 public:
   ~PlainTableFactory() {
   }
-  PlainTableFactory(int user_key_size, int key_prefix_len) :
-      user_key_size_(user_key_size), key_prefix_len_(key_prefix_len) {
+  // user_key_size is the length of the user key. key_prefix_len is the
+  // length of the prefix used for im-memory indexes. bloom_num_bits is
+  // number of bits is used for bloom filer per key. hash_table_ratio is
+  // the desired ultilization of the hash table used for prefix hashing.
+  // hash_table_ratio = number of prefixes / #buckets in the hash table
+  PlainTableFactory(int user_key_size, int key_prefix_len,
+                    int bloom_num_bits = 0, double hash_table_ratio = 0.75) :
+      user_key_size_(user_key_size), key_prefix_len_(key_prefix_len),
+      bloom_num_bits_(bloom_num_bits), hash_table_ratio_(hash_table_ratio) {
   }
   const char* Name() const override {
     return "PlainTable";
@@ -64,6 +71,8 @@ class PlainTableFactory: public TableFactory {
 private:
   int user_key_size_;
   int key_prefix_len_;
+  int bloom_num_bits_;
+  double hash_table_ratio_;
 };
 
 }  // namespace rocksdb
diff --git a/table/plain_table_factory.cc b/table/plain_table_factory.cc
@@ -19,13 +19,13 @@ Status PlainTableFactory::GetTableReader(const Options& options,
                                          unique_ptr<TableReader>* table)
      const {
   return PlainTableReader::Open(options, soptions, std::move(file), file_size,
-                                  table, user_key_size_, key_prefix_len_);
+                                  table, user_key_size_, key_prefix_len_,
+                                  bloom_num_bits_, hash_table_ratio_);
 }
 
 TableBuilder* PlainTableFactory::GetTableBuilder(
     const Options& options, WritableFile* file,
     CompressionType compression_type) const {
-  return new PlainTableBuilder(options, file, user_key_size_,
-                                 key_prefix_len_);
+  return new PlainTableBuilder(options, file, user_key_size_, key_prefix_len_);
 }
 }  // namespace rocksdb
diff --git a/table/plain_table_reader.cc b/table/plain_table_reader.cc
@@ -40,16 +40,26 @@ namespace rocksdb {
 
 PlainTableReader::PlainTableReader(const EnvOptions& storage_options,
                                    uint64_t file_size, int user_key_size,
-                                   int key_prefix_len) :
-    soptions_(storage_options), file_size_(file_size),
-    user_key_size_(user_key_size), key_prefix_len_(key_prefix_len) {
+                                   int key_prefix_len, int bloom_bits_per_key,
+                                   double hash_table_ratio) :
+    hash_table_size_(0), soptions_(storage_options), file_size_(file_size),
+    user_key_size_(user_key_size), key_prefix_len_(key_prefix_len),
+    hash_table_ratio_(hash_table_ratio) {
+  if (bloom_bits_per_key > 0) {
+    filter_policy_ = NewBloomFilterPolicy(bloom_bits_per_key);
+  } else {
+    filter_policy_ = nullptr;
+  }
   hash_table_ = nullptr;
 }
 
 PlainTableReader::~PlainTableReader() {
   if (hash_table_ != nullptr) {
     delete[] hash_table_;
   }
+  if (filter_policy_ != nullptr) {
+    delete filter_policy_;
+  }
 }
 
 Status PlainTableReader::Open(const Options& options,
@@ -58,12 +68,16 @@ Status PlainTableReader::Open(const Options& options,
                               uint64_t file_size,
                               unique_ptr<TableReader>* table_reader,
                               const int user_key_size,
-                              const int key_prefix_len) {
+                              const int key_prefix_len,
+                              const int bloom_num_bits,
+                              double hash_table_ratio) {
   assert(options.allow_mmap_reads);
 
   PlainTableReader* t = new PlainTableReader(soptions, file_size,
                                              user_key_size,
-                                             key_prefix_len);
+                                             key_prefix_len,
+                                             bloom_num_bits,
+                                             hash_table_ratio);
   t->file_ = std::move(file);
   t->options_ = options;
   Status s = t->PopulateIndex(file_size);
@@ -146,14 +160,25 @@ Status PlainTableReader::PopulateIndex(uint64_t file_size) {
     delete[] hash_table_;
   }
   // Make the hash table 3/5 full
-  hash_table_size_ = tmp_index.size() * 1.66;
+  std::vector<Slice> filter_entries(0); // for creating bloom filter;
+  if (filter_policy_ != nullptr) {
+    filter_entries.resize(tmp_index.size());
+  }
+  double hash_table_size_multipier =
+      (hash_table_ratio_ < 1.0) ? 1.0 : 1.0 / hash_table_ratio_;
+  hash_table_size_ = tmp_index.size() * hash_table_size_multipier + 1;
   hash_table_ = new char[GetHashTableRecordLen() * hash_table_size_];
   for (int i = 0; i < hash_table_size_; i++) {
     memcpy(GetHashTableBucketPtr(i) + key_prefix_len_, &file_size_,
            kOffsetLen);
   }
 
+  size_t count = 0;
   for (auto it = tmp_index.begin(); it != tmp_index.end(); ++it) {
+    if (filter_policy_ != nullptr) {
+      filter_entries[count++] = it->first;
+    }
+
     int bucket = GetHashTableBucket(it->first);
     uint64_t* hash_value;
     while (true) {
@@ -168,6 +193,10 @@ Status PlainTableReader::PopulateIndex(uint64_t file_size) {
     memcpy(bucket_ptr, it->first.data(), key_prefix_len_);
     memcpy(bucket_ptr + key_prefix_len_, &it->second, kOffsetLen);
   }
+  if (filter_policy_ != nullptr) {
+    filter_policy_->CreateFilter(&filter_entries[0], count, &filter_str_);
+    filter_slice_ = Slice(filter_str_.data(), filter_str_.size());
+  }
 
   Log(options_.info_log, "Number of prefixes: %d, suffix_map length %ld",
       hash_table_size_, sub_index_.length());
@@ -187,7 +216,6 @@ inline void PlainTableReader::GetHashValue(int bucket, uint64_t** ret_value) {
 
 Status PlainTableReader::GetOffset(const Slice& target, uint64_t* offset) {
   Status s;
-
   int bucket = GetHashTableBucket(target);
   uint64_t* found_value;
   Slice hash_key;
@@ -248,6 +276,12 @@ Status PlainTableReader::GetOffset(const Slice& target, uint64_t* offset) {
   return s;
 }
 
+bool PlainTableReader::MayHavePrefix(const Slice& target_prefix) {
+  return filter_policy_ == nullptr
+      || filter_policy_->KeyMayMatch(target_prefix, filter_slice_);
+}
+
+
 uint64_t PlainTableReader::Next(uint64_t offset, Slice* key, Slice* value,
                                 Slice* tmp_slice) {
   if (offset >= file_size_) {
@@ -321,6 +355,11 @@ void PlainTableIterator::SeekToLast() {
 }
 
 void PlainTableIterator::Seek(const Slice& target) {
+  if (!table_->MayHavePrefix(Slice(target.data(), table_->key_prefix_len_))) {
+    offset_ = next_offset_ = table_->file_size_;
+    return;
+  }
+
   Status s = table_->GetOffset(target, &next_offset_);
   if (!s.ok()) {
     status_ = s;
diff --git a/table/plain_table_reader.h b/table/plain_table_reader.h
@@ -57,7 +57,8 @@ class PlainTableReader: public TableReader {
   static Status Open(const Options& options, const EnvOptions& soptions,
                      unique_ptr<RandomAccessFile> && file, uint64_t file_size,
                      unique_ptr<TableReader>* table, const int user_key_size,
-                     const int key_prefix_len);
+                     const int key_prefix_len, const int bloom_num_bits,
+                     double hash_table_ratio);
 
   bool PrefixMayMatch(const Slice& internal_prefix);
 
@@ -79,7 +80,8 @@ class PlainTableReader: public TableReader {
   }
 
   PlainTableReader(const EnvOptions& storage_options, uint64_t file_size,
-                   int user_key_size, int key_prefix_len);
+                   int user_key_size, int key_prefix_len, int bloom_num_bits,
+                   double hash_table_ratio);
   ~PlainTableReader();
 
 private:
@@ -95,6 +97,10 @@ class PlainTableReader: public TableReader {
   uint64_t file_size_;
   const size_t user_key_size_;
   const size_t key_prefix_len_;
+  const double hash_table_ratio_;
+  const FilterPolicy* filter_policy_;
+  std::string filter_str_;
+  Slice filter_slice_;
 
   TableProperties tbl_props;
 
@@ -123,6 +129,7 @@ class PlainTableReader: public TableReader {
   Status PopulateIndex(uint64_t file_size);
   uint64_t Next(uint64_t offset, Slice* key, Slice* value, Slice* tmp_slice);
   Status GetOffset(const Slice& target, uint64_t* offset);
+  bool MayHavePrefix(const Slice& target_prefix);
 
   // No copying allowed
   explicit PlainTableReader(const TableReader&) = delete;
diff --git a/table/table_reader_bench.cc b/table/table_reader_bench.cc
@@ -242,7 +242,7 @@ int main(int argc, char** argv) {
   if (FLAGS_plain_table) {
     options.allow_mmap_reads = true;
     env_options.use_mmap_reads = true;
-    tf = new rocksdb::PlainTableFactory(16, FLAGS_prefix_len);
+    tf = new rocksdb::PlainTableFactory(16, FLAGS_prefix_len, FLAGS_prefix_len);
   } else {
     tf = new rocksdb::BlockBasedTableFactory();
   }

Original file line number	Diff line number	Diff line change
`@@ -59,7 +59,7 @@ class PlainTableDBTest {`
`59`	`59`	`// Return the current option configuration.`
`60`	`60`	`Options CurrentOptions() {`
`61`	`61`	`Options options;`
`62`		`- options.table_factory.reset(new PlainTableFactory(16, 8));`
	`62`	`+ options.table_factory.reset(new PlainTableFactory(16, 8, 2, 0.8));`
`63`	`63`	`options.allow_mmap_reads = true;`
`64`	`64`	`return options;`
`65`	`65`	`}`
Original file line number	Diff line number	Diff line change
`@@ -242,7 +242,7 @@ int main(int argc, char** argv) {`
`242`	`242`	`if (FLAGS_plain_table) {`
`243`	`243`	`options.allow_mmap_reads = true;`
`244`	`244`	`env_options.use_mmap_reads = true;`
`245`		`- tf = new rocksdb::PlainTableFactory(16, FLAGS_prefix_len);`
	`245`	`+ tf = new rocksdb::PlainTableFactory(16, FLAGS_prefix_len, FLAGS_prefix_len);`
`246`	`246`	`} else {`
`247`	`247`	`tf = new rocksdb::BlockBasedTableFactory();`
`248`	`248`	`}`