Move compaction picker and internal key comparator to ColumnFamilyData

Summary: Compaction picker and internal key comparator are different for each column family (not global), so they should live in ColumnFamilyData

Test Plan: make check

Reviewers: dhruba, haobo

CC: leveldb

Differential Revision: https://reviews.facebook.net/D15801

Author: igorcanadi

db/column_family.cc

@@ -14,6 +14,7 @@
 #include <algorithm>
 
 #include "db/version_set.h"
+#include "db/compaction_picker.h"
 
 namespace rocksdb {
 
@@ -65,14 +66,21 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
       dummy_versions_(dummy_versions),
       current_(nullptr),
       options_(options),
+      icmp_(options_.comparator),
       mem_(nullptr),
       imm_(options.min_write_buffer_number_to_merge),
       super_version_(nullptr),
       super_version_number_(0),
       next_(nullptr),
       prev_(nullptr),
       log_number_(0),
-      need_slowdown_for_num_level0_files_(false) {}
+      need_slowdown_for_num_level0_files_(false) {
+  if (options_.compaction_style == kCompactionStyleUniversal) {
+    compaction_picker_.reset(new UniversalCompactionPicker(&options_, &icmp_));
+  } else {
+    compaction_picker_.reset(new LevelCompactionPicker(&options_, &icmp_));
+  }
+}
 
 ColumnFamilyData::~ColumnFamilyData() {
   if (super_version_ != nullptr) {
@@ -114,6 +122,18 @@ void ColumnFamilyData::CreateNewMemtable() {
   mem_->Ref();
 }
 
+Compaction* ColumnFamilyData::PickCompaction() {
+  return compaction_picker_->PickCompaction(current_);
+}
+
+Compaction* ColumnFamilyData::CompactRange(int input_level, int output_level,
+                                           const InternalKey* begin,
+                                           const InternalKey* end,
+                                           InternalKey** compaction_end) {
+  return compaction_picker_->CompactRange(current_, input_level, output_level,
+                                          begin, end, compaction_end);
+}
+
 SuperVersion* ColumnFamilyData::InstallSuperVersion(
     SuperVersion* new_superversion) {
   new_superversion->Init(mem_, imm_.current(), current_);

db/column_family.h

@@ -23,6 +23,9 @@ class Version;
 class VersionSet;
 class MemTable;
 class MemTableListVersion;
+class CompactionPicker;
+class Compaction;
+class InternalKey;
 
 // holds references to memtable, all immutable memtables and version
 struct SuperVersion {
@@ -62,6 +65,8 @@ class ColumnFamilyData {
   uint32_t GetID() const { return id_; }
   const std::string& GetName() { return name_; }
 
+  int NumberLevels() const { return options_.num_levels; }
+
   void SetLogNumber(uint64_t log_number) { log_number_ = log_number; }
   uint64_t GetLogNumber() const { return log_number_; }
 
@@ -75,6 +80,17 @@ class ColumnFamilyData {
   void SetCurrent(Version* current);
   void CreateNewMemtable();
 
+  // See documentation in compaction_picker.h
+  Compaction* PickCompaction();
+  Compaction* CompactRange(int input_level, int output_level,
+                           const InternalKey* begin, const InternalKey* end,
+                           InternalKey** compaction_end);
+
+  CompactionPicker* compaction_picker() const {
+    return compaction_picker_.get();
+  }
+  const InternalKeyComparator& internal_comparator() const { return icmp_; }
+
   SuperVersion* GetSuperVersion() const { return super_version_; }
   uint64_t GetSuperVersionNumber() const {
     return super_version_number_.load();
@@ -102,6 +118,8 @@ class ColumnFamilyData {
   Version* current_;         // == dummy_versions->prev_
   ColumnFamilyOptions options_;
 
+  const InternalKeyComparator icmp_;
+
   MemTable* mem_;
   MemTableList imm_;
   SuperVersion* super_version_;
@@ -124,6 +142,10 @@ class ColumnFamilyData {
   // A flag indicating whether we should delay writes because
   // we have too many level 0 files
   bool need_slowdown_for_num_level0_files_;
+
+  // An object that keeps all the compaction stats
+  // and picks the next compaction
+  std::unique_ptr<CompactionPicker> compaction_picker_;
 };
 
 // Thread safe only for reading without a writer. All access should be

db/compaction_picker.cc

@@ -41,7 +41,7 @@ uint64_t MultiplyCheckOverflow(uint64_t op1, int op2) {
 
 }  // anonymous namespace
 
-CompactionPicker::CompactionPicker(const Options* options,
+CompactionPicker::CompactionPicker(const ColumnFamilyOptions* options,
                                    const InternalKeyComparator* icmp)
     : compactions_in_progress_(options->num_levels),
       options_(options),

db/compaction_picker.h

@@ -24,7 +24,8 @@ class Version;
 
 class CompactionPicker {
  public:
-  CompactionPicker(const Options* options, const InternalKeyComparator* icmp);
+  CompactionPicker(const ColumnFamilyOptions* options,
+                   const InternalKeyComparator* icmp);
   virtual ~CompactionPicker();
 
   // Pick level and inputs for a new compaction.
@@ -115,7 +116,7 @@ class CompactionPicker {
   // Per-level max bytes
   std::unique_ptr<uint64_t[]> level_max_bytes_;
 
-  const Options* const options_;
+  const ColumnFamilyOptions* const options_;
  private:
   int num_levels_;
 
@@ -124,7 +125,7 @@ class CompactionPicker {
 
 class UniversalCompactionPicker : public CompactionPicker {
  public:
-  UniversalCompactionPicker(const Options* options,
+  UniversalCompactionPicker(const ColumnFamilyOptions* options,
                             const InternalKeyComparator* icmp)
       : CompactionPicker(options, icmp) {}
   virtual Compaction* PickCompaction(Version* version) override;
@@ -141,7 +142,7 @@ class UniversalCompactionPicker : public CompactionPicker {
 
 class LevelCompactionPicker : public CompactionPicker {
  public:
-  LevelCompactionPicker(const Options* options,
+  LevelCompactionPicker(const ColumnFamilyOptions* options,
                         const InternalKeyComparator* icmp)
       : CompactionPicker(options, icmp) {}
   virtual Compaction* PickCompaction(Version* version) override;

db/db_impl.cc

@@ -1312,7 +1312,10 @@ int DBImpl::FindMinimumEmptyLevelFitting(int level) {
     // stop if level i is not empty
     if (current->NumLevelFiles(i) > 0) break;
     // stop if level i is too small (cannot fit the level files)
-    if (versions_->MaxBytesForLevel(i) < current->NumLevelBytes(level)) break;
+    if (default_cfd_->compaction_picker()->MaxBytesForLevel(i) <
+        current->NumLevelBytes(level)) {
+      break;
+    }
 
     minimum_level = i;
   }
@@ -1943,8 +1946,8 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress,
   if (is_manual) {
     ManualCompaction* m = manual_compaction_;
     assert(m->in_progress);
-    c.reset(versions_->CompactRange(
-        m->input_level, m->output_level, m->begin, m->end, &manual_end));
+    c.reset(default_cfd_->CompactRange(m->input_level, m->output_level,
+                                       m->begin, m->end, &manual_end));
     if (!c) {
       m->done = true;
     }
@@ -1959,7 +1962,7 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress,
              ? "(end)"
              : manual_end->DebugString().c_str()));
   } else if (!options_.disable_auto_compactions) {
-    c.reset(versions_->PickCompaction());
+    c.reset(default_cfd_->PickCompaction());
   }
 
   Status status;
@@ -1983,14 +1986,14 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress,
         static_cast<unsigned long long>(f->number), c->level() + 1,
         static_cast<unsigned long long>(f->file_size),
         status.ToString().c_str(), default_cfd_->current()->LevelSummary(&tmp));
-    versions_->ReleaseCompactionFiles(c.get(), status);
+    default_cfd_->compaction_picker()->ReleaseCompactionFiles(c.get(), status);
     *madeProgress = true;
   } else {
     MaybeScheduleFlushOrCompaction(); // do more compaction work in parallel.
     CompactionState* compact = new CompactionState(c.get());
     status = DoCompactionWork(compact, deletion_state);
     CleanupCompaction(compact, status);
-    versions_->ReleaseCompactionFiles(c.get(), status);
+    default_cfd_->compaction_picker()->ReleaseCompactionFiles(c.get(), status);
     c->ReleaseInputs();
     *madeProgress = true;
   }
@@ -2121,7 +2124,8 @@ Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) {
     // Over-estimate slightly so we don't end up just barely crossing
     // the threshold.
     compact->outfile->SetPreallocationBlockSize(
-      1.1 * versions_->MaxFileSizeForLevel(compact->compaction->output_level()));
+        1.1 * default_cfd_->compaction_picker()->MaxFileSizeForLevel(
+                  compact->compaction->output_level()));
 
     CompressionType compression_type = GetCompressionType(
         options_, compact->compaction->output_level(),
@@ -3534,9 +3538,7 @@ bool DBImpl::GetProperty(const ColumnFamilyHandle& column_family,
                          const Slice& property, std::string* value) {
   value->clear();
   MutexLock l(&mutex_);
-  return internal_stats_.GetProperty(property, value, versions_.get(),
-                                     default_cfd_->current(),
-                                     default_cfd_->imm()->size());
+  return internal_stats_.GetProperty(property, value, default_cfd_);
 }
 
 void DBImpl::GetApproximateSizes(const ColumnFamilyHandle& column_family,

db/internal_stats.cc

@@ -8,14 +8,14 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #include "db/internal_stats.h"
+#include "db/column_family.h"
 
 #include <vector>
 
 namespace rocksdb {
 
 bool InternalStats::GetProperty(const Slice& property, std::string* value,
-                                VersionSet* version_set, Version* current,
-                                int immsize) {
+                                ColumnFamilyData* cfd) {
   Slice in = property;
   Slice prefix("rocksdb.");
   if (!in.starts_with(prefix)) return false;
@@ -30,7 +30,7 @@ bool InternalStats::GetProperty(const Slice& property, std::string* value,
     } else {
       char buf[100];
       snprintf(buf, sizeof(buf), "%d",
-               current->NumLevelFiles(static_cast<int>(level)));
+               cfd->current()->NumLevelFiles(static_cast<int>(level)));
       *value = buf;
       return true;
     }
@@ -43,8 +43,8 @@ bool InternalStats::GetProperty(const Slice& property, std::string* value,
 
     for (int level = 0; level < number_levels_; level++) {
       snprintf(buf, sizeof(buf), "%3d %8d %8.0f\n", level,
-               current->NumLevelFiles(level),
-               current->NumLevelBytes(level) / 1048576.0);
+               cfd->current()->NumLevelFiles(level),
+               cfd->current()->NumLevelBytes(level) / 1048576.0);
       value->append(buf);
     }
     return true;
@@ -87,7 +87,7 @@ bool InternalStats::GetProperty(const Slice& property, std::string* value,
              );
     value->append(buf);
     for (int level = 0; level < number_levels_; level++) {
-      int files = current->NumLevelFiles(level);
+      int files = cfd->current()->NumLevelFiles(level);
       if (compaction_stats_[level].micros > 0 || files > 0) {
         int64_t bytes_read = compaction_stats_[level].bytes_readn +
                              compaction_stats_[level].bytes_readnp1;
@@ -117,9 +117,9 @@ bool InternalStats::GetProperty(const Slice& property, std::string* value,
                  "%3d %8d %8.0f %5.1f %9.0f %9.0f %9.0f %9.0f %9.0f %9.0f "
                  "%10.1f %9.1f %11.1f %8d %8d %8d %8d %8d %8d %9.1f %9.1f "
                  "%9lu\n",
-                 level, files, current->NumLevelBytes(level) / 1048576.0,
-                 current->NumLevelBytes(level) /
-                     version_set->MaxBytesForLevel(level),
+                 level, files, cfd->current()->NumLevelBytes(level) / 1048576.0,
+                 cfd->current()->NumLevelBytes(level) /
+                     cfd->compaction_picker()->MaxBytesForLevel(level),
                  compaction_stats_[level].micros / 1e6, bytes_read / 1048576.0,
                  compaction_stats_[level].bytes_written / 1048576.0,
                  compaction_stats_[level].bytes_readn / 1048576.0,
@@ -285,10 +285,10 @@ bool InternalStats::GetProperty(const Slice& property, std::string* value,
 
     return true;
   } else if (in == "sstables") {
-    *value = current->DebugString();
+    *value = cfd->current()->DebugString();
     return true;
   } else if (in == "num-immutable-mem-table") {
-    *value = std::to_string(immsize);
+    *value = std::to_string(cfd->imm()->size());
     return true;
   }
 

db/internal_stats.h

@@ -16,6 +16,8 @@
 #include <vector>
 #include <string>
 
+class ColumnFamilyData;
+
 namespace rocksdb {
 class InternalStats {
  public:
@@ -100,7 +102,7 @@ class InternalStats {
   }
 
   bool GetProperty(const Slice& property, std::string* value,
-                   VersionSet* version_set, Version* current, int immsize);
+                   ColumnFamilyData* cfd);
 
  private:
   std::vector<CompactionStats> compaction_stats_;