Compacting column families

Summary: This diff enables non-default column families to get compacted both automatically and also by calling CompactRange()

Test Plan: make check

Reviewers: dhruba, haobo, kailiu, sdong

CC: leveldb

Differential Revision: https://reviews.facebook.net/D15813

Author: igorcanadi

db/column_family_test.cc

@@ -83,6 +83,46 @@ class ColumnFamilyTest {
     return result;
   }
 
+  void Compact(int cf, const Slice& start, const Slice& limit) {
+    ASSERT_OK(db_->CompactRange(handles_[cf], &start, &limit));
+  }
+
+  int NumTableFilesAtLevel(int cf, int level) {
+    string property;
+    ASSERT_TRUE(db_->GetProperty(
+        handles_[cf], "rocksdb.num-files-at-level" + NumberToString(level),
+        &property));
+    return atoi(property.c_str());
+  }
+
+  // Return spread of files per level
+  string FilesPerLevel(int cf) {
+    string result;
+    int last_non_zero_offset = 0;
+    for (int level = 0; level < column_family_options_.num_levels; level++) {
+      int f = NumTableFilesAtLevel(cf, level);
+      char buf[100];
+      snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
+      result += buf;
+      if (f > 0) {
+        last_non_zero_offset = result.size();
+      }
+    }
+    result.resize(last_non_zero_offset);
+    return result;
+  }
+
+  // Do n memtable flushes, each of which produces an sstable
+  // covering the range [small,large].
+  void MakeTables(int cf, int n, const string& small,
+                  const string& large) {
+    for (int i = 0; i < n; i++) {
+      ASSERT_OK(Put(cf, small, "begin"));
+      ASSERT_OK(Put(cf, large, "end"));
+      ASSERT_OK(db_->Flush(FlushOptions(), handles_[cf]));
+    }
+  }
+
   void CopyFile(const string& source, const string& destination,
                 uint64_t size = 0) {
     const EnvOptions soptions;
@@ -111,7 +151,7 @@ class ColumnFamilyTest {
   ColumnFamilyOptions column_family_options_;
   DBOptions db_options_;
   string dbname_;
-  DB* db_;
+  DB* db_ = nullptr;
   Env* env_;
 };
 
@@ -274,6 +314,52 @@ TEST(ColumnFamilyTest, FlushTest) {
   Close();
 }
 
+// This is the same as DBTest::ManualCompaction, but it does all
+// operations on non-default column family
+TEST(ColumnFamilyTest, ManualCompaction) {
+  // iter - 0 with 7 levels
+  // iter - 1 with 3 levels
+  int cf = 1;
+  for (int iter = 0; iter < 2; ++iter) {
+    column_family_options_.num_levels = (iter == 0) ? 3 : 7;
+    Destroy();
+    ASSERT_OK(Open({"default"}));
+    CreateColumnFamilies({"one"});
+    Close();
+    ASSERT_OK(Open({"default", "one"}));
+
+    MakeTables(cf, 3, "p", "q");
+    ASSERT_EQ("1,1,1", FilesPerLevel(cf));
+
+    // Compaction range falls before files
+    Compact(cf, "", "c");
+    ASSERT_EQ("1,1,1", FilesPerLevel(cf));
+
+    // Compaction range falls after files
+    Compact(cf, "r", "z");
+    ASSERT_EQ("1,1,1", FilesPerLevel(cf));
+
+    // Compaction range overlaps files
+    Compact(cf, "p1", "p9");
+    ASSERT_EQ("0,0,1", FilesPerLevel(cf));
+
+    // Populate a different range
+    MakeTables(cf, 3, "c", "e");
+    ASSERT_EQ("1,1,2", FilesPerLevel(cf));
+
+    // Compact just the new range
+    Compact(cf, "b", "f");
+    ASSERT_EQ("0,0,2", FilesPerLevel(cf));
+
+    // Compact all
+    MakeTables(cf, 1, "a", "z");
+    ASSERT_EQ("0,1,2", FilesPerLevel(cf));
+    Compact(cf, "", "zzz");
+    ASSERT_EQ("0,0,1", FilesPerLevel(cf));
+  }
+  Close();
+}
+
 
 }  // namespace rocksdb
 

db/compaction.cc

@@ -8,6 +8,7 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #include "db/compaction.h"
+#include "db/column_family.h"
 
 namespace rocksdb {
 
@@ -29,6 +30,7 @@ Compaction::Compaction(Version* input_version, int level, int out_level,
       max_grandparent_overlap_bytes_(max_grandparent_overlap_bytes),
       input_version_(input_version),
       number_levels_(input_version_->NumberLevels()),
+      cfd_(input_version_->cfd_),
       seek_compaction_(seek_compaction),
       enable_compression_(enable_compression),
       grandparent_index_(0),
@@ -43,6 +45,7 @@ Compaction::Compaction(Version* input_version, int level, int out_level,
 
   input_version_->Ref();
   edit_ = new VersionEdit();
+  edit_->SetColumnFamily(cfd_->GetID());
   for (int i = 0; i < number_levels_; i++) {
     level_ptrs_[i] = 0;
   }
@@ -170,6 +173,10 @@ void Compaction::ReleaseInputs() {
   }
 }
 
+void Compaction::ReleaseCompactionFiles(Status status) {
+  cfd_->compaction_picker()->ReleaseCompactionFiles(this, status);
+}
+
 void Compaction::ResetNextCompactionIndex() {
   input_version_->ResetNextCompactionIndex(level_);
 }

db/compaction.h

@@ -13,6 +13,7 @@
 namespace rocksdb {
 
 class Version;
+class ColumnFamilyData;
 
 // A Compaction encapsulates information about a compaction.
 class Compaction {
@@ -36,6 +37,8 @@ class Compaction {
   // Returns input version of the compaction
   Version* input_version() const { return input_version_; }
 
+  ColumnFamilyData* column_family_data() const { return cfd_; }
+
   // Return the ith input file at "level()+which" ("which" must be 0 or 1).
   FileMetaData* input(int which, int i) const { return inputs_[which][i]; }
 
@@ -67,6 +70,10 @@ class Compaction {
   // is successful.
   void ReleaseInputs();
 
+  // Clear all files to indicate that they are not being compacted
+  // Delete this compaction from the list of running compactions.
+  void ReleaseCompactionFiles(Status status);
+
   void Summary(char* output, int len);
 
   // Return the score that was used to pick this compaction run.
@@ -94,6 +101,7 @@ class Compaction {
   Version* input_version_;
   VersionEdit* edit_;
   int number_levels_;
+  ColumnFamilyData* cfd_;
 
   bool seek_compaction_;
   bool enable_compression_;