Support for adding TTL-ed column family

Summary:
This enables user to add a TTL column family to normal DB.

Next step should be to expand StackableDB and create StackableColumnFamily, such that users can for example add geo-spatial column families to normal DB.

Test Plan: added a test

Reviewers: dhruba, haobo, ljin

Reviewed By: haobo

CC: leveldb

Differential Revision: https://reviews.facebook.net/D18201

Author: igorcanadi

include/utilities/db_ttl.h

@@ -0,0 +1,68 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#pragma once
+#ifndef ROCKSDB_LITE
+
+#include <string>
+#include <vector>
+
+#include "utilities/stackable_db.h"
+#include "rocksdb/db.h"
+
+namespace rocksdb {
+
+// Database with TTL support.
+//
+// USE-CASES:
+// This API should be used to open the db when key-values inserted are
+//  meant to be removed from the db in a non-strict 'ttl' amount of time
+//  Therefore, this guarantees that key-values inserted will remain in the
+//  db for >= ttl amount of time and the db will make efforts to remove the
+//  key-values as soon as possible after ttl seconds of their insertion.
+//
+// BEHAVIOUR:
+// TTL is accepted in seconds
+// (int32_t)Timestamp(creation) is suffixed to values in Put internally
+// Expired TTL values deleted in compaction only:(Timestamp+ttl<time_now)
+// Get/Iterator may return expired entries(compaction not run on them yet)
+// Different TTL may be used during different Opens
+// Example: Open1 at t=0 with ttl=4 and insert k1,k2, close at t=2
+//          Open2 at t=3 with ttl=5. Now k1,k2 should be deleted at t>=5
+// read_only=true opens in the usual read-only mode. Compactions will not be
+//  triggered(neither manual nor automatic), so no expired entries removed
+//
+// CONSTRAINTS:
+// Not specifying/passing or non-positive TTL behaves like TTL = infinity
+//
+// !!!WARNING!!!:
+// Calling DB::Open directly to re-open a db created by this API will get
+//  corrupt values(timestamp suffixed) and no ttl effect will be there
+//  during the second Open, so use this API consistently to open the db
+// Be careful when passing ttl with a small positive value because the
+//  whole database may be deleted in a small amount of time
+
+class DBWithTTL : public StackableDB {
+ public:
+  virtual Status CreateColumnFamilyWithTtl(
+      const ColumnFamilyOptions& options, const std::string& column_family_name,
+      ColumnFamilyHandle** handle, int ttl) = 0;
+
+  static Status Open(const Options& options, const std::string& dbname,
+                     DBWithTTL** dbptr, int32_t ttl = 0,
+                     bool read_only = false);
+
+  static Status Open(const DBOptions& db_options, const std::string& dbname,
+                     const std::vector<ColumnFamilyDescriptor>& column_families,
+                     std::vector<ColumnFamilyHandle*>* handles,
+                     DBWithTTL** dbptr, std::vector<int32_t> ttls,
+                     bool read_only = false);
+
+ protected:
+  explicit DBWithTTL(DB* db) : StackableDB(db) {}
+};
+
+}  // namespace rocksdb
+#endif  // ROCKSDB_LITE

include/utilities/stackable_db.h

@@ -21,6 +21,16 @@ class StackableDB : public DB {
     return db_;
   }
 
+  virtual Status CreateColumnFamily(const ColumnFamilyOptions& options,
+                                    const std::string& column_family_name,
+                                    ColumnFamilyHandle** handle) {
+    return db_->CreateColumnFamily(options, column_family_name, handle);
+  }
+
+  virtual Status DropColumnFamily(ColumnFamilyHandle* column_family) {
+    return db_->DropColumnFamily(column_family);
+  }
+
   using DB::Put;
   virtual Status Put(const WriteOptions& options,
                      ColumnFamilyHandle* column_family, const Slice& key,

include/utilities/utility_db.h

@@ -8,55 +8,22 @@
 #include <string>
 
 #include "utilities/stackable_db.h"
+#include "utilities/db_ttl.h"
 #include "rocksdb/db.h"
 
 namespace rocksdb {
 
-// This class contains APIs to open rocksdb with specific support eg. TTL
+// Please don't use this class. It's deprecated
 class UtilityDB {
-
-  public:
-    // Open the database with TTL support.
-    //
-    // USE-CASES:
-    // This API should be used to open the db when key-values inserted are
-    //  meant to be removed from the db in a non-strict 'ttl' amount of time
-    //  Therefore, this guarantees that key-values inserted will remain in the
-    //  db for >= ttl amount of time and the db will make efforts to remove the
-    //  key-values as soon as possible after ttl seconds of their insertion.
-    //
-    // BEHAVIOUR:
-    // TTL is accepted in seconds
-    // (int32_t)Timestamp(creation) is suffixed to values in Put internally
-    // Expired TTL values deleted in compaction only:(Timestamp+ttl<time_now)
-    // Get/Iterator may return expired entries(compaction not run on them yet)
-    // Different TTL may be used during different Opens
-    // Example: Open1 at t=0 with ttl=4 and insert k1,k2, close at t=2
-    //          Open2 at t=3 with ttl=5. Now k1,k2 should be deleted at t>=5
-    // read_only=true opens in the usual read-only mode. Compactions will not be
-    //  triggered(neither manual nor automatic), so no expired entries removed
-    //
-    // CONSTRAINTS:
-    // Not specifying/passing or non-positive TTL behaves like TTL = infinity
-    //
-    // !!!WARNING!!!:
-    // Calling DB::Open directly to re-open a db created by this API will get
-    //  corrupt values(timestamp suffixed) and no ttl effect will be there
-    //  during the second Open, so use this API consistently to open the db
-    // Be careful when passing ttl with a small positive value because the
-    //  whole database may be deleted in a small amount of time
-    static Status OpenTtlDB(const Options& options,
-                            const std::string& name,
-                            StackableDB** dbptr,
-                            int32_t ttl = 0,
-                            bool read_only = false);
-
-    // OpenTtlDB with column family support
-    static Status OpenTtlDB(
-        const DBOptions& db_options, const std::string& name,
-        const std::vector<ColumnFamilyDescriptor>& column_families,
-        std::vector<ColumnFamilyHandle*>* handles, StackableDB** dbptr,
-        std::vector<int32_t> ttls, bool read_only = false);
+ public:
+  // This function is here only for backwards compatibility. Please use the
+  // functions defined in DBWithTTl (utilities/db_ttl.h)
+  // (deprecated)
+  __attribute__((deprecated)) static Status OpenTtlDB(const Options& options,
+                                                      const std::string& name,
+                                                      StackableDB** dbptr,
+                                                      int32_t ttl = 0,
+                                                      bool read_only = false);
 };
 
 } //  namespace rocksdb

util/ldb_cmd.cc

@@ -14,6 +14,7 @@
 #include "rocksdb/write_batch.h"
 #include "rocksdb/cache.h"
 #include "util/coding.h"
+#include "utilities/ttl/db_ttl_impl.h"
 
 #include <ctime>
 #include <dirent.h>
@@ -909,11 +910,11 @@ void DBDumperCommand::DoCommand() {
   int max_keys = max_keys_;
   int ttl_start;
   if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) {
-    ttl_start = DBWithTTL::kMinTimestamp; // TTL introduction time
+    ttl_start = DBWithTTLImpl::kMinTimestamp;  // TTL introduction time
   }
   int ttl_end;
   if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) {
-    ttl_end = DBWithTTL::kMaxTimestamp; // Max time allowed by TTL feature
+    ttl_end = DBWithTTLImpl::kMaxTimestamp;  // Max time allowed by TTL feature
   }
   if (ttl_end < ttl_start) {
     fprintf(stderr, "Error: End time can't be less than start time\n");
@@ -1600,11 +1601,11 @@ void ScanCommand::DoCommand() {
   }
   int ttl_start;
   if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) {
-    ttl_start = DBWithTTL::kMinTimestamp; // TTL introduction time
+    ttl_start = DBWithTTLImpl::kMinTimestamp;  // TTL introduction time
   }
   int ttl_end;
   if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) {
-    ttl_end = DBWithTTL::kMaxTimestamp; // Max time allowed by TTL feature
+    ttl_end = DBWithTTLImpl::kMaxTimestamp;  // Max time allowed by TTL feature
   }
   if (ttl_end < ttl_start) {
     fprintf(stderr, "Error: End time can't be less than start time\n");

util/ldb_cmd.h

@@ -19,8 +19,8 @@
 #include "util/logging.h"
 #include "util/ldb_cmd_execute_result.h"
 #include "util/string_util.h"
-#include "utilities/utility_db.h"
-#include "utilities/ttl/db_ttl.h"
+#include "utilities/db_ttl.h"
+#include "utilities/ttl/db_ttl_impl.h"
 
 using std::string;
 using std::map;
@@ -149,7 +149,7 @@ class LDBCommand {
   LDBCommandExecuteResult exec_state_;
   string db_path_;
   DB* db_;
-  StackableDB* sdb_;
+  DBWithTTL* db_ttl_;
 
   /**
    * true implies that this command can work if the db is opened in read-only
@@ -217,11 +217,11 @@ class LDBCommand {
     Status st;
     if (is_db_ttl_) {
       if (is_read_only_) {
-        st = UtilityDB::OpenTtlDB(opt, db_path_, &sdb_, 0, true);
+        st = DBWithTTL::Open(opt, db_path_, &db_ttl_, 0, true);
       } else {
-        st = UtilityDB::OpenTtlDB(opt, db_path_, &sdb_);
+        st = DBWithTTL::Open(opt, db_path_, &db_ttl_);
       }
-      db_ = sdb_;
+      db_ = db_ttl_;
     } else if (is_read_only_) {
       st = DB::OpenForReadOnly(opt, db_path_, &db_);
     } else {