Merge branch 'master' into columnfamilies

Conflicts:
	db/db_impl.cc
	include/rocksdb/options.h
	util/options.cc

Author: igorcanadi

HISTORY.md

@@ -13,19 +13,21 @@
 * Removed arena.h from public header files.
 * By default, checksums are verified on every read from database
 * Added is_manual_compaction to CompactionFilter::Context
-* Added "virtual void WaitForJoin() = 0" in class Env
+* Added "virtual void WaitForJoin()" in class Env. Default operation is no-op.
 * Removed BackupEngine::DeleteBackupsNewerThan() function
 * Added new option -- verify_checksums_in_compaction
 * Chagned Options.prefix_extractor from raw pointer to shared_ptr (take ownership)
   Changed HashSkipListRepFactory and HashLinkListRepFactory constructor to not take SliceTransform object (use Options.prefix_extractor implicitly)
 * Added Env::GetThreadPoolQueueLen(), which returns the waiting queue length of thread pools
 * Added a command "checkconsistency" in ldb tool, which checks
   if file system state matches DB state (file existence and file sizes)
+* CompactionFilter::Context is now CompactionFilterContext. It is shared by CompactionFilter and CompactionFilterV2
 
 ### New Features
 * If we find one truncated record at the end of the MANIFEST or WAL files,
   we will ignore it. We assume that writers of these records were interrupted
   and that we can safely ignore it.
+* Now compaction filter has a V2 interface. It buffers the kv-pairs sharing the same key prefix, process them in batches, and return the batched results back to DB.
 
 ## 2.7.0 (01/28/2014)
 

db/builder.cc

@@ -73,6 +73,7 @@ Status BuildTable(const std::string& dbname, Env* env, const Options& options,
 
     MergeHelper merge(internal_comparator.user_comparator(),
                       options.merge_operator.get(), options.info_log.get(),
+                      options.min_partial_merge_operands,
                       true /* internal key corruption is not ok */);
 
     if (purge) {

db/c.cc

@@ -159,12 +159,10 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
       const char* const* operands_list, const size_t* operands_list_length,
       int num_operands,
       unsigned char* success, size_t* new_value_length);
-  char* (*partial_merge_)(
-      void*,
-      const char* key, size_t key_length,
-      const char* left_operand, size_t left_operand_length,
-      const char* right_operand, size_t right_operand_length,
-      unsigned char* success, size_t* new_value_length);
+  char* (*partial_merge_)(void*, const char* key, size_t key_length,
+                          const char* const* operands_list,
+                          const size_t* operands_list_length, int num_operands,
+                          unsigned char* success, size_t* new_value_length);
   void (*delete_value_)(
       void*,
       const char* value, size_t value_length);
@@ -219,21 +217,23 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
     return success;
   }
 
-  virtual bool PartialMerge(
-      const Slice& key,
-      const Slice& left_operand,
-      const Slice& right_operand,
-      std::string* new_value,
-      Logger* logger) const {
+  virtual bool PartialMergeMulti(const Slice& key,
+                                 const std::deque<Slice>& operand_list,
+                                 std::string* new_value, Logger* logger) const {
+    size_t operand_count = operand_list.size();
+    std::vector<const char*> operand_pointers(operand_count);
+    std::vector<size_t> operand_sizes(operand_count);
+    for (size_t i = 0; i < operand_count; ++i) {
+      Slice operand(operand_list[i]);
+      operand_pointers[i] = operand.data();
+      operand_sizes[i] = operand.size();
+    }
 
     unsigned char success;
     size_t new_value_len;
     char* tmp_new_value = (*partial_merge_)(
-        state_,
-        key.data(), key.size(),
-        left_operand.data(), left_operand.size(),
-        right_operand.data(), right_operand.size(),
-        &success, &new_value_len);
+        state_, key.data(), key.size(), &operand_pointers[0], &operand_sizes[0],
+        operand_count, &success, &new_value_len);
     new_value->assign(tmp_new_value, new_value_len);
 
     if (delete_value_ != nullptr) {
@@ -1094,24 +1094,18 @@ rocksdb_filterpolicy_t* rocksdb_filterpolicy_create_bloom(int bits_per_key) {
 }
 
 rocksdb_mergeoperator_t* rocksdb_mergeoperator_create(
-    void* state,
-    void (*destructor)(void*),
-    char* (*full_merge)(
-        void*,
-        const char* key, size_t key_length,
-        const char* existing_value, size_t existing_value_length,
-        const char* const* operands_list, const size_t* operands_list_length,
-        int num_operands,
-        unsigned char* success, size_t* new_value_length),
-    char* (*partial_merge)(
-        void*,
-        const char* key, size_t key_length,
-        const char* left_operand, size_t left_operand_length,
-        const char* right_operand, size_t right_operand_length,
-        unsigned char* success, size_t* new_value_length),
-    void (*delete_value)(
-        void*,
-        const char* value, size_t value_length),
+    void* state, void (*destructor)(void*),
+    char* (*full_merge)(void*, const char* key, size_t key_length,
+                        const char* existing_value,
+                        size_t existing_value_length,
+                        const char* const* operands_list,
+                        const size_t* operands_list_length, int num_operands,
+                        unsigned char* success, size_t* new_value_length),
+    char* (*partial_merge)(void*, const char* key, size_t key_length,
+                           const char* const* operands_list,
+                           const size_t* operands_list_length, int num_operands,
+                           unsigned char* success, size_t* new_value_length),
+    void (*delete_value)(void*, const char* value, size_t value_length),
     const char* (*name)(void*)) {
   rocksdb_mergeoperator_t* result = new rocksdb_mergeoperator_t;
   result->state_ = state;

db/c_test.c

@@ -175,8 +175,8 @@ static char* MergeOperatorFullMerge(
 static char* MergeOperatorPartialMerge(
     void* arg,
     const char* key, size_t key_length,
-    const char* left_operand, size_t left_operand_length,
-    const char* right_operand, size_t right_operand_length,
+    const char* const* operands_list, const size_t* operands_list_length,
+    int num_operands,
     unsigned char* success, size_t* new_value_length) {
   *new_value_length = 4;
   *success = 1;