Merge branch 'master' into performance

Author: liukai

INSTALL.md

@@ -19,13 +19,13 @@ libraries. You are on your own.
 
 * **Linux**
     * Upgrade your gcc to version at least 4.7 to get C++11 support.
-    * Install gflags. If you're on Ubuntu, here's a nice tutorial:
+    * Install gflags. First, try: `sudo apt-get install libgflags-dev`.
+      If this doesn't work and you're using Ubuntu, here's a nice tutorial:
       (http://askubuntu.com/questions/312173/installing-gflags-12-04)
     * Install snappy. This is usually as easy as:
       `sudo apt-get install libsnappy-dev`.
     * Install zlib. Try: `sudo apt-get install zlib1g-dev`.
     * Install bzip2: `sudo apt-get install libbz2-dev`.
-    * Install gflags: `sudo apt-get install libgflags-dev`.
 * **OS X**:
     * Install latest C++ compiler that supports C++ 11:
         * Update XCode:  run `xcode-select --install` (or install it from XCode App's settting).

README

@@ -16,8 +16,8 @@ The core of this code has been derived from open-source leveldb.
 The code under this directory implements a system for maintaining a
 persistent key/value store.
 
-See doc/index.html for more explanation.
-See doc/impl.html for a brief overview of the implementation.
+See doc/index.html and github wiki (https://github.com/facebook/rocksdb/wiki)
+for more explanation.
 
 The public interface is in include/*.  Callers should not include or
 rely on the details of any other header files in this package.  Those

build_tools/regression_build_test.sh

@@ -65,7 +65,7 @@ OPT=-DNDEBUG make db_bench -j$(nproc)
     --sync=0 \
     --threads=8 > ${STAT_FILE}.overwrite
 
-# fill up the db for readrandom benchmark
+# fill up the db for readrandom benchmark (1GB total size)
 ./db_bench \
     --benchmarks=fillseq \
     --db=$DATA_DIR \
@@ -83,7 +83,7 @@ OPT=-DNDEBUG make db_bench -j$(nproc)
     --sync=0 \
     --threads=1 > /dev/null
 
-# measure readrandom
+# measure readrandom with 6GB block cache
 ./db_bench \
     --benchmarks=readrandom \
     --db=$DATA_DIR \
@@ -102,6 +102,25 @@ OPT=-DNDEBUG make db_bench -j$(nproc)
     --sync=0 \
     --threads=32 > ${STAT_FILE}.readrandom
 
+# measure readrandom with 300MB block cache
+./db_bench \
+    --benchmarks=readrandom \
+    --db=$DATA_DIR \
+    --use_existing_db=1 \
+    --bloom_bits=10 \
+    --num=$NUM \
+    --reads=$NUM \
+    --cache_size=314572800 \
+    --cache_numshardbits=8 \
+    --open_files=55000 \
+    --disable_seek_compaction=1 \
+    --statistics=1 \
+    --histogram=1 \
+    --disable_data_sync=1 \
+    --disable_wal=1 \
+    --sync=0 \
+    --threads=32 > ${STAT_FILE}.readrandomsmallblockcache
+
 # measure memtable performance -- none of the data gets flushed to disk
 ./db_bench \
     --benchmarks=fillrandom,readrandom, \
@@ -154,5 +173,6 @@ function send_benchmark_to_ods {
 send_benchmark_to_ods overwrite overwrite $STAT_FILE.overwrite
 send_benchmark_to_ods fillseq fillseq $STAT_FILE.fillseq
 send_benchmark_to_ods readrandom readrandom $STAT_FILE.readrandom
+send_benchmark_to_ods readrandom readrandom_smallblockcache $STAT_FILE.readrandomsmallblockcache
 send_benchmark_to_ods fillrandom memtablefillrandom $STAT_FILE.memtablefillreadrandom
 send_benchmark_to_ods readrandom memtablereadrandom $STAT_FILE.memtablefillreadrandom

db/db_bench.cc

@@ -191,6 +191,10 @@ DEFINE_int32(universal_max_merge_width, 0, "The max number of files to compact"
 DEFINE_int32(universal_max_size_amplification_percent, 0,
              "The max size amplification for universal style compaction");
 
+DEFINE_int32(universal_compression_size_percent, -1,
+             "The percentage of the database to compress for universal "
+             "compaction. -1 means compress everything.");
+
 DEFINE_int64(cache_size, -1, "Number of bytes to use as a cache of uncompressed"
              "data. Negative means use default settings.");
 
@@ -325,6 +329,23 @@ DEFINE_string(compression_type, "snappy",
 static enum rocksdb::CompressionType FLAGS_compression_type_e =
     rocksdb::kSnappyCompression;
 
+DEFINE_int32(compression_level, -1,
+             "Compression level. For zlib this should be -1 for the "
+             "default level, or between 0 and 9.");
+
+static bool ValidateCompressionLevel(const char* flagname, int32_t value) {
+  if (value < -1 || value > 9) {
+    fprintf(stderr, "Invalid value for --%s: %d, must be between -1 and 9\n",
+            flagname, value);
+    return false;
+  }
+  return true;
+}
+
+static const bool FLAGS_compression_level_dummy =
+  google::RegisterFlagValidator(&FLAGS_compression_level,
+                                &ValidateCompressionLevel);
+
 DEFINE_int32(min_level_to_compress, -1, "If non-negative, compression starts"
              " from this level. Levels with number < min_level_to_compress are"
              " not compressed. Otherwise, apply compression_type to "
@@ -434,12 +455,11 @@ static bool ValidatePrefixSize(const char* flagname, int32_t value) {
   }
   return true;
 }
-DEFINE_int32(prefix_size, 0, "Control the prefix size for PrefixHashRep");
+DEFINE_int32(prefix_size, 0, "Control the prefix size for HashSkipList");
 
 enum RepFactory {
   kSkipList,
   kPrefixHash,
-  kUnsorted,
   kVectorRep
 };
 enum RepFactory StringToRepFactory(const char* ctype) {
@@ -449,8 +469,6 @@ enum RepFactory StringToRepFactory(const char* ctype) {
     return kSkipList;
   else if (!strcasecmp(ctype, "prefix_hash"))
     return kPrefixHash;
-  else if (!strcasecmp(ctype, "unsorted"))
-    return kUnsorted;
   else if (!strcasecmp(ctype, "vector"))
     return kVectorRep;
 
@@ -807,9 +825,6 @@ class Benchmark {
       case kSkipList:
         fprintf(stdout, "Memtablerep: skip_list\n");
         break;
-      case kUnsorted:
-        fprintf(stdout, "Memtablerep: unsorted\n");
-        break;
       case kVectorRep:
         fprintf(stdout, "Memtablerep: vector\n");
         break;
@@ -1334,14 +1349,8 @@ class Benchmark {
     }
     switch (FLAGS_rep_factory) {
       case kPrefixHash:
-        options.memtable_factory.reset(
-          new PrefixHashRepFactory(NewFixedPrefixTransform(FLAGS_prefix_size))
-        );
-        break;
-      case kUnsorted:
-        options.memtable_factory.reset(
-          new UnsortedRepFactory
-        );
+        options.memtable_factory.reset(NewHashSkipListRepFactory(
+            NewFixedPrefixTransform(FLAGS_prefix_size)));
         break;
       case kSkipList:
         // no need to do anything
@@ -1368,6 +1377,7 @@ class Benchmark {
     options.level0_slowdown_writes_trigger =
       FLAGS_level0_slowdown_writes_trigger;
     options.compression = FLAGS_compression_type_e;
+    options.compression_opts.level = FLAGS_compression_level;
     options.WAL_ttl_seconds = FLAGS_wal_ttl_seconds;
     options.WAL_size_limit_MB = FLAGS_wal_size_limit_MB;
     if (FLAGS_min_level_to_compress >= 0) {
@@ -1429,6 +1439,10 @@ class Benchmark {
       options.compaction_options_universal.max_size_amplification_percent =
         FLAGS_universal_max_size_amplification_percent;
     }
+    if (FLAGS_universal_compression_size_percent != -1) {
+      options.compaction_options_universal.compression_size_percent =
+        FLAGS_universal_compression_size_percent;
+    }
 
     Status s;
     if(FLAGS_readonly) {

db/db_impl.cc

@@ -51,6 +51,7 @@
 #include "util/auto_roll_logger.h"
 #include "util/build_version.h"
 #include "util/coding.h"
+#include "util/hash_skiplist_rep.h"
 #include "util/logging.h"
 #include "util/mutexlock.h"
 #include "util/perf_context_imp.h"
@@ -163,10 +164,10 @@ Options SanitizeOptions(const std::string& dbname,
     Log(result.info_log, "Compaction filter specified, ignore factory");
   }
   if (result.prefix_extractor) {
-    // If a prefix extractor has been supplied and a PrefixHashRepFactory is
+    // If a prefix extractor has been supplied and a HashSkipListRepFactory is
     // being used, make sure that the latter uses the former as its transform
     // function.
-    auto factory = dynamic_cast<PrefixHashRepFactory*>(
+    auto factory = dynamic_cast<HashSkipListRepFactory*>(
       result.memtable_factory.get());
     if (factory &&
         factory->GetTransform() != result.prefix_extractor) {
@@ -236,7 +237,7 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname)
       mutex_(options.use_adaptive_mutex),
       shutting_down_(nullptr),
       bg_cv_(&mutex_),
-      mem_rep_factory_(options_.memtable_factory),
+      mem_rep_factory_(options_.memtable_factory.get()),
       mem_(new MemTable(internal_comparator_, mem_rep_factory_,
         NumberLevels(), options_)),
       logfile_number_(0),
@@ -516,6 +517,19 @@ void DBImpl::FindObsoleteFiles(DeletionState& deletion_state,
 // files in sst_delete_files and log_delete_files.
 // It is not necessary to hold the mutex when invoking this method.
 void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
+
+  // free pending memtables
+  for (auto m : state.memtables_to_free) {
+    delete m;
+  }
+
+  // check if there is anything to do
+  if (!state.all_files.size() &&
+      !state.sst_delete_files.size() &&
+      !state.log_delete_files.size()) {
+    return;
+  }
+
   // this checks if FindObsoleteFiles() was run before. If not, don't do
   // PurgeObsoleteFiles(). If FindObsoleteFiles() was run, we need to also
   // run PurgeObsoleteFiles(), even if disable_delete_obsolete_files_ is true
@@ -1170,7 +1184,7 @@ Status DBImpl::FlushMemTableToOutputFile(bool* madeProgress,
   // Replace immutable memtable with the generated Table
   s = imm_.InstallMemtableFlushResults(
     mems, versions_.get(), s, &mutex_, options_.info_log.get(),
-    file_number, pending_outputs_);
+    file_number, pending_outputs_, &deletion_state.memtables_to_free);
 
   if (s.ok()) {
     if (madeProgress) {
@@ -1656,7 +1670,7 @@ Status DBImpl::BackgroundFlush(bool* madeProgress,
 
 void DBImpl::BackgroundCallFlush() {
   bool madeProgress = false;
-  DeletionState deletion_state;
+  DeletionState deletion_state(options_.max_write_buffer_number);
   assert(bg_flush_scheduled_);
   MutexLock l(&mutex_);
 
@@ -1702,7 +1716,7 @@ void DBImpl::TEST_PurgeObsoleteteWAL() {
 
 void DBImpl::BackgroundCallCompaction() {
   bool madeProgress = false;
-  DeletionState deletion_state;
+  DeletionState deletion_state(options_.max_write_buffer_number);
 
   MaybeDumpStats();
 
@@ -1732,6 +1746,7 @@ void DBImpl::BackgroundCallCompaction() {
   // FindObsoleteFiles(). This is because deletion_state does not catch
   // all created files if compaction failed.
   FindObsoleteFiles(deletion_state, !s.ok());
+
   // delete unnecessary files if any, this is done outside the mutex
   if (deletion_state.HaveSomethingToDelete()) {
     mutex_.Unlock();
@@ -2492,25 +2507,20 @@ struct IterState {
 
 static void CleanupIteratorState(void* arg1, void* arg2) {
   IterState* state = reinterpret_cast<IterState*>(arg1);
-  std::vector<MemTable*> to_delete;
-  to_delete.reserve(state->mem.size());
+  DBImpl::DeletionState deletion_state(state->db->GetOptions().
+                                       max_write_buffer_number);
   state->mu->Lock();
   for (unsigned int i = 0; i < state->mem.size(); i++) {
     MemTable* m = state->mem[i]->Unref();
     if (m != nullptr) {
-      to_delete.push_back(m);
+      deletion_state.memtables_to_free.push_back(m);
     }
   }
   state->version->Unref();
-  // delete only the sst obsolete files
-  DBImpl::DeletionState deletion_state;
   // fast path FindObsoleteFiles
   state->db->FindObsoleteFiles(deletion_state, false, true);
   state->mu->Unlock();
   state->db->PurgeObsoleteFiles(deletion_state);
-
-  // delete obsolete memtables outside the db-mutex
-  for (MemTable* m : to_delete) delete m;
   delete state;
 }
 }  // namespace
@@ -2612,8 +2622,10 @@ Status DBImpl::GetImpl(const ReadOptions& options,
   BumpPerfTime(&perf_context.get_snapshot_time, &snapshot_timer);
   if (mem->Get(lkey, value, &s, merge_context, options_)) {
     // Done
+    RecordTick(options_.statistics.get(), MEMTABLE_HIT);
   } else if (imm.Get(lkey, value, &s, merge_context, options_)) {
     // Done
+    RecordTick(options_.statistics.get(), MEMTABLE_HIT);
   } else {
     StopWatchNano from_files_timer(env_, false);
     StartPerfTimer(&from_files_timer);
@@ -2622,6 +2634,7 @@ Status DBImpl::GetImpl(const ReadOptions& options,
                  options_, value_found);
     have_stat_update = true;
     BumpPerfTime(&perf_context.get_from_output_files_time, &from_files_timer);
+    RecordTick(options_.statistics.get(), MEMTABLE_MISS);
   }
 
   StopWatchNano post_process_timer(env_, false);
@@ -3514,6 +3527,33 @@ void DBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata) {
   return versions_->GetLiveFilesMetaData(metadata);
 }
 
+Status DBImpl::GetDbIdentity(std::string& identity) {
+  std::string idfilename = IdentityFileName(dbname_);
+  unique_ptr<SequentialFile> idfile;
+  const EnvOptions soptions;
+  Status s = env_->NewSequentialFile(idfilename, &idfile, soptions);
+  if (!s.ok()) {
+    return s;
+  }
+  uint64_t file_size;
+  s = env_->GetFileSize(idfilename, &file_size);
+  if (!s.ok()) {
+    return s;
+  }
+  char buffer[file_size];
+  Slice id;
+  s = idfile->Read(file_size, &id, buffer);
+  if (!s.ok()) {
+    return s;
+  }
+  identity.assign(id.ToString());
+  // If last character is '\n' remove it from identity
+  if (identity.size() > 0 && identity.back() == '\n') {
+    identity.pop_back();
+  }
+  return s;
+}
+
 // Default implementations of convenience methods that subclasses of DB
 // can call if they wish
 Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) {