Merge branch 'master' into performance

Conflicts:
	db/table_cache.cc

Author: liukai

.clang-format

@@ -0,0 +1,47 @@
+# Complete list of style options can be found at: 
+# http://clang.llvm.org/docs/ClangFormatStyleOptions.html
+---
+BasedOnStyle: Google
+AccessModifierOffset: -1
+ConstructorInitializerIndentWidth: 4
+AlignEscapedNewlinesLeft: true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakTemplateDeclarations: true
+AlwaysBreakBeforeMultilineStrings: true
+BreakBeforeBinaryOperators: false
+BreakConstructorInitializersBeforeComma: false
+BinPackParameters: false
+ColumnLimit:     80
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+DerivePointerBinding: true
+ExperimentalAutoDetectBinPacking: true
+IndentCaseLabels: false
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 10
+PenaltyBreakComment: 60
+PenaltyBreakString: 1000
+PenaltyBreakFirstLessLess: 20
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerBindsToType: true
+SpacesBeforeTrailingComments: 2
+Cpp11BracedListStyle: true
+Standard:        Cpp11
+IndentWidth:     2
+TabWidth:        8
+UseTab:          Never
+BreakBeforeBraces: Attach
+IndentFunctionDeclarationAfterType: false
+SpacesInParentheses: false
+SpacesInAngles: false
+SpaceInEmptyParentheses: false
+SpacesInCStyleCastParentheses: false
+SpaceAfterControlStatementKeyword: true
+SpaceBeforeAssignmentOperators: true
+ContinuationIndentWidth: 4
+...

INSTALL.md

@@ -44,5 +44,7 @@ libraries. You are on your own.
 `make clean; make` will compile librocksdb.a (RocskDB static library) and all
 the unit tests. You can run all unit tests with `make check`.
 
+For shared library builds, exec `make librocksdb.so` instead.
+
 If you followed the above steps and your compile or unit tests fail,
 please submit an issue: (https://github.com/facebook/rocksdb/issues)

Makefile

@@ -45,8 +45,8 @@ VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)
 VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full
 
 TESTS = \
-	autovector_test \
 	db_test \
+	autovector_test \
 	table_properties_collector_test \
 	arena_test \
 	auto_roll_logger_test \

README.fb

@@ -1,3 +1,3 @@
 * Detailed instructions on how to compile using fbcode and jemalloc
 
-* Latest release is 2.5.fb
+* Latest release is 2.7.fb

build_tools/build_detect_platform

@@ -198,7 +198,7 @@ EOF
       }
 EOF
     if [ "$?" = 0 ]; then
-        COMMON_FLAGS="$PLATFORM_LDFLAGS -DROCKSDB_FALLOCATE_PRESENT"
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT"
     fi
 
     # Test whether Snappy library is installed

build_tools/build_detect_version

@@ -8,7 +8,7 @@
 #
 
 # create git version file
-VFILE=$ROCKSDB_ROOT/util/build_version.cc.tmp
+VFILE=$PWD/util/build_version.cc.tmp
 trap "rm $VFILE" EXIT
 
 # check to see if git is in the path
@@ -36,7 +36,7 @@ echo "const char* rocksdb_build_git_datetime = \"rocksdb_build_git_datetime:$(da
 echo "const char* rocksdb_build_compile_date = __DATE__;" >> ${VFILE}
 echo "const char* rocksdb_build_compile_time = __TIME__;" >> ${VFILE}
 
-OUTFILE=$ROCKSDB_ROOT/util/build_version.cc
+OUTFILE=$PWD/util/build_version.cc
 if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then
     cp $VFILE $OUTFILE
 fi

build_tools/fbcode.gcc471.sh

@@ -54,7 +54,7 @@ RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
 CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic"
 CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
 CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT"
 CFLAGS+=" -DSNAPPY -DGFLAGS -DZLIB -DBZIP2"
 
 EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"

build_tools/fbcode.gcc481.sh

@@ -61,7 +61,7 @@ RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
 
 CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic"
 CFLAGS+=" -nostdlib $LIBGCC_INCLUDE $GLIBC_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT"
 CFLAGS+=" -DSNAPPY -DGFLAGS -DZLIB -DBZIP2"
 
 EXEC_LDFLAGS="-Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"

build_tools/make_new_version.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+#  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+#  This source code is licensed under the BSD-style license found in the
+#  LICENSE file in the root directory of this source tree. An additional grant
+#  of patent rights can be found in the PATENTS file in the same directory.
+
+set -e
+# Print out the colored progress info so that it can be brainlessly 
+# distinguished by users.
+function title() {
+  echo -e "\033[1;32m$*\033[0m"
+}
+
+usage="Create new rocksdb version and prepare it for the release process\n"
+usage+="USAGE: ./make_new_version.sh <version>"
+
+# -- Pre-check
+if [[ $# < 1 ]]; then
+  echo -e $usage
+  exit 1
+fi
+
+ROCKSDB_VERSION=$1
+
+GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
+if [ $GIT_BRANCH != "master" ]; then
+  echo "Error: Current branch is '$GIT_BRANCH', Please switch to master branch."
+fi
+
+# --Step 1: cutting new tag
+title "Adding new tag for this release ..."
+git tag -a "$ROCKSDB_VERSION.fb" -m "Rocksdb $ROCKSDB_VERSION"
+
+# Setting up the proxy for remote repo access
+export http_proxy=http://172.31.255.99:8080
+export https_proxy="$http_proxy";
+
+title "Pushing new tag to remote repo ..."
+proxycmd.sh git push origin --tags
+
+# --Step 2: Update README.fb
+title "Updating the latest version info in README.fb ..."
+sed -i "s/Latest release is [0-9]\+.[0-9]\+.fb/Latest release is $ROCKSDB_VERSION.fb/" README.fb
+git commit README.fb -m "update the latest version in README.fb to $ROCKSDB_VERSION"
+proxycmd.sh git push
+
+# --Step 3: Prepare this repo for 3rd release
+title "Cleaning up repo ..."
+make clean
+git clean -fxd
+
+title "Generating the build info ..."
+# Comment out the call of `build_detection_version` so that the SHA number and build date of this
+# release will remain constant. Otherwise everytime we run "make" util/build_version.cc will be 
+# overridden.
+sed -i 's/^\$PWD\/build_tools\/build_detect_version$//' build_tools/build_detect_platform
+
+# Generate util/build_version.cc
+build_tools/build_detect_version
+
+title "Done!"

db/db_filesnapshot.cc

@@ -22,20 +22,34 @@ namespace rocksdb {
 
 Status DBImpl::DisableFileDeletions() {
   MutexLock l(&mutex_);
-  disable_delete_obsolete_files_ = true;
-  Log(options_.info_log, "File Deletions Disabled");
+  ++disable_delete_obsolete_files_;
+  if (disable_delete_obsolete_files_ == 1) {
+    // if not, it has already been disabled, so don't log anything
+    Log(options_.info_log, "File Deletions Disabled");
+  }
   return Status::OK();
 }
 
-Status DBImpl::EnableFileDeletions() {
+Status DBImpl::EnableFileDeletions(bool force) {
   DeletionState deletion_state;
+  bool should_purge_files = false;
   {
     MutexLock l(&mutex_);
-    disable_delete_obsolete_files_ = false;
-    Log(options_.info_log, "File Deletions Enabled");
-    FindObsoleteFiles(deletion_state, true);
+    if (force) {
+      // if force, we need to enable file deletions right away
+      disable_delete_obsolete_files_ = 0;
+    } else if (disable_delete_obsolete_files_ > 0) {
+      --disable_delete_obsolete_files_;
+    }
+    if (disable_delete_obsolete_files_ == 0)  {
+      Log(options_.info_log, "File Deletions Enabled");
+      should_purge_files = true;
+      FindObsoleteFiles(deletion_state, true);
+    }
+  }
+  if (should_purge_files)  {
+    PurgeObsoleteFiles(deletion_state);
   }
-  PurgeObsoleteFiles(deletion_state);
   LogFlush(options_.info_log);
   return Status::OK();
 }

db/db_impl.cc

@@ -247,7 +247,7 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname)
       bg_logstats_scheduled_(false),
       manual_compaction_(nullptr),
       logger_(nullptr),
-      disable_delete_obsolete_files_(false),
+      disable_delete_obsolete_files_(0),
       delete_obsolete_files_last_run_(options.env->NowMicros()),
       purge_wal_files_last_run_(0),
       last_stats_dump_time_microsec_(0),
@@ -509,7 +509,7 @@ void DBImpl::FindObsoleteFiles(DeletionState& deletion_state,
   mutex_.AssertHeld();
 
   // if deletion is disabled, do nothing
-  if (disable_delete_obsolete_files_) {
+  if (disable_delete_obsolete_files_ > 0) {
     return;
   }
 
@@ -1262,7 +1262,7 @@ Status DBImpl::FlushMemTableToOutputFile(bool* madeProgress,
 
     MaybeScheduleLogDBDeployStats();
 
-    if (!disable_delete_obsolete_files_) {
+    if (disable_delete_obsolete_files_ == 0) {
       // add to deletion state
       deletion_state.log_delete_files.insert(
           deletion_state.log_delete_files.end(),

db/db_impl.h

@@ -73,7 +73,7 @@ class DBImpl : public DB {
   virtual const Options& GetOptions() const;
   virtual Status Flush(const FlushOptions& options);
   virtual Status DisableFileDeletions();
-  virtual Status EnableFileDeletions();
+  virtual Status EnableFileDeletions(bool force);
   // All the returned filenames start with "/"
   virtual Status GetLiveFiles(std::vector<std::string>&,
                               uint64_t* manifest_file_size,
@@ -417,7 +417,12 @@ class DBImpl : public DB {
   int64_t volatile last_log_ts;
 
   // shall we disable deletion of obsolete files
-  bool disable_delete_obsolete_files_;
+  // if 0 the deletion is enabled.
+  // if non-zero, files will not be getting deleted
+  // This enables two different threads to call
+  // EnableFileDeletions() and DisableFileDeletions()
+  // without any synchronization
+  int disable_delete_obsolete_files_;
 
   // last time when DeleteObsoleteFiles was invoked
   uint64_t delete_obsolete_files_last_run_;

db/db_impl_readonly.h

@@ -55,7 +55,7 @@ class DBImplReadOnly : public DBImpl {
  virtual Status DisableFileDeletions() {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
- virtual Status EnableFileDeletions() {
+ virtual Status EnableFileDeletions(bool force) {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
  virtual Status GetLiveFiles(std::vector<std::string>&,

db/db_test.cc

@@ -4381,7 +4381,7 @@ class ModelDB: public DB {
   virtual Status DisableFileDeletions() {
     return Status::OK();
   }
-  virtual Status EnableFileDeletions() {
+  virtual Status EnableFileDeletions(bool force) {
     return Status::OK();
   }
   virtual Status GetLiveFiles(std::vector<std::string>&, uint64_t* size,

db/table_cache.cc

@@ -29,6 +29,11 @@ static void UnrefEntry(void* arg1, void* arg2) {
   cache->Release(h);
 }
 
+static Slice GetSliceForFileNumber(uint64_t* file_number) {
+  return Slice(reinterpret_cast<const char*>(file_number),
+               sizeof(*file_number));
+}
+
 TableCache::TableCache(const std::string& dbname,
                        const Options* options,
                        const EnvOptions& storage_options,
@@ -50,8 +55,7 @@ Status TableCache::FindTable(const EnvOptions& toptions,
                              Cache::Handle** handle, bool* table_io,
                              const bool no_io) {
   Status s;
-  Slice key(reinterpret_cast<const char*>(&file_number), sizeof(file_number));
-
+  Slice key = GetSliceForFileNumber(&file_number);
   *handle = cache_->Lookup(key);
   if (*handle == nullptr) {
     if (no_io) { // Dont do IO and return a not-found status
@@ -164,8 +168,7 @@ bool TableCache::PrefixMayMatch(const ReadOptions& options,
 }
 
 void TableCache::Evict(uint64_t file_number) {
-  Slice key(reinterpret_cast<const char*>(&file_number), sizeof(file_number));
-  cache_->Erase(key);
+  cache_->Erase(GetSliceForFileNumber(&file_number));
 }
 
 }  // namespace rocksdb

include/rocksdb/db.h

@@ -247,7 +247,15 @@ class DB {
   virtual Status DisableFileDeletions() = 0;
 
   // Allow compactions to delete obselete files.
-  virtual Status EnableFileDeletions() = 0;
+  // If force == true, the call to EnableFileDeletions() will guarantee that
+  // file deletions are enabled after the call, even if DisableFileDeletions()
+  // was called multiple times before.
+  // If force == false, EnableFileDeletions will only enable file deletion
+  // after it's been called at least as many times as DisableFileDeletions(),
+  // enabling the two methods to be called by two threads concurrently without
+  // synchronization -- i.e., file deletions will be enabled only after both
+  // threads call EnableFileDeletions()
+  virtual Status EnableFileDeletions(bool force = true) = 0;
 
   // GetLiveFiles followed by GetSortedWalFiles can generate a lossless backup
 

include/utilities/stackable_db.h

@@ -123,8 +123,8 @@ class StackableDB : public DB {
     return db_->DisableFileDeletions();
   }
 
-  virtual Status EnableFileDeletions() override {
-    return db_->EnableFileDeletions();
+  virtual Status EnableFileDeletions(bool force) override {
+    return db_->EnableFileDeletions(force);
   }
 
   virtual Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs,

utilities/backupable/backupable_db_test.cc

@@ -48,7 +48,7 @@ class DummyDB : public StackableDB {
     return options_;
   }
 
-  virtual Status EnableFileDeletions() override {
+  virtual Status EnableFileDeletions(bool force) override {
     ASSERT_TRUE(!deletions_enabled_);
     deletions_enabled_ = true;
     return Status::OK();