Add KeyManagedEncryptedEnv and AESBlockCipher (#151)

Summary:
Introduce `KeyManagedEncryptedEnv` which wraps around `EncryptedEnv` but provides an `KeyManager` API to enable key management per file. Also implements `AESBlockCipher` with OpenSSL.

Test Plan:
not tested yet. will update.

Signed-off-by: Yi Wu <yiwu@pingcap.com>

Author: yiwu-arbug

.travis.yml

@@ -25,6 +25,7 @@ addons:
       - libsnappy-dev
       - mingw-w64
       - zlib1g-dev
+      - libssl-dev
 env:
   - TEST_GROUP=platform_dependent # 16-18 minutes
   - TEST_GROUP=1 # 33-35 minutes
@@ -36,6 +37,7 @@ env:
   - JOB_NAME=lite_build # 3-4 minutes
   # Build examples
   - JOB_NAME=examples # 5-7 minutes
+  - JOB_NAME=encrypted_env # 16-18 minutes
 
 matrix:
   exclude:
@@ -49,13 +51,15 @@ matrix:
     env: TEST_GROUP=4
   - os : linux
     compiler: clang
-  - os : osx
+  - os: osx
     compiler: gcc
+  - os: osx
+    env: JOB_NAME=encrypted_env
 
 # https://docs.travis-ci.com/user/caching/#ccache-cache
 install:
   - if [ "${TRAVIS_OS_NAME}" == osx ]; then
-      brew install ccache zstd lz4 snappy xz;
+      brew install ccache zstd lz4 snappy xz openssl;
       PATH=$PATH:/usr/local/opt/ccache/libexec;
     fi
 
@@ -91,6 +95,9 @@ script:
     examples)
       OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4
       ;;
+    encrypted_env)
+      OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test ENCRYPTED_ENV=1 make -j4 all_but_some_tests check_some
+      ;;
     esac
 notifications:
     email:

CMakeLists.txt

@@ -55,6 +55,7 @@ option(WITH_SNAPPY "build with SNAPPY" OFF)
 option(WITH_LZ4 "build with lz4" OFF)
 option(WITH_ZLIB "build with zlib" OFF)
 option(WITH_ZSTD "build with zstd" OFF)
+option(WITH_OPENSSL "build with openssl" OFF)
 option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files, regardles of the system code page" OFF)
 if (WITH_WINDOWS_UTF8_FILENAMES)
   add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES)
@@ -131,6 +132,14 @@ else()
     include_directories(${ZSTD_INCLUDE_DIR})
     list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARIES})
   endif()
+
+  if(WITH_OPENSSL)
+    find_package(OpenSSL REQUIRED)
+    add_definitions(-DOPENSSL)
+    include_directories(${OPENSSL_INCLUDE_DIR})
+    # Only the crypto library is needed.
+    list(APPEND THIRDPARTY_LIBS ${OPENSSL_CRYPTO_LIBRARIES})
+  endif()
 endif()
 
 string(TIMESTAMP GIT_DATE_TIME "%Y/%m/%d %H:%M:%S" UTC)
@@ -545,6 +554,7 @@ set(SOURCES
         db/write_batch_base.cc
         db/write_controller.cc
         db/write_thread.cc
+        encryption/encryption.cc
         env/env.cc
         env/env_chroot.cc
         env/env_encryption.cc

build_tools/build_detect_platform

@@ -406,6 +406,19 @@ EOF
         fi
     fi
 
+    if ! test $ROCKSDB_DISABLE_OPENSSL; then
+        # Test whether OpenSSL library is installed
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+          #include <openssl/crypto.h>
+          int main() {}
+EOF
+        if [ "$?" = 0 ]; then
+            COMMON_FLAGS="$COMMON_FLAGS -DOPENSSL"
+            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lcrypto"
+            JAVA_LDFLAGS="$JAVA_LDFLAGS -lcrypto"
+        fi
+    fi
+
     if ! test $ROCKSDB_DISABLE_PTHREAD_MUTEX_ADAPTIVE_NP; then
         # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
         $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF

db/db_basic_test.cc

@@ -938,7 +938,7 @@ TEST_F(DBBasicTest, DBClose) {
 
 TEST_F(DBBasicTest, DBCloseFlushError) {
   std::unique_ptr<FaultInjectionTestEnv> fault_injection_env(
-      new FaultInjectionTestEnv(Env::Default()));
+      new FaultInjectionTestEnv(env_));
   Options options = GetDefaultOptions();
   options.create_if_missing = true;
   options.manual_wal_flush = true;

db/db_options_test.cc

@@ -602,7 +602,7 @@ TEST_F(DBOptionsTest, MaxOpenFilesChange) {
 }
 
 TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) {
-  Options options;
+  Options options = CurrentOptions();
   options.delayed_write_rate = 0;
   Reopen(options);
   ASSERT_EQ(16 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate);
@@ -748,10 +748,11 @@ TEST_F(DBOptionsTest, CompactionReadaheadSizeChange) {
 }
 
 TEST_F(DBOptionsTest, FIFOTtlBackwardCompatible) {
-  Options options;
+  Options options = CurrentOptions();
   options.compaction_style = kCompactionStyleFIFO;
   options.write_buffer_size = 10 << 10;  // 10KB
   options.create_if_missing = true;
+  options.max_open_files = -1;
 
   ASSERT_OK(TryReopen(options));
 

db/db_properties_test.cc

@@ -1406,7 +1406,7 @@ TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) {
 }
 
 TEST_F(DBPropertiesTest, EstimateNumKeysUnderflow) {
-  Options options;
+  Options options = CurrentOptions();
   Reopen(options);
   Put("foo", "bar");
   Delete("foo");
@@ -1417,7 +1417,7 @@ TEST_F(DBPropertiesTest, EstimateNumKeysUnderflow) {
 }
 
 TEST_F(DBPropertiesTest, EstimateOldestKeyTime) {
-  std::unique_ptr<MockTimeEnv> mock_env(new MockTimeEnv(Env::Default()));
+  std::unique_ptr<MockTimeEnv> mock_env(new MockTimeEnv(env_));
   uint64_t oldest_key_time = 0;
   Options options;
   options.env = mock_env.get();
@@ -1517,7 +1517,7 @@ TEST_F(DBPropertiesTest, SstFilesSize) {
   };
   std::shared_ptr<TestListener> listener = std::make_shared<TestListener>();
 
-  Options options;
+  Options options = CurrentOptions();
   options.disable_auto_compactions = true;
   options.listeners.push_back(listener);
   Reopen(options);
@@ -1599,7 +1599,7 @@ TEST_F(DBPropertiesTest, MinObsoleteSstNumberToKeep) {
 }
 
 TEST_F(DBPropertiesTest, BlockCacheProperties) {
-  Options options;
+  Options options = CurrentOptions();
   uint64_t value;
 
   // Block cache properties are not available for tables other than

db/db_test2.cc

@@ -2145,6 +2145,9 @@ TEST_F(DBTest2, ReadAmpBitmap) {
 
 #ifndef OS_SOLARIS // GetUniqueIdFromFile is not implemented
 TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
+  if (getenv("ENCRYPTED_ENV")) {
+    return;
+  }
   {
     const int kIdBufLen = 100;
     char id_buf[kIdBufLen];
@@ -2706,7 +2709,7 @@ TEST_F(DBTest2, RateLimitedCompactionReads) {
 // Make sure DB can be reopen with reduced number of levels, given no file
 // is on levels higher than the new num_levels.
 TEST_F(DBTest2, ReduceLevel) {
-  Options options;
+  Options options = CurrentOptions();
   options.disable_auto_compactions = true;
   options.num_levels = 7;
   Reopen(options);
@@ -2732,7 +2735,7 @@ TEST_F(DBTest2, ReduceLevel) {
 
 // Test that ReadCallback is actually used in both memtbale and sst tables
 TEST_F(DBTest2, ReadCallbackTest) {
-  Options options;
+  Options options = CurrentOptions();
   options.disable_auto_compactions = true;
   options.num_levels = 7;
   Reopen(options);
@@ -2990,7 +2993,8 @@ TEST_F(DBTest2, TraceAndReplay) {
   column_families.push_back(
       ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions()));
   std::vector<ColumnFamilyHandle*> handles;
-  ASSERT_OK(DB::Open(DBOptions(), dbname2, column_families, &handles, &db2));
+  ASSERT_OK(
+      DB::Open(CurrentOptions(), dbname2, column_families, &handles, &db2));
 
   env_->SleepForMicroseconds(100);
   // Verify that the keys don't already exist
@@ -3065,7 +3069,8 @@ TEST_F(DBTest2, TraceWithLimit) {
   column_families.push_back(
       ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions()));
   std::vector<ColumnFamilyHandle*> handles;
-  ASSERT_OK(DB::Open(DBOptions(), dbname2, column_families, &handles, &db2));
+  ASSERT_OK(
+      DB::Open(CurrentOptions(), dbname2, column_families, &handles, &db2));
 
   env_->SleepForMicroseconds(100);
   // Verify that the keys don't already exist
@@ -3133,7 +3138,8 @@ TEST_F(DBTest2, TraceWithSampling) {
   column_families.push_back(
       ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions()));
   std::vector<ColumnFamilyHandle*> handles;
-  ASSERT_OK(DB::Open(DBOptions(), dbname2, column_families, &handles, &db2));
+  ASSERT_OK(
+      DB::Open(CurrentOptions(), dbname2, column_families, &handles, &db2));
 
   env_->SleepForMicroseconds(100);
   ASSERT_TRUE(db2->Get(ro, handles[0], "a", &value).IsNotFound());
@@ -3233,7 +3239,8 @@ TEST_F(DBTest2, TraceWithFilter) {
   column_families.push_back(
       ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions()));
   std::vector<ColumnFamilyHandle*> handles;
-  ASSERT_OK(DB::Open(DBOptions(), dbname2, column_families, &handles, &db2));
+  ASSERT_OK(
+      DB::Open(CurrentOptions(), dbname2, column_families, &handles, &db2));
 
   env_->SleepForMicroseconds(100);
   // Verify that the keys don't already exist
@@ -3279,7 +3286,8 @@ TEST_F(DBTest2, TraceWithFilter) {
   handles.clear();
 
   DB* db3 =  nullptr;
-  ASSERT_OK(DB::Open(DBOptions(), dbname3, column_families, &handles, &db3));
+  ASSERT_OK(
+      DB::Open(CurrentOptions(), dbname3, column_families, &handles, &db3));
 
   env_->SleepForMicroseconds(100);
   // Verify that the keys don't already exist
@@ -3334,6 +3342,9 @@ TEST_F(DBTest2, TraceWithFilter) {
 #endif  // ROCKSDB_LITE
 
 TEST_F(DBTest2, PinnableSliceAndMmapReads) {
+  if (getenv("ENCRYPTED_ENV")) {
+    return;
+  }
   Options options = CurrentOptions();
   options.allow_mmap_reads = true;
   options.max_open_files = 100;
@@ -3612,7 +3623,7 @@ TEST_F(DBTest2, TestCompactFiles) {
   });
   SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options;
+  Options options = CurrentOptions();
   options.num_levels = 2;
   options.disable_auto_compactions = true;
   Reopen(options);

db/db_test_util.cc

@@ -14,6 +14,14 @@
 
 namespace rocksdb {
 
+#ifdef OPENSSL
+const std::string TestKeyManager::default_key =
+    "\x12\x34\x56\x78\x12\x34\x56\x78\x12\x34\x56\x78\x12\x34\x56\x78\x12\x34"
+    "\x56\x78\x12\x34\x56\x78";
+const std::string TestKeyManager::default_iv =
+    "\xaa\xbb\xcc\xdd\xaa\xbb\xcc\xdd\xaa\xbb\xcc\xdd\xaa\xbb\xcc\xdd";
+#endif
+
 // Special Env used to delay background operations
 
 SpecialEnv::SpecialEnv(Env* base)
@@ -68,8 +76,13 @@ DBTestBase::DBTestBase(const std::string path)
   }
 #ifndef ROCKSDB_LITE
   if (getenv("ENCRYPTED_ENV")) {
-    encrypted_env_ = NewEncryptedEnv(mem_env_ ? mem_env_ : base_env,
-                                     new CTREncryptionProvider(rot13Cipher_));
+#ifdef OPENSSL
+    std::shared_ptr<encryption::KeyManager> key_manager(new TestKeyManager);
+    encrypted_env_ = NewKeyManagedEncryptedEnv(Env::Default(), key_manager);
+#else
+    fprintf(stderr, "EncryptedEnv is not available without OpenSSL.");
+    assert(false);
+#endif
   }
 #endif  // !ROCKSDB_LITE
   env_ = new SpecialEnv(encrypted_env_ ? encrypted_env_

db/db_test_util.h

@@ -52,8 +52,45 @@
 #include "util/string_util.h"
 #include "utilities/merge_operators.h"
 
+#include "rocksdb/encryption.h"
+
 namespace rocksdb {
 
+//TODO(yiwu): Use InMemoryKeyManager instead for tests.
+#ifdef OPENSSL
+class TestKeyManager : public encryption::KeyManager {
+ public:
+  virtual ~TestKeyManager() = default;
+
+  static const std::string default_key;
+  static const std::string default_iv;
+
+  Status GetFile(const std::string& /*fname*/,
+                 encryption::FileEncryptionInfo* file_info) override {
+    file_info->method = encryption::EncryptionMethod::kAES192_CTR;
+    file_info->key = default_key;
+    file_info->iv = default_iv;
+    return Status::OK();
+  }
+
+  Status NewFile(const std::string& /*fname*/,
+                 encryption::FileEncryptionInfo* file_info) override {
+    file_info->method = encryption::EncryptionMethod::kAES192_CTR;
+    file_info->key = default_key;
+    file_info->iv = default_iv;
+    return Status::OK();
+  }
+
+  Status DeleteFile(const std::string&) override { return Status::OK(); }
+  Status LinkFile(const std::string&, const std::string&) override {
+    return Status::OK();
+  }
+  Status RenameFile(const std::string&, const std::string&) override {
+    return Status::OK();
+  }
+};
+#endif
+
 namespace anon {
 class AtomicCounter {
  public:

db/db_wal_test.cc

@@ -948,6 +948,9 @@ class RecoveryTestHelper {
 // at the end of any of the logs
 // - We do not expect to open the data store for corruption
 TEST_F(DBWALTest, kTolerateCorruptedTailRecords) {
+  if (getenv("ENCRYPTED_ENV")) {
+    return;
+  }
   const int jstart = RecoveryTestHelper::kWALFileOffset;
   const int jend = jstart + RecoveryTestHelper::kWALFilesCount;
 
@@ -1019,6 +1022,9 @@ TEST_F(DBWALTest, kAbsoluteConsistency) {
 // We don't expect the data store to be opened if there is any inconsistency
 // between WAL and SST files
 TEST_F(DBWALTest, kPointInTimeRecoveryCFConsistency) {
+  if (getenv("ENCRYPTED_ENV")) {
+    return;
+  }
   Options options = CurrentOptions();
   options.avoid_flush_during_recovery = true;
 
@@ -1052,6 +1058,9 @@ TEST_F(DBWALTest, kPointInTimeRecoveryCFConsistency) {
 // - We expect to open data store under all circumstances
 // - We expect only data upto the point where the first error was encountered
 TEST_F(DBWALTest, kPointInTimeRecovery) {
+  if (getenv("ENCRYPTED_ENV")) {
+    return;
+  }
   const int jstart = RecoveryTestHelper::kWALFileOffset;
   const int jend = jstart + RecoveryTestHelper::kWALFilesCount;
   const int maxkeys =
@@ -1103,6 +1112,9 @@ TEST_F(DBWALTest, kPointInTimeRecovery) {
 // - We expect to open the data store under all scenarios
 // - We expect to have recovered records past the corruption zone
 TEST_F(DBWALTest, kSkipAnyCorruptedRecords) {
+  if (getenv("ENCRYPTED_ENV")) {
+    return;
+  }
   const int jstart = RecoveryTestHelper::kWALFileOffset;
   const int jend = jstart + RecoveryTestHelper::kWALFilesCount;
 
@@ -1310,6 +1322,9 @@ TEST_F(DBWALTest, RecoverWithoutFlushMultipleCF) {
 //   3. Append more data without flushing, which creates new WAL log.
 //   4. Open again. See if it can correctly handle previous corruption.
 TEST_F(DBWALTest, RecoverFromCorruptedWALWithoutFlush) {
+  if (getenv("ENCRYPTED_ENV")) {
+    return;
+  }
   const int jstart = RecoveryTestHelper::kWALFileOffset;
   const int jend = jstart + RecoveryTestHelper::kWALFilesCount;
   const int kAppendKeys = 100;