Add KeyManagedEncryptedEnv and AESBlockCipher (#4)

apache/incubator-pegasus#1575

Cherry-pick from
tikv@113b363

Summary:
Introduce `KeyManagedEncryptedEnv` which wraps around `EncryptedEnv` but
provides an `KeyManager` API to enable key management per file. Also
implements `AESBlockCipher` with OpenSSL.

Test Plan:
not tested yet. will update.

Signed-off-by: Yi Wu <yiwu@pingcap.com>
Signed-off-by: tabokie <xy.tao@outlook.com>

Author: acelyc111

.github/workflows/jobs-linux-run-tests.yml

@@ -78,7 +78,7 @@ jobs:
     - uses: "./.github/actions/pre-steps"
     - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j5 && ctest -j5
     - uses: "./.github/actions/post-steps"
-  build-linux-encrypted_env-no_compression:
+  build-linux-encrypted_env-no_compression-no_openssl:
     runs-on: ubuntu-latest
     container:
       image: zjay437/rocksdb:0.6
@@ -88,3 +88,12 @@ jobs:
     - run: mkdir build && cd build && cmake -DWITH_SNAPPY=0 -DWITH_ZLIB=0 -DWITH_BZ2=0 -DWITH_LZ4=0 -DWITH_ZSTD=0 .. && make V=1 -j5 && ctest -j5 -V
     - run: "cd build/tools && ./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression'"
     - uses: "./.github/actions/post-steps"
+  build-linux-encrypted_env-openssl:
+    runs-on: ubuntu-latest
+    container:
+      image: zjay437/rocksdb:0.6
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: "./.github/actions/pre-steps"
+    - run: mkdir build && cd build && cmake -DWITH_OPENSSL=1 -DENCRYPTED_ENV=1 .. && make V=1 -j5 && ctest -j5 -V
+    - uses: "./.github/actions/post-steps"

CMakeLists.txt

@@ -70,6 +70,7 @@ option(WITH_SNAPPY "build with SNAPPY" OFF)
 option(WITH_LZ4 "build with lz4" OFF)
 option(WITH_ZLIB "build with zlib" OFF)
 option(WITH_ZSTD "build with zstd" OFF)
+option(WITH_OPENSSL "build with openssl" OFF)
 option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files, regardles of the system code page" OFF)
 if (WITH_WINDOWS_UTF8_FILENAMES)
   add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES)
@@ -174,6 +175,14 @@ else()
     include_directories(${ZSTD_INCLUDE_DIR})
     list(APPEND THIRDPARTY_LIBS zstd::zstd)
   endif()
+
+  if(WITH_OPENSSL)
+    find_package(OpenSSL REQUIRED)
+    add_definitions(-DOPENSSL)
+    include_directories(${OPENSSL_INCLUDE_DIR})
+    # Only the crypto library is needed.
+    list(APPEND THIRDPARTY_LIBS ${OPENSSL_CRYPTO_LIBRARIES})
+  endif()
 endif()
 
 option(WITH_MD_LIBRARY "build with MD" ON)
@@ -736,6 +745,7 @@ set(SOURCES
         db/write_controller.cc
         db/write_stall_stats.cc
         db/write_thread.cc
+        encryption/encryption.cc
         env/composite_env.cc
         env/env.cc
         env/env_chroot.cc

TARGETS

@@ -108,6 +108,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "db/write_controller.cc",
         "db/write_stall_stats.cc",
         "db/write_thread.cc",
+        "encryption/encryption.cc",
         "env/composite_env.cc",
         "env/env.cc",
         "env/env_chroot.cc",

build_tools/build_detect_platform

@@ -485,6 +485,19 @@ EOF
         fi
     fi
 
+    if ! test $ROCKSDB_DISABLE_OPENSSL; then
+        # Test whether OpenSSL library is installed
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+          #include <openssl/crypto.h>
+          int main() {}
+EOF
+        if [ "$?" = 0 ]; then
+            COMMON_FLAGS="$COMMON_FLAGS -DOPENSSL"
+            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lcrypto"
+            JAVA_LDFLAGS="$JAVA_LDFLAGS -lcrypto"
+        fi
+    fi
+
     if ! test $ROCKSDB_DISABLE_PTHREAD_MUTEX_ADAPTIVE_NP; then
         # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
         $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null  <<EOF

db/db_test_util.cc

@@ -31,6 +31,14 @@ int64_t MaybeCurrentTime(Env* env) {
 }
 }  // anonymous namespace
 
+#ifdef OPENSSL
+const std::string TestKeyManager::default_key =
+    "\x12\x34\x56\x78\x12\x34\x56\x78\x12\x34\x56\x78\x12\x34\x56\x78\x12\x34"
+    "\x56\x78\x12\x34\x56\x78";
+const std::string TestKeyManager::default_iv =
+    "\xaa\xbb\xcc\xdd\xaa\xbb\xcc\xdd\xaa\xbb\xcc\xdd\xaa\xbb\xcc\xdd";
+#endif
+
 // Special Env used to delay background operations
 
 SpecialEnv::SpecialEnv(Env* base, bool time_elapse_only_sleep)
@@ -71,15 +79,13 @@ DBTestBase::DBTestBase(const std::string path, bool env_do_fsync)
     mem_env_ = MockEnv::Create(base_env, base_env->GetSystemClock());
   }
   if (getenv("ENCRYPTED_ENV")) {
-    std::shared_ptr<EncryptionProvider> provider;
-    std::string provider_id = getenv("ENCRYPTED_ENV");
-    if (provider_id.find("=") == std::string::npos &&
-        !EndsWith(provider_id, "://test")) {
-      provider_id = provider_id + "://test";
-    }
-    EXPECT_OK(EncryptionProvider::CreateFromString(ConfigOptions(), provider_id,
-                                                   &provider));
-    encrypted_env_ = NewEncryptedEnv(mem_env_ ? mem_env_ : base_env, provider);
+#ifdef OPENSSL
+    std::shared_ptr<encryption::KeyManager> key_manager(new TestKeyManager);
+    encrypted_env_ = NewKeyManagedEncryptedEnv(Env::Default(), key_manager);
+#else
+    fprintf(stderr, "EncryptedEnv is not available without OpenSSL.");
+    assert(false);
+#endif
   }
   env_ = new SpecialEnv(encrypted_env_ ? encrypted_env_
                                        : (mem_env_ ? mem_env_ : base_env));

db/db_test_util.h

@@ -29,6 +29,7 @@
 #include "rocksdb/compaction_filter.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
+#include "rocksdb/encryption.h"
 #include "rocksdb/env.h"
 #include "rocksdb/file_system.h"
 #include "rocksdb/filter_policy.h"
@@ -55,6 +56,41 @@
 namespace ROCKSDB_NAMESPACE {
 class MockEnv;
 
+// TODO(yiwu): Use InMemoryKeyManager instead for tests.
+#ifdef OPENSSL
+class TestKeyManager : public encryption::KeyManager {
+ public:
+  virtual ~TestKeyManager() = default;
+
+  static const std::string default_key;
+  static const std::string default_iv;
+
+  Status GetFile(const std::string& /*fname*/,
+                 encryption::FileEncryptionInfo* file_info) override {
+    file_info->method = encryption::EncryptionMethod::kAES192_CTR;
+    file_info->key = default_key;
+    file_info->iv = default_iv;
+    return Status::OK();
+  }
+
+  Status NewFile(const std::string& /*fname*/,
+                 encryption::FileEncryptionInfo* file_info) override {
+    file_info->method = encryption::EncryptionMethod::kAES192_CTR;
+    file_info->key = default_key;
+    file_info->iv = default_iv;
+    return Status::OK();
+  }
+
+  Status DeleteFile(const std::string&) override { return Status::OK(); }
+  Status LinkFile(const std::string&, const std::string&) override {
+    return Status::OK();
+  }
+  Status RenameFile(const std::string&, const std::string&) override {
+    return Status::OK();
+  }
+};
+#endif
+
 namespace anon {
 class AtomicCounter {
  public: