feat: implements kms v2beta1 APIs (#210)

Signed-off-by: Nilekh Chaudhari <1626598+nilekhc@users.noreply.github.com>

Author: nilekhc

.pipelines/templates/e2e-kind-template.yml

@@ -5,34 +5,34 @@ jobs:
     workspace:
       clean: all
     variables:
-      - name: REGISTRY_NAME
-        value: kind-registry
-      - name: REGISTRY_PORT
-        value: 5000
-      - name: KUBERNETES_VERSION
-        value: v1.23.5
-      - name: KIND_CLUSTER_NAME
-        value: kms
-      - name: KIND_NETWORK
-        value: kind
-      # contains the following environment variables:
-      # - AZURE_CLIENT_ID
-      # - AZURE_CLIENT_SECRET
-      # - AZURE_TENANT_ID
-      # - KEYVAULT_NAME
-      # - KEY_NAME
-      # - KEY_VERSION
-      - group: kubernetes-kms
+    - name: REGISTRY_NAME
+      value: kind-registry
+    - name: REGISTRY_PORT
+      value: 5000
+    - name: KUBERNETES_VERSION
+      value: v1.27.1
+    - name: KIND_CLUSTER_NAME
+      value: kms
+    - name: KIND_NETWORK
+      value: kind
+    # contains the following environment variables:
+    # - AZURE_CLIENT_ID
+    # - AZURE_CLIENT_SECRET
+    # - AZURE_TENANT_ID
+    # - KEYVAULT_NAME
+    # - KEY_NAME
+    # - KEY_VERSION
+    - group: kubernetes-kms
     strategy:
       matrix:
-        kind_v1_23_13:
-          KUBERNETES_VERSION: v1.23.13
-        kind_v1_24_7:
+        kmsv1_kind_v1_24_7:
           KUBERNETES_VERSION: v1.24.7
-        kind_v1_25_3:
+        kmsv1_kind_v1_25_3:
           KUBERNETES_VERSION: v1.25.3
-        kind_v1_26_0:
+        kmsv1_kind_v1_26_0:
           KUBERNETES_VERSION: v1.26.0
+        kmsv1_kind_v1_27_1:
+          KUBERNETES_VERSION: v1.27.1
     steps:
       - task: GoTool@0
         inputs:
@@ -53,5 +53,55 @@ jobs:
       - template: cluster-health-template.yml
       - template: kind-debug-template.yml
       - script: make e2e-test
-        displayName: "Run e2e tests"
+        displayName: "Run e2e tests for KMS v1"
+      - template: cleanup-template.yml
+  - job:
+    timeoutInMinutes: 15
+    cancelTimeoutInMinutes: 5
+    workspace:
+      clean: all
+    variables:
+    - name: REGISTRY_NAME
+      value: kind-registry
+    - name: REGISTRY_PORT
+      value: 5000
+    - name: KUBERNETES_VERSION
+      value: v1.27.1
+    - name: KIND_CLUSTER_NAME
+      value: kms
+    - name: KIND_NETWORK
+      value: kind
+    # contains the following environment variables:
+    # - AZURE_CLIENT_ID
+    # - AZURE_CLIENT_SECRET
+    # - AZURE_TENANT_ID
+    # - KEYVAULT_NAME
+    # - KEY_NAME
+    # - KEY_VERSION
+    - group: kubernetes-kms
+    strategy:
+      matrix:
+        kmsv2_kind_v1_27_1:
+          KUBERNETES_VERSION: v1.27.1
+    steps:
+      - task: GoTool@0
+        inputs:
+          version: 1.19
+      - script: make e2e-install-prerequisites
+        displayName: "Install e2e test prerequisites"
+      - script: |
+          make e2e-kmsv2-setup-kind
+        displayName: "Setup kind cluster with azure kms plugin"
+        env:
+          REGISTRY_NAME: $(REGISTRY_NAME)
+          REGISTRY_PORT: $(REGISTRY_PORT)
+          KUBERNETES_VERSION: $(KUBERNETES_VERSION)
+          KIND_CLUSTER_NAME: $(KIND_CLUSTER_NAME)
+          KIND_NETWORK: $(KIND_NETWORK)
+          AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
+          AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
+      - template: cluster-health-template.yml
+      - template: kind-debug-template.yml
+      - script: make e2e-kmsv2-test
+        displayName: "Run e2e tests for KMS v2"
       - template: cleanup-template.yml

Makefile

@@ -29,8 +29,8 @@ DOCKER_BUILDKIT = 1
 export DOCKER_BUILDKIT
 
 # Testing var
-KIND_VERSION ?= 0.15.0
-KUBERNETES_VERSION ?= v1.25.0
+KIND_VERSION ?= 0.18.0
+KUBERNETES_VERSION ?= v1.27.1
 BATS_VERSION ?= 1.4.1
 
 ## --------------------------------------
@@ -132,6 +132,11 @@ e2e-setup-kind: setup-local-registry
 	./scripts/connect-registry.sh &
 	sleep 90s
 
+e2e-kmsv2-setup-kind: setup-local-registry
+	./scripts/setup-kmsv2-kind-cluster.sh &
+	./scripts/connect-registry.sh &
+	sleep 90s
+
 .PHONY: setup-local-registry
 setup-local-registry:
 	./scripts/setup-local-registry.sh
@@ -148,3 +153,7 @@ e2e-delete-kind:
 e2e-test:
 	# Run test suite with kind cluster
 	bats -t tests/e2e/test.bats
+
+e2e-kmsv2-test:
+	# Run test suite with kind cluster
+	bats -t tests/e2e/testkmsv2.bats

README.md

@@ -55,7 +55,7 @@ Now that Azure KMS provider is running in your cluster and the encryption config
    sudo ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/certs/ca.crt --cert=/etc/kubernetes/certs/etcdclient.crt --key=/etc/kubernetes/certs/etcdclient.key get /registry/secrets/default/secret1
    ```
 
-3. Check that the stored secret is prefixed with `k8s:enc:kms:v1:azurekmsprovider`. This indicates the Azure KMS provider has encrypted the data.
+3. Check that the stored secret is prefixed with `k8s:enc:kms:v1:azurekmsprovider` when KMSv1 is used for encryption, or with `k8s:enc:kms:v2:azurekmsprovider` when KMSv2 is used. This prefix indicates that the data has been encrypted by the Azure KMS provider.
 
 4. Verify the secret is decrypted correctly when retrieved via the Kubernetes API:
 
@@ -69,6 +69,9 @@ Now that Azure KMS provider is running in your cluster and the encryption config
 
 Refer to [doc](docs/rotation.md) for steps to rotate the KMS Key on an existing cluster.
 
+## Metrics
+Refer to [doc](docs/metrics.md) for details on the metrics exposed by the KMS Key Vault plugin.
+
 ## Contributing
 
 The KMS Plugin for Key Vault project welcomes contributions and suggestions. Please see [CONTRIBUTING](CONTRIBUTING.md) for details.

cmd/server/main.go

@@ -16,16 +16,18 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/Azure/kubernetes-kms/pkg/config"
 	"github.com/Azure/kubernetes-kms/pkg/metrics"
 	"github.com/Azure/kubernetes-kms/pkg/plugin"
 	"github.com/Azure/kubernetes-kms/pkg/utils"
 	"github.com/Azure/kubernetes-kms/pkg/version"
 
 	"google.golang.org/grpc"
-	pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
 	logsapi "k8s.io/component-base/logs/api/v1"
 	json "k8s.io/component-base/logs/json"
 	"k8s.io/klog/v2"
+	kmsv1 "k8s.io/kms/apis/v1beta1"
+	kmsv2 "k8s.io/kms/apis/v2"
 )
 
 var (
@@ -57,7 +59,12 @@ func main() {
 
 	if *logFormatJSON {
 		jsonFactory := json.Factory{}
-		logger, _ := jsonFactory.Create(logsapi.LoggingConfiguration{Format: "json"})
+		logger, _ := jsonFactory.Create(
+			logsapi.LoggingConfiguration{
+				Format: "json",
+			},
+			logsapi.LoggingOptions{},
+		)
 		klog.SetLogger(logger)
 	}
 
@@ -80,7 +87,7 @@ func main() {
 
 	klog.InfoS("Starting KeyManagementServiceServer service", "version", version.BuildVersion, "buildDate", version.BuildDate)
 
-	pc := &plugin.Config{
+	pluginConfig := &plugin.Config{
 		KeyVaultName:   *keyvaultName,
 		KeyName:        *keyName,
 		KeyVersion:     *keyVersion,
@@ -90,9 +97,25 @@ func main() {
 		ProxyPort:      *proxyPort,
 		ConfigFilePath: *configFilePath,
 	}
-	kmsServer, err := plugin.New(pc)
+
+	azureConfig, err := config.GetAzureConfig(pluginConfig.ConfigFilePath)
 	if err != nil {
-		klog.ErrorS(err, "failed to create server")
+		klog.ErrorS(err, "failed to get azure config")
+		os.Exit(1)
+	}
+
+	kvClient, err := plugin.NewKeyVaultClient(
+		azureConfig,
+		pluginConfig.KeyVaultName,
+		pluginConfig.KeyName,
+		pluginConfig.KeyVersion,
+		pluginConfig.ProxyMode,
+		pluginConfig.ProxyAddress,
+		pluginConfig.ProxyPort,
+		pluginConfig.ManagedHSM,
+	)
+	if err != nil {
+		klog.ErrorS(err, "failed to create key vault client")
 		os.Exit(1)
 	}
 
@@ -112,23 +135,41 @@ func main() {
 		klog.ErrorS(err, "failed to listen", "addr", addr, "proto", proto)
 		os.Exit(1)
 	}
+
 	opts := []grpc.ServerOption{
 		grpc.UnaryInterceptor(utils.UnaryServerInterceptor),
 	}
 
 	s := grpc.NewServer(opts...)
-	pb.RegisterKeyManagementServiceServer(s, kmsServer)
+
+	// register kms v1 server
+	kmsV1Server, err := plugin.NewKMSv1Server(kvClient)
+	if err != nil {
+		klog.ErrorS(err, "failed to create server")
+		os.Exit(1)
+	}
+	kmsv1.RegisterKeyManagementServiceServer(s, kmsV1Server)
+
+	// register kms v2 server
+	kmsV2Server, err := plugin.NewKMSv2Server(kvClient)
+	if err != nil {
+		klog.ErrorS(err, "failed to create kms V2 server")
+		os.Exit(1)
+	}
+	kmsv2.RegisterKeyManagementServiceServer(s, kmsV2Server)
 
 	klog.InfoS("Listening for connections", "addr", listener.Addr().String())
 	go func() {
 		if err := s.Serve(listener); err != nil {
-			klog.ErrorS(err, "failed to serve")
+			klog.ErrorS(err, "failed to serve kms server")
 			os.Exit(1)
 		}
 	}()
 
+	// Health check for kms v1 and v2
 	healthz := &plugin.HealthZ{
-		KMSServer: kmsServer,
+		KMSv1Server: kmsV1Server,
+		KMSv2Server: kmsV2Server,
 		HealthCheckURL: &url.URL{
 			Host: net.JoinHostPort("", strconv.FormatUint(uint64(*healthzPort), 10)),
 			Path: *healthzPath,