Replicate medusa bucket secrets to each Cass DC

Author: rzvoncek

CHANGELOG/CHANGELOG-1.12.md

@@ -15,6 +15,7 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
 
 ## unreleased
 
+* [ENHANCEMENT] [#1159](https://github.com/k8ssandra/k8ssandra-operator/issues/1159) Replicate bucket key secrets to namespaces hosting clusters
 * [CHANGE] Upgrade to Medusa v0.17.2
 * [CHANGE] [#1158](https://github.com/k8ssandra/k8ssandra-operator/issues/1158) Use the MedusaConfiguration API when creating Medusa configuration
 * [CHANGE] [#1050](https://github.com/k8ssandra/k8ssandra-operator/issues/1050) Remove unnecessary requeues in the Medusa controllers

controllers/k8ssandra/medusa_reconciler.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/adutra/goalesce"
 	medusaapi "github.com/k8ssandra/k8ssandra-operator/apis/medusa/v1alpha1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/go-logr/logr"
@@ -36,7 +37,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa(
 	if medusaSpec != nil {
 		logger.Info("Medusa is enabled")
 
-		mergeResult := mergeStorageProperties(ctx, remoteClient, namespace, medusaSpec, logger, kc)
+		mergeResult := r.mergeStorageProperties(ctx, remoteClient, namespace, medusaSpec, logger, kc)
 		if mergeResult.IsError() {
 			return result.Error(mergeResult.GetError())
 		}
@@ -49,6 +50,11 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa(
 				return result.Error(fmt.Errorf("medusa encryption certificates were not provided despite client encryption being enabled"))
 			}
 		}
+
+		recRes := r.reconcileBucketSecrets(ctx, remoteClient, kc, logger, namespace)
+		if recRes.IsError() {
+			return recRes
+		}
 		if medusaSpec.StorageProperties.StorageSecretRef.Name == "" {
 			return result.Error(fmt.Errorf("medusa storage secret is not defined for storage provider %s", medusaSpec.StorageProperties.StorageProvider))
 		}
@@ -92,7 +98,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa(
 		// Reconcile the Medusa standalone deployment
 		kcKey := utils.GetKey(kc)
 		desiredMedusaStandalone.SetLabels(labels.CleanedUpByLabels(kcKey))
-		recRes := reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *desiredMedusaStandalone)
+		recRes = reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *desiredMedusaStandalone)
 		switch {
 		case recRes.IsError():
 			return recRes
@@ -213,7 +219,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusaConfigMap(
 	return result.Continue()
 }
 
-func mergeStorageProperties(
+func (r *K8ssandraClusterReconciler) mergeStorageProperties(
 	ctx context.Context,
 	remoteClient client.Client,
 	namespace string,
@@ -229,7 +235,7 @@ func mergeStorageProperties(
 	configKey := types.NamespacedName{Namespace: namespace, Name: medusaSpec.MedusaConfigurationRef.Name}
 	if err := remoteClient.Get(ctx, configKey, storageProperties); err != nil {
 		logger.Error(err, fmt.Sprintf("failed to get MedusaConfiguration %s", configKey))
-		return result.Error(err)
+		return result.RequeueSoon(r.DefaultDelay)
 	}
 	// check if the StorageProperties from the cluster have the prefix field set
 	// it is required to be present because that's the single thing that differentiates backups of two different clusters
@@ -247,3 +253,63 @@ func mergeStorageProperties(
 	mergedProperties.DeepCopyInto(&desiredKc.Spec.Medusa.StorageProperties)
 	return result.Continue()
 }
+
+func (r *K8ssandraClusterReconciler) reconcileBucketSecrets(ctx context.Context, remoteClient client.Client, kc *api.K8ssandraCluster, logger logr.Logger, namespace string) result.ReconcileResult {
+	logger.Info("Reconciling Medusa bucket secrets")
+	medusaSpec := kc.Spec.Medusa
+	// there is nothing to reconcile if we're not using Medusa configuration reference
+	if medusaSpec.MedusaConfigurationRef.Name == "" {
+		logger.Info("MedusaConfigurationRef is not set, skipping bucket secret reconciliation")
+		return result.Continue()
+	}
+
+	// fetch the referenced configuration
+	medusaConfigNamespace := utils.FirstNonEmptyString(medusaSpec.MedusaConfigurationRef.Namespace, namespace)
+	medusaConfigName := medusaSpec.MedusaConfigurationRef.Name
+	medusaConfig := &medusaapi.MedusaConfiguration{}
+	medusaConfigKey := types.NamespacedName{Namespace: medusaConfigNamespace, Name: medusaConfigName}
+	if err := remoteClient.Get(ctx, medusaConfigKey, medusaConfig); err != nil {
+		logger.Error(err, "failed to get MedusaConfiguration", "key", medusaConfigKey)
+		return result.RequeueSoon(r.DefaultDelay)
+	}
+
+	// fetch the referenced medusa configuration's bucket secret
+	bucketSecretName := medusaConfig.Spec.StorageProperties.StorageSecretRef.Name
+	bucketSecret := &corev1.Secret{}
+	bucketSecretKey := types.NamespacedName{Namespace: medusaConfigNamespace, Name: bucketSecretName}
+	err := remoteClient.Get(ctx, bucketSecretKey, bucketSecret)
+	if err != nil {
+		logger.Error(err, "failed to get bucket secret", "key", bucketSecretKey)
+		return result.RequeueSoon(r.DefaultDelay)
+	}
+
+	// write the secret into the namespace of the K8ssandraCluster
+	clusterBucketSecret := bucketSecret.DeepCopy()
+	clusterBucketSecret.ResourceVersion = ""
+	clusterBucketSecret.Name = fmt.Sprintf("%s-%s", kc.Name, bucketSecret.Name)
+	clusterBucketSecret.Namespace = kc.Namespace
+	labels.SetReplicatedBy(clusterBucketSecret, utils.GetKey(kc))
+	err = remoteClient.Create(ctx, clusterBucketSecret)
+	if err != nil {
+		if !errors.IsAlreadyExists(err) {
+			logger.Error(err, fmt.Sprintf("failed to create cluster bucket secret %s", clusterBucketSecret))
+			return result.Error(err)
+		}
+		// we already have the bucket secret, so continue to updating the cluster (it might have failed before)
+	}
+
+	// finally, update the storage properties to point to a newly created secret
+	if kc.Spec.Medusa.StorageProperties.StorageSecretRef.Name == clusterBucketSecret.Name {
+		logger.Info("The cluster bucket secret is already set")
+		return result.Continue()
+	} else {
+		newSecretRef := corev1.LocalObjectReference{Name: clusterBucketSecret.Name}
+		kc.Spec.Medusa.StorageProperties.StorageSecretRef = newSecretRef
+		if err = remoteClient.Update(ctx, kc); err != nil {
+			logger.Error(err, "failed to update K8ssandraCluster, re-queueing")
+			return result.RequeueSoon(r.DefaultDelay)
+		}
+		logger.Info("Medusa bucket secrets successfully updated", "cluster", kc.Name, "namespace", kc.Namespace)
+		return result.Continue()
+	}
+}

docs/content/en/tasks/backup-restore/_index.md

@@ -88,7 +88,7 @@ spec:
       #   size: 100Mi
 ```
 
-The definition above requires a secret named `medusa-bucket-key` to be created in the target namespace before the `K8ssandraCluster` object gets created. Use the following format for this secret: 
+The definition above requires a secret named `medusa-bucket-key` to be present in the target namespace before the `K8ssandraCluster` object gets created. Use the following format for this secret: 
 
 ```yaml
 apiVersion: v1
@@ -106,9 +106,11 @@ stringData:
 
 The file should always specify `credentials` as shown in the example above; in that section, provide the expected format and credential values that are expected by Medusa for the chosen storage backend. For more, refer to the [Medusa documentation](https://github.com/thelastpickle/cassandra-medusa/blob/master/docs/Installation.md) to know which file format should used for each supported storage backend.
 
-A successful deployment should inject a new init container named `medusa-restore` and a new container named `medusa` in the Cassandra StatefulSet pods.  
+If using a shared Medusa configuration (see below), this secret can be created in the same namespace as the `MedusaConfiguration` object. The K8ssandra operator will then make sure the secret is replicated to the namespaces hosting the Cassandra clusters.
 
-## Using shared medusa configuration properties
+A successful deployment should inject a new init container named `medusa-restore` and a new container named `medusa` in the Cassandra StatefulSet pods.
+
+## Using shared Medusa configuration properties
 
 Medusa configuration properties can be shared across multiple K8ssandraClusters by creating a `MedusaConfiguration` custom resource in the Control Plane K8ssandra cluster.
 Example:

test/e2e/cluster_scope_test.go

@@ -23,6 +23,7 @@ func multiDcMultiCluster(t *testing.T, ctx context.Context, klusterNamespace str
 
 	dc1Key := framework.ClusterKey{K8sContext: f.DataPlaneContexts[0], NamespacedName: types.NamespacedName{Namespace: dc1Namespace, Name: "dc1"}}
 	checkDatacenterReady(t, ctx, dc1Key, f)
+	checkBucketKeyPresent(t, f, ctx, dc1Namespace, dc1Key.K8sContext, k8ssandra)
 
 	t.Log("check k8ssandra cluster status")
 	require.Eventually(func() bool {
@@ -41,6 +42,7 @@ func multiDcMultiCluster(t *testing.T, ctx context.Context, klusterNamespace str
 
 	dc2Key := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: dc2Namespace, Name: "dc2"}}
 	checkDatacenterReady(t, ctx, dc2Key, f)
+	checkBucketKeyPresent(t, f, ctx, dc2Namespace, dc2Key.K8sContext, k8ssandra)
 
 	t.Log("check k8ssandra cluster status")
 	require.Eventually(func() bool {

test/e2e/medusa_test.go

@@ -22,8 +22,9 @@ import (
 )
 
 const (
-	backupName  = "backup1"
-	clusterName = "test"
+	backupName             = "backup1"
+	clusterName            = "test"
+	globalBucketSecretName = "global-bucket-key"
 )
 
 func createSingleMedusaJob(t *testing.T, ctx context.Context, namespace string, f *framework.E2eFramework) {
@@ -69,6 +70,11 @@ func createMultiMedusaJob(t *testing.T, ctx context.Context, namespace string, f
 	dc1Key := framework.ClusterKey{K8sContext: "kind-k8ssandra-0", NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}}
 	dc2Key := framework.ClusterKey{K8sContext: "kind-k8ssandra-1", NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc2"}}
 
+	// Check that Medusa's bucket key has been replicated to the current namespace
+	for _, dcKey := range []framework.ClusterKey{dc1Key, dc2Key} {
+		checkBucketKeyPresent(t, f, ctx, namespace, dcKey.K8sContext, kc)
+	}
+
 	// Check that both DCs are ready and have Medusa containers
 	for _, dcKey := range []framework.ClusterKey{dc1Key, dc2Key} {
 		checkDatacenterReady(t, ctx, dcKey, f)
@@ -112,6 +118,25 @@ func createMultiDcSingleMedusaJob(t *testing.T, ctx context.Context, namespace s
 	verifyBackupJobFinished(t, ctx, f, dcKey, backupKey)
 }
 
+func checkBucketKeyPresent(t *testing.T, f *framework.E2eFramework, ctx context.Context, namespace string, k8sContext string, kc *k8ssandraapi.K8ssandraCluster) {
+	require := require.New(t)
+
+	// work out the name of the replicated bucket key. should be "clusterName-<original-bucket-key-name>"
+	localBucketKeyName := fmt.Sprintf("%s-%s", kc.Name, globalBucketSecretName)
+
+	// Check that the bucket key has been replicated to the current namespace
+	bucketKey := &corev1.Secret{}
+	require.Eventually(func() bool {
+		err := f.Get(ctx, framework.NewClusterKey(k8sContext, namespace, localBucketKeyName), bucketKey)
+		if err != nil {
+			return false
+		}
+		return true
+	}, polling.medusaConfigurationReady.timeout, polling.medusaConfigurationReady.interval,
+		fmt.Sprintf("Error getting the Medusa bucket key secret. Context: %s, ClusterName: %s, Namespace: %s", k8sContext, kc.SanitizedName(), namespace),
+	)
+}
+
 func checkMedusaContainersExist(t *testing.T, ctx context.Context, namespace string, dcKey framework.ClusterKey, f *framework.E2eFramework, kc *api.K8ssandraCluster) {
 	require := require.New(t)
 	// Get the Cassandra pod

test/e2e/suite_test.go

@@ -277,6 +277,7 @@ func TestOperator(t *testing.T) {
 			clusterScoped:        true,
 			sutNamespace:         "test-0",
 			additionalNamespaces: []string{"test-1", "test-2"},
+			installMinio:         true,
 		}))
 	})
 	t.Run("CreateSingleMedusaJob", e2eTest(ctx, &e2eTestOpts{

test/testdata/fixtures/multi-dc-cluster-scope/k8ssandra.yaml

@@ -4,6 +4,12 @@ metadata:
   name: test
   namespace: test-0
 spec:
+  medusa:
+    medusaConfigurationRef:
+      name: global-medusa-config
+      namespace: test-0
+    storageProperties:
+      prefix: test
   cassandra:
     serverVersion: "3.11.14"
     storageConfig:

test/testdata/fixtures/multi-dc-cluster-scope/kustomization.yaml

@@ -1,4 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - medusa-config.yaml
   - k8ssandra.yaml

test/testdata/fixtures/multi-dc-cluster-scope/medusa-config.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: global-bucket-key
+  namespace: test-1
+type: Opaque
+stringData:
+  # Note that this currently has to be set to credentials!
+  credentials: |-
+    [default]
+    aws_access_key_id = k8ssandra
+    aws_secret_access_key = k8ssandra
+---
+apiVersion: medusa.k8ssandra.io/v1alpha1
+kind: MedusaConfiguration
+metadata:
+  name: global-medusa-config
+  namespace: test-1
+spec:
+  storageProperties:
+    storageProvider: s3_compatible
+    bucketName: k8ssandra-medusa
+    storageSecretRef:
+      name: global-bucket-key
+    host: minio-service.minio.svc.cluster.local
+    port: 9000
+    secure: false

test/testdata/fixtures/multi-dc-encryption-medusa/k8ssandra.yaml

@@ -21,15 +21,10 @@ metadata:
   name: test
 spec:
   medusa:
+    medusaConfigurationRef:
+      name: global-medusa-config
     storageProperties:
-      storageProvider: s3_compatible
-      bucketName: k8ssandra-medusa
       prefix: test
-      storageSecretRef:
-        name: medusa-bucket-key
-      host: minio-service.minio.svc.cluster.local
-      port: 9000
-      secure: false 
     certificatesSecretRef:
       name: client-certificates
   cassandra:

test/testdata/fixtures/multi-dc-encryption-medusa/kustomization.yaml

@@ -1,4 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - medusa-config.yaml
   - k8ssandra.yaml

test/testdata/fixtures/multi-dc-encryption-medusa/medusa-config.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: global-bucket-key
+type: Opaque
+stringData:
+  # Note that this currently has to be set to credentials!
+  credentials: |-
+    [default]
+    aws_access_key_id = k8ssandra
+    aws_secret_access_key = k8ssandra
+---
+apiVersion: medusa.k8ssandra.io/v1alpha1
+kind: MedusaConfiguration
+metadata:
+  name: global-medusa-config
+spec:
+  storageProperties:
+    storageProvider: s3_compatible
+    bucketName: k8ssandra-medusa
+    storageSecretRef:
+      name: global-bucket-key
+    host: minio-service.minio.svc.cluster.local
+    port: 9000
+    secure: false