Replicate medusa bucket secrets to each Cass DC

Author: rzvoncek

CHANGELOG/CHANGELOG-1.12.md

@@ -15,6 +15,7 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
 
 ## unreleased
 
+* [ENHANCEMENT] [#1159](https://github.com/k8ssandra/k8ssandra-operator/issues/1159) Replicate bucket key secrets to namespaces hosting clusters
 * [CHANGE] Upgrade to Medusa v0.17.2
 * [CHANGE] [#1158](https://github.com/k8ssandra/k8ssandra-operator/issues/1158) Use the MedusaConfiguration API when creating Medusa configuration
 * [CHANGE] [#1050](https://github.com/k8ssandra/k8ssandra-operator/issues/1050) Remove unnecessary requeues in the Medusa controllers

controllers/k8ssandra/k8ssandracluster_controller_test.go

@@ -103,6 +103,7 @@ func TestK8ssandraCluster(t *testing.T) {
 	t.Run("CreateMultiDcClusterWithMedusa", testEnv.ControllerTest(ctx, createMultiDcClusterWithMedusa))
 	t.Run("CreateSingleDcClusterWithMedusaConfigRef", testEnv.ControllerTest(ctx, createSingleDcClusterWithMedusaConfigRef))
 	t.Run("CreatingSingleDcClusterWithoutPrefixInClusterSpecFail", testEnv.ControllerTest(ctx, creatingSingleDcClusterWithoutPrefixInClusterSpecFails))
+	t.Run("BucketSecretsGetReplicated", testEnv.ControllerTest(ctx, bucketSecretGetsReplicated))
 	t.Run("CreateSingleDcClusterNoAuth", testEnv.ControllerTest(ctx, createSingleDcClusterNoAuth))
 	t.Run("CreateSingleDcClusterAuth", testEnv.ControllerTest(ctx, createSingleDcClusterAuth))
 	t.Run("CreateSingleDcClusterAuthExternalSecrets", testEnv.ControllerTest(ctx, createSingleDcClusterAuthExternalSecrets))

controllers/k8ssandra/medusa_reconciler.go

@@ -4,11 +4,9 @@ import (
 	"context"
 	"fmt"
 	"github.com/adutra/goalesce"
-	medusaapi "github.com/k8ssandra/k8ssandra-operator/apis/medusa/v1alpha1"
-	"k8s.io/apimachinery/pkg/types"
-
 	"github.com/go-logr/logr"
 	api "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1"
+	medusaapi "github.com/k8ssandra/k8ssandra-operator/apis/medusa/v1alpha1"
 	cassandra "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/labels"
 	medusa "github.com/k8ssandra/k8ssandra-operator/pkg/medusa"
@@ -18,6 +16,8 @@ import (
 	"github.com/k8ssandra/k8ssandra-operator/pkg/utils"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -36,7 +36,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa(
 	if medusaSpec != nil {
 		logger.Info("Medusa is enabled")
 
-		mergeResult := mergeStorageProperties(ctx, remoteClient, namespace, medusaSpec, logger, kc)
+		mergeResult := r.mergeStorageProperties(ctx, remoteClient, namespace, medusaSpec, logger, kc)
 		if mergeResult.IsError() {
 			return result.Error(mergeResult.GetError())
 		}
@@ -49,6 +49,11 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa(
 				return result.Error(fmt.Errorf("medusa encryption certificates were not provided despite client encryption being enabled"))
 			}
 		}
+
+		recRes := r.reconcileBucketSecrets(ctx, remoteClient, kc, logger, namespace)
+		if recRes.IsError() {
+			return recRes
+		}
 		if medusaSpec.StorageProperties.StorageSecretRef.Name == "" {
 			return result.Error(fmt.Errorf("medusa storage secret is not defined for storage provider %s", medusaSpec.StorageProperties.StorageProvider))
 		}
@@ -92,7 +97,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa(
 		// Reconcile the Medusa standalone deployment
 		kcKey := utils.GetKey(kc)
 		desiredMedusaStandalone.SetLabels(labels.CleanedUpByLabels(kcKey))
-		recRes := reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *desiredMedusaStandalone)
+		recRes = reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *desiredMedusaStandalone)
 		switch {
 		case recRes.IsError():
 			return recRes
@@ -213,7 +218,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusaConfigMap(
 	return result.Continue()
 }
 
-func mergeStorageProperties(
+func (r *K8ssandraClusterReconciler) mergeStorageProperties(
 	ctx context.Context,
 	remoteClient client.Client,
 	namespace string,
@@ -229,7 +234,7 @@ func mergeStorageProperties(
 	configKey := types.NamespacedName{Namespace: namespace, Name: medusaSpec.MedusaConfigurationRef.Name}
 	if err := remoteClient.Get(ctx, configKey, storageProperties); err != nil {
 		logger.Error(err, fmt.Sprintf("failed to get MedusaConfiguration %s", configKey))
-		return result.Error(err)
+		return result.RequeueSoon(r.DefaultDelay)
 	}
 	// check if the StorageProperties from the cluster have the prefix field set
 	// it is required to be present because that's the single thing that differentiates backups of two different clusters
@@ -247,3 +252,64 @@ func mergeStorageProperties(
 	mergedProperties.DeepCopyInto(&desiredKc.Spec.Medusa.StorageProperties)
 	return result.Continue()
 }
+
+func (r *K8ssandraClusterReconciler) reconcileBucketSecrets(ctx context.Context, remoteClient client.Client, kc *api.K8ssandraCluster, logger logr.Logger, namespace string) result.ReconcileResult {
+	logger.Info("Reconciling Medusa bucket secrets")
+	medusaSpec := kc.Spec.Medusa
+	// there is nothing to reconcile if we're not using Medusa configuration reference
+	if medusaSpec.MedusaConfigurationRef.Name == "" {
+		logger.Info("MedusaConfigurationRef is not set, skipping bucket secret reconciliation")
+		return result.Continue()
+	}
+
+	// fetch the referenced configuration
+	medusaConfigNamespace := utils.FirstNonEmptyString(medusaSpec.MedusaConfigurationRef.Namespace, namespace)
+	medusaConfigName := medusaSpec.MedusaConfigurationRef.Name
+	medusaConfig := &medusaapi.MedusaConfiguration{}
+	medusaConfigKey := types.NamespacedName{Namespace: medusaConfigNamespace, Name: medusaConfigName}
+	if err := remoteClient.Get(ctx, medusaConfigKey, medusaConfig); err != nil {
+		logger.Error(err, "failed to get MedusaConfiguration", "key", medusaConfigKey)
+		return result.Error(err)
+	}
+
+	// fetch the referenced medusa configuration's bucket secret
+	bucketSecretName := medusaConfig.Spec.StorageProperties.StorageSecretRef.Name
+	bucketSecret := &corev1.Secret{}
+	bucketSecretKey := types.NamespacedName{Namespace: medusaConfigNamespace, Name: bucketSecretName}
+	err := remoteClient.Get(ctx, bucketSecretKey, bucketSecret)
+	if err != nil {
+		logger.Error(err, "failed to get bucket secret", "key", bucketSecretKey)
+		return result.Error(err)
+	}
+
+	// write the secret into the namespace of the K8ssandraCluster
+	clusterBucketSecret := bucketSecret.DeepCopy()
+	clusterBucketSecret.ResourceVersion = ""
+	clusterBucketSecret.Name = fmt.Sprintf("%s-%s", kc.Name, bucketSecret.Name)
+	clusterBucketSecret.Namespace = kc.Namespace
+	labels.SetReplicatedBy(clusterBucketSecret, utils.GetKey(kc))
+	err = remoteClient.Create(ctx, clusterBucketSecret)
+	if err != nil {
+		if !errors.IsAlreadyExists(err) {
+			logger.Error(err, fmt.Sprintf("failed to create cluster bucket secret %s", clusterBucketSecret))
+			return result.Error(err)
+		}
+		// we already have the bucket secret, so continue to updating the cluster (it might have failed before)
+	}
+
+	// finally, update the storage properties to point to a newly created secret
+	//if kc.Spec.Medusa.StorageProperties.StorageSecretRef.Name == clusterBucketSecret.Name {
+	//	logger.Info("The cluster bucket secret is already set")
+	//	return result.Continue()
+	//} else {
+	//	newSecretRef := &corev1.LocalObjectReference{Name: clusterBucketSecret.Name}
+	//	kc.Spec.Medusa.StorageProperties.StorageSecretRef = *newSecretRef
+	//	if err = remoteClient.Update(ctx, kc); err != nil {
+	//		logger.Error(err, "failed to update K8ssandraCluster")
+	//		return result.Error(err)
+	//	}
+	//	logger.Info("Medusa bucket secrets successfully updated", "cluster", kc.Name, "namespace", kc.Namespace)
+	//	return result.Continue()
+	//}
+	return result.Continue()
+}

controllers/k8ssandra/medusa_reconciler_test.go

@@ -538,3 +538,77 @@ func creatingSingleDcClusterWithoutPrefixInClusterSpecFails(t *testing.T, ctx co
 	// verify the cluster still doesn't get created
 	require.Never(f.DatacenterExists(ctx, dc1Key), timeout, interval)
 }
+
+func keyFor(f *framework.Framework, object metav1.Object, contextIndex int) framework.ClusterKey {
+	return framework.ClusterKey{NamespacedName: utils.GetKey(object), K8sContext: f.DataPlaneContexts[contextIndex]}
+}
+
+func bucketSecretGetsReplicated(t *testing.T, ctx context.Context, f *framework.Framework, namespace string) {
+	require := require.New(t)
+
+	// create a storage secret, then a MedusaConfiguration that points to it
+	medusaSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      storageSecret,
+			Namespace: namespace,
+		},
+		StringData: map[string]string{
+			"credentials": "some-credentials",
+		},
+	}
+	err := f.Create(ctx, keyFor(f, medusaSecret, 0), medusaSecret)
+	require.NoError(err, "failed to create secret")
+
+	medusaConfig := MedusaConfig(namespace)
+	medusaConfig.Spec.StorageProperties.StorageSecretRef = corev1.LocalObjectReference{
+		Name: medusaSecret.Name,
+	}
+	err = f.Create(ctx, keyFor(f, medusaConfig, 0), medusaConfig)
+	require.NoError(err, "failed to create MedusaConfiguration")
+
+	// create a 2-dc K8ssandraCluster with Medusa featuring the reference to the above MedusaConfiguration
+	kc := &api.K8ssandraCluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      k8ssandraClusterName,
+			Namespace: namespace,
+		},
+		Spec: api.K8ssandraClusterSpec{
+			Cassandra: &api.CassandraClusterTemplate{
+				Datacenters: []api.CassandraDatacenterTemplate{
+					dcTemplate("dc1", f.DataPlaneContexts[0]),
+					dcTemplate("dc2", f.DataPlaneContexts[2]),
+				},
+			},
+			Medusa: &medusaapi.MedusaClusterTemplate{
+				MedusaConfigurationRef: corev1.ObjectReference{
+					Namespace: namespace,
+					Name:      medusaConfig.Name,
+				},
+				StorageProperties: medusaapi.Storage{
+					Prefix: "some-prefix",
+					StorageSecretRef: corev1.LocalObjectReference{
+						Name: fmt.Sprintf("%s-%s", k8ssandraClusterName, medusaConfig.Name),
+					},
+				},
+			},
+		},
+	}
+
+	//key := framework.ClusterKey{NamespacedName: utils.GetKey(kc), K8sContext: f.DataPlaneContexts[1]}
+	//err = f.Create(ctx, key, kc)
+	err = f.Client.Create(ctx, kc)
+
+	require.NoError(err, "failed to create K8ssandraCluster")
+
+	verifySuperuserSecretCreated(ctx, t, f, kc)
+	verifyReplicatedSecretReconciled(ctx, t, f, kc)
+
+	dc1Key := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}, K8sContext: f.DataPlaneContexts[0]}
+	require.Eventually(f.DatacenterExists(ctx, dc1Key), timeout, interval)
+	t.Log("update dc1 status to ready")
+	err = f.SetDatacenterStatusReady(ctx, dc1Key)
+	require.NoError(err, "failed to update dc1 status to ready")
+	dc2Key := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc2"}, K8sContext: f.DataPlaneContexts[2]}
+	require.Eventually(f.DatacenterExists(ctx, dc2Key), timeout, interval)
+
+}

docs/content/en/tasks/backup-restore/_index.md

@@ -88,7 +88,7 @@ spec:
       #   size: 100Mi
 ```
 
-The definition above requires a secret named `medusa-bucket-key` to be created in the target namespace before the `K8ssandraCluster` object gets created. Use the following format for this secret: 
+The definition above requires a secret named `medusa-bucket-key` to be present in the target namespace before the `K8ssandraCluster` object gets created. Use the following format for this secret: 
 
 ```yaml
 apiVersion: v1
@@ -106,9 +106,11 @@ stringData:
 
 The file should always specify `credentials` as shown in the example above; in that section, provide the expected format and credential values that are expected by Medusa for the chosen storage backend. For more, refer to the [Medusa documentation](https://github.com/thelastpickle/cassandra-medusa/blob/master/docs/Installation.md) to know which file format should used for each supported storage backend.
 
-A successful deployment should inject a new init container named `medusa-restore` and a new container named `medusa` in the Cassandra StatefulSet pods.  
+If using a shared Medusa configuration (see below), this secret can be created in the same namespace as the `MedusaConfiguration` object. The K8ssandra operator will then make sure the secret is replicated to the namespaces hosting the Cassandra clusters.
 
-## Using shared medusa configuration properties
+A successful deployment should inject a new init container named `medusa-restore` and a new container named `medusa` in the Cassandra StatefulSet pods.
+
+## Using shared Medusa configuration properties
 
 Medusa configuration properties can be shared across multiple K8ssandraClusters by creating a `MedusaConfiguration` custom resource in the Control Plane K8ssandra cluster.
 Example:

test/e2e/cluster_scope_test.go

@@ -23,6 +23,7 @@ func multiDcMultiCluster(t *testing.T, ctx context.Context, klusterNamespace str
 
 	dc1Key := framework.ClusterKey{K8sContext: f.DataPlaneContexts[0], NamespacedName: types.NamespacedName{Namespace: dc1Namespace, Name: "dc1"}}
 	checkDatacenterReady(t, ctx, dc1Key, f)
+	checkBucketKeyPresent(t, f, ctx, dc1Namespace, dc1Key.K8sContext, k8ssandra)
 
 	t.Log("check k8ssandra cluster status")
 	require.Eventually(func() bool {
@@ -41,6 +42,7 @@ func multiDcMultiCluster(t *testing.T, ctx context.Context, klusterNamespace str
 
 	dc2Key := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: dc2Namespace, Name: "dc2"}}
 	checkDatacenterReady(t, ctx, dc2Key, f)
+	checkBucketKeyPresent(t, f, ctx, dc2Namespace, dc2Key.K8sContext, k8ssandra)
 
 	t.Log("check k8ssandra cluster status")
 	require.Eventually(func() bool {

test/e2e/medusa_test.go

@@ -22,8 +22,9 @@ import (
 )
 
 const (
-	backupName  = "backup1"
-	clusterName = "test"
+	backupName             = "backup1"
+	clusterName            = "test"
+	globalBucketSecretName = "global-bucket-key"
 )
 
 func createSingleMedusaJob(t *testing.T, ctx context.Context, namespace string, f *framework.E2eFramework) {
@@ -69,6 +70,11 @@ func createMultiMedusaJob(t *testing.T, ctx context.Context, namespace string, f
 	dc1Key := framework.ClusterKey{K8sContext: "kind-k8ssandra-0", NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}}
 	dc2Key := framework.ClusterKey{K8sContext: "kind-k8ssandra-1", NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc2"}}
 
+	// Check that Medusa's bucket key has been replicated to the current namespace
+	for _, dcKey := range []framework.ClusterKey{dc1Key, dc2Key} {
+		checkBucketKeyPresent(t, f, ctx, namespace, dcKey.K8sContext, kc)
+	}
+
 	// Check that both DCs are ready and have Medusa containers
 	for _, dcKey := range []framework.ClusterKey{dc1Key, dc2Key} {
 		checkDatacenterReady(t, ctx, dcKey, f)
@@ -112,6 +118,22 @@ func createMultiDcSingleMedusaJob(t *testing.T, ctx context.Context, namespace s
 	verifyBackupJobFinished(t, ctx, f, dcKey, backupKey)
 }
 
+func checkBucketKeyPresent(t *testing.T, f *framework.E2eFramework, ctx context.Context, namespace string, k8sContext string, kc *k8ssandraapi.K8ssandraCluster) {
+	require := require.New(t)
+
+	// work out the name of the replicated bucket key. should be "clusterName-<original-bucket-key-name>"
+	localBucketKeyName := fmt.Sprintf("%s-%s", kc.Name, globalBucketSecretName)
+
+	// Check that the bucket key has been replicated to the current namespace
+	bucketKey := &corev1.Secret{}
+	require.Eventually(func() bool {
+		err := f.Get(ctx, framework.NewClusterKey(k8sContext, namespace, localBucketKeyName), bucketKey)
+		return err == nil
+	}, polling.medusaConfigurationReady.timeout, polling.medusaConfigurationReady.interval,
+		fmt.Sprintf("Error getting the Medusa bucket key secret. Context: %s, ClusterName: %s, Namespace: %s", k8sContext, kc.SanitizedName(), namespace),
+	)
+}
+
 func checkMedusaContainersExist(t *testing.T, ctx context.Context, namespace string, dcKey framework.ClusterKey, f *framework.E2eFramework, kc *api.K8ssandraCluster) {
 	require := require.New(t)
 	// Get the Cassandra pod

test/e2e/suite_test.go

@@ -277,6 +277,7 @@ func TestOperator(t *testing.T) {
 			clusterScoped:        true,
 			sutNamespace:         "test-0",
 			additionalNamespaces: []string{"test-1", "test-2"},
+			installMinio:         true,
 		}))
 	})
 	t.Run("CreateSingleMedusaJob", e2eTest(ctx, &e2eTestOpts{

test/testdata/fixtures/multi-dc-cluster-scope/k8ssandra.yaml

@@ -4,6 +4,12 @@ metadata:
   name: test
   namespace: test-0
 spec:
+  medusa:
+    medusaConfigurationRef:
+      name: global-medusa-config
+      namespace: test-0
+    storageProperties:
+      prefix: test
   cassandra:
     serverVersion: "3.11.14"
     storageConfig:

test/testdata/fixtures/multi-dc-cluster-scope/kustomization.yaml

@@ -1,4 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - medusa-config.yaml
   - k8ssandra.yaml

test/testdata/fixtures/multi-dc-cluster-scope/medusa-config.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: global-bucket-key
+  namespace: test-1
+type: Opaque
+stringData:
+  # Note that this currently has to be set to credentials!
+  credentials: |-
+    [default]
+    aws_access_key_id = k8ssandra
+    aws_secret_access_key = k8ssandra
+---
+apiVersion: medusa.k8ssandra.io/v1alpha1
+kind: MedusaConfiguration
+metadata:
+  name: global-medusa-config
+  namespace: test-1
+spec:
+  storageProperties:
+    storageProvider: s3_compatible
+    bucketName: k8ssandra-medusa
+    storageSecretRef:
+      name: global-bucket-key
+    host: minio-service.minio.svc.cluster.local
+    port: 9000
+    secure: false

test/testdata/fixtures/multi-dc-encryption-medusa/k8ssandra.yaml

@@ -21,15 +21,10 @@ metadata:
   name: test
 spec:
   medusa:
+    medusaConfigurationRef:
+      name: global-medusa-config
     storageProperties:
-      storageProvider: s3_compatible
-      bucketName: k8ssandra-medusa
       prefix: test
-      storageSecretRef:
-        name: medusa-bucket-key
-      host: minio-service.minio.svc.cluster.local
-      port: 9000
-      secure: false 
     certificatesSecretRef:
       name: client-certificates
   cassandra:

test/testdata/fixtures/multi-dc-encryption-medusa/kustomization.yaml

@@ -1,4 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - medusa-config.yaml
   - k8ssandra.yaml