Replicate medusa bucket secrets to each Cass DC

Author: rzvoncek

CHANGELOG/CHANGELOG-1.12.md

@@ -15,6 +15,7 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
 
 ## unreleased
 
+* [ENHANCEMENT] [#1159](https://github.com/k8ssandra/k8ssandra-operator/issues/1159) Replicate bucket key secrets to namespaces hosting clusters
 * [CHANGE] Upgrade to Medusa v0.17.2
 * [CHANGE] [#1158](https://github.com/k8ssandra/k8ssandra-operator/issues/1158) Use the MedusaConfiguration API when creating Medusa configuration
 * [CHANGE] [#1050](https://github.com/k8ssandra/k8ssandra-operator/issues/1050) Remove unnecessary requeues in the Medusa controllers

controllers/k8ssandra/k8ssandracluster_controller_test.go

@@ -103,6 +103,7 @@ func TestK8ssandraCluster(t *testing.T) {
 	t.Run("CreateMultiDcClusterWithMedusa", testEnv.ControllerTest(ctx, createMultiDcClusterWithMedusa))
 	t.Run("CreateSingleDcClusterWithMedusaConfigRef", testEnv.ControllerTest(ctx, createSingleDcClusterWithMedusaConfigRef))
 	t.Run("CreatingSingleDcClusterWithoutPrefixInClusterSpecFail", testEnv.ControllerTest(ctx, creatingSingleDcClusterWithoutPrefixInClusterSpecFails))
+	t.Run("CreateMultiDcClusterWithReplicatedSecrets", testEnv.ControllerTest(ctx, createMultiDcClusterWithReplicatedSecrets))
 	t.Run("CreateSingleDcClusterNoAuth", testEnv.ControllerTest(ctx, createSingleDcClusterNoAuth))
 	t.Run("CreateSingleDcClusterAuth", testEnv.ControllerTest(ctx, createSingleDcClusterAuth))
 	t.Run("CreateSingleDcClusterAuthExternalSecrets", testEnv.ControllerTest(ctx, createSingleDcClusterAuthExternalSecrets))

controllers/k8ssandra/medusa_reconciler.go

@@ -4,11 +4,9 @@ import (
 	"context"
 	"fmt"
 	"github.com/adutra/goalesce"
-	medusaapi "github.com/k8ssandra/k8ssandra-operator/apis/medusa/v1alpha1"
-	"k8s.io/apimachinery/pkg/types"
-
 	"github.com/go-logr/logr"
 	api "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1"
+	medusaapi "github.com/k8ssandra/k8ssandra-operator/apis/medusa/v1alpha1"
 	cassandra "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/labels"
 	medusa "github.com/k8ssandra/k8ssandra-operator/pkg/medusa"
@@ -18,6 +16,8 @@ import (
 	"github.com/k8ssandra/k8ssandra-operator/pkg/utils"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -32,11 +32,11 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa(
 	kc := desiredKc.DeepCopy()
 	namespace := utils.FirstNonEmptyString(dcConfig.Meta.Namespace, kc.Namespace)
 	logger.Info("Medusa reconcile for " + dcConfig.CassDcName() + " on namespace " + namespace)
-	medusaSpec := kc.Spec.Medusa
-	if medusaSpec != nil {
+	if kc.Spec.Medusa != nil {
 		logger.Info("Medusa is enabled")
 
-		mergeResult := mergeStorageProperties(ctx, remoteClient, namespace, medusaSpec, logger, kc)
+		mergeResult := r.mergeStorageProperties(ctx, r.Client, namespace, kc.Spec.Medusa, logger, kc)
+		medusaSpec := kc.Spec.Medusa
 		if mergeResult.IsError() {
 			return result.Error(mergeResult.GetError())
 		}
@@ -49,6 +49,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa(
 				return result.Error(fmt.Errorf("medusa encryption certificates were not provided despite client encryption being enabled"))
 			}
 		}
+
 		if medusaSpec.StorageProperties.StorageSecretRef.Name == "" {
 			return result.Error(fmt.Errorf("medusa storage secret is not defined for storage provider %s", medusaSpec.StorageProperties.StorageProvider))
 		}
@@ -119,7 +120,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa(
 		}
 		if !ready {
 			logger.Info("Medusa standalone deployment is not ready yet")
-			return result.RequeueSoon(r.DefaultDelay)
+			return result.Error(err)
 		}
 		// Create a cron job to purge Medusa backups
 		purgeCronJob, err := medusa.PurgeCronJob(dcConfig, kc.SanitizedName(), namespace, logger)
@@ -183,6 +184,11 @@ func (r *K8ssandraClusterReconciler) reconcileMedusaSecrets(
 		}
 	}
 
+	if err := r.reconcileBucketSecrets(ctx, r.ClientCache.GetLocalClient(), kc, logger); err != nil {
+		logger.Error(err, "Failed to reconcile Medusa bucket secrets")
+		return result.Error(err)
+	}
+
 	logger.Info("Medusa user secrets successfully reconciled")
 	return result.Continue()
 }
@@ -213,7 +219,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusaConfigMap(
 	return result.Continue()
 }
 
-func mergeStorageProperties(
+func (r *K8ssandraClusterReconciler) mergeStorageProperties(
 	ctx context.Context,
 	remoteClient client.Client,
 	namespace string,
@@ -229,7 +235,7 @@ func mergeStorageProperties(
 	configKey := types.NamespacedName{Namespace: namespace, Name: medusaSpec.MedusaConfigurationRef.Name}
 	if err := remoteClient.Get(ctx, configKey, storageProperties); err != nil {
 		logger.Error(err, fmt.Sprintf("failed to get MedusaConfiguration %s", configKey))
-		return result.Error(err)
+		return result.RequeueSoon(r.DefaultDelay)
 	}
 	// check if the StorageProperties from the cluster have the prefix field set
 	// it is required to be present because that's the single thing that differentiates backups of two different clusters
@@ -243,7 +249,65 @@ func mergeStorageProperties(
 		logger.Error(err, "failed to merge MedusaConfiguration StorageProperties")
 		return result.Error(err)
 	}
+	// medusaapi.MedusaConfiguration comes with a storage corev1.Secret containing the credentials to access the storage
+	// we make a copy of that secret for each cluster/dc, and then point to it with a corev1.LocalObjectReference
+	// when we do the copy, we name the secret as <cluster-name>-<original-secret-name>
+	// here we need to update the reference to point to that copied secret
+	mergedProperties.StorageSecretRef.Name = fmt.Sprintf("%s-%s", desiredKc.Name, mergedProperties.StorageSecretRef.Name)
+
 	// copy the merged properties back into the cluster
 	mergedProperties.DeepCopyInto(&desiredKc.Spec.Medusa.StorageProperties)
 	return result.Continue()
 }
+
+func (r *K8ssandraClusterReconciler) reconcileBucketSecrets(
+	ctx context.Context,
+	c client.Client,
+	kc *api.K8ssandraCluster,
+	logger logr.Logger,
+) error {
+
+	logger.Info("Reconciling Medusa bucket secrets")
+	medusaSpec := kc.Spec.Medusa
+
+	// there is nothing to reconcile if we're not using Medusa configuration reference
+	if medusaSpec == nil || medusaSpec.MedusaConfigurationRef.Name == "" {
+		logger.Info("MedusaConfigurationRef is not set, skipping bucket secret reconciliation")
+		return nil
+	}
+
+	// fetch the referenced configuration
+	medusaConfigName := medusaSpec.MedusaConfigurationRef.Name
+	medusaConfigNamespace := medusaSpec.MedusaConfigurationRef.Namespace
+	medusaConfigKey := types.NamespacedName{Namespace: medusaConfigNamespace, Name: medusaConfigName}
+	medusaConfig := &medusaapi.MedusaConfiguration{}
+	if err := c.Get(ctx, medusaConfigKey, medusaConfig); err != nil {
+		logger.Error(err, "could not get MedusaConfiguration")
+		return err
+	}
+
+	// fetch the referenced medusa configuration's bucket secret
+	bucketSecretName := medusaConfig.Spec.StorageProperties.StorageSecretRef.Name
+	bucketSecret := &corev1.Secret{}
+	bucketSecretKey := types.NamespacedName{Namespace: medusaConfigNamespace, Name: bucketSecretName}
+	if err := c.Get(ctx, bucketSecretKey, bucketSecret); err != nil {
+		logger.Error(err, "could not get bucket Secret")
+		return err
+	}
+
+	// write the secret into the namespace of the K8ssandraCluster
+	clusterBucketSecret := bucketSecret.DeepCopy()
+	clusterBucketSecret.ResourceVersion = ""
+	clusterBucketSecret.Name = fmt.Sprintf("%s-%s", kc.Name, bucketSecret.Name)
+	clusterBucketSecret.Namespace = kc.Namespace
+	labels.SetReplicatedBy(clusterBucketSecret, utils.GetKey(kc))
+	if err := c.Create(ctx, clusterBucketSecret); err != nil {
+		if !errors.IsAlreadyExists(err) {
+			logger.Error(err, fmt.Sprintf("failed to create cluster bucket secret %s", clusterBucketSecret))
+			return err
+		}
+		// we already have the bucket secret, so continue to updating the cluster (it might have failed before)
+	}
+
+	return nil
+}

controllers/k8ssandra/medusa_reconciler_test.go

@@ -53,10 +53,10 @@ func dcTemplate(dcName string, dataPlaneContext string) api.CassandraDatacenterT
 	}
 }
 
-func MedusaConfig(namespace string) *medusaapi.MedusaConfiguration {
+func MedusaConfig(name, namespace string) *medusaapi.MedusaConfiguration {
 	return &medusaapi.MedusaConfiguration{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      medusaConfigName,
+			Name:      name,
 			Namespace: namespace,
 		},
 		Spec: medusaapi.MedusaConfigurationSpec{
@@ -442,7 +442,7 @@ func createSingleDcClusterWithMedusaConfigRef(t *testing.T, ctx context.Context,
 
 	t.Log("Creating Medusa Configuration object")
 	medusaConfigKey := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: medusaConfigName}, K8sContext: f.DataPlaneContexts[0]}
-	err := f.Create(ctx, medusaConfigKey, MedusaConfig(namespace))
+	err := f.Create(ctx, medusaConfigKey, MedusaConfig(medusaConfigName, namespace))
 	require.NoError(err, "failed to create Medusa Configuration")
 	require.Eventually(f.MedusaConfigExists(ctx, f.DataPlaneContexts[0], medusaConfigKey), timeout, interval)
 
@@ -520,7 +520,7 @@ func creatingSingleDcClusterWithoutPrefixInClusterSpecFails(t *testing.T, ctx co
 	// create the MedusaConfiguration object
 	t.Log("Creating Medusa Configuration object")
 	medusaConfigKey := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: medusaConfigName}, K8sContext: f.DataPlaneContexts[0]}
-	err = f.Create(ctx, medusaConfigKey, MedusaConfig(namespace))
+	err = f.Create(ctx, medusaConfigKey, MedusaConfig(medusaConfigName, namespace))
 	require.NoError(err, "failed to create Medusa Configuration")
 	require.Eventually(f.MedusaConfigExists(ctx, f.DataPlaneContexts[0], medusaConfigKey), timeout, interval)
 
@@ -538,3 +538,127 @@ func creatingSingleDcClusterWithoutPrefixInClusterSpecFails(t *testing.T, ctx co
 	// verify the cluster still doesn't get created
 	require.Never(f.DatacenterExists(ctx, dc1Key), timeout, interval)
 }
+
+func controlPlaneContextKey(f *framework.Framework, object metav1.Object, contextName string) framework.ClusterKey {
+	return framework.ClusterKey{NamespacedName: utils.GetKey(object), K8sContext: contextName}
+}
+
+func dataPlaneContextKey(f *framework.Framework, object metav1.Object, dataPlaneContextIndex int) framework.ClusterKey {
+	return framework.ClusterKey{NamespacedName: utils.GetKey(object), K8sContext: f.DataPlaneContexts[dataPlaneContextIndex]}
+}
+
+func createMultiDcClusterWithReplicatedSecrets(t *testing.T, ctx context.Context, f *framework.Framework, namespace string) {
+	require := require.New(t)
+
+	clusterName := "test-cluster"
+	originalConfigName := "test-config"
+	originalSecretName := fmt.Sprintf("%s-bucket-key", originalConfigName)
+	clusterSecretName := fmt.Sprintf("%s-%s", clusterName, originalSecretName)
+
+	// create a storage secret, then a MedusaConfiguration that points to it
+	// the ReplicatedSecrets controller is not loaded in env tests, so we "mock" it by replicating the secrets manually
+	medusaSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      originalSecretName,
+			Namespace: namespace,
+		},
+		StringData: map[string]string{
+			"credentials": "some-credentials",
+		},
+	}
+	// create the secret in the control plane
+	cpMedusaSecret := medusaSecret.DeepCopy()
+	err := f.Create(ctx, controlPlaneContextKey(f, cpMedusaSecret, f.ControlPlaneContext), cpMedusaSecret)
+	require.NoError(err, fmt.Sprintf("failed to create secret in control plane %s", f.ControlPlaneContext))
+	//create the secret in the data planes
+	for i, n := range f.DataPlaneContexts {
+		dpMedusaSecret := medusaSecret.DeepCopy()
+		dpMedusaSecret.Name = clusterSecretName
+		err := f.Create(ctx, dataPlaneContextKey(f, dpMedusaSecret, i), dpMedusaSecret)
+		require.NoError(err, fmt.Sprintf("failed to create secret in context %d (%s)", i, n))
+	}
+
+	// create medusa config in the control plane only
+	medusaConfig := MedusaConfig(originalConfigName, namespace)
+	medusaConfig.Spec.StorageProperties.StorageSecretRef = corev1.LocalObjectReference{
+		Name: originalSecretName,
+	}
+	cpMedusaConfig := medusaConfig.DeepCopy()
+	err = f.Create(ctx, controlPlaneContextKey(f, cpMedusaConfig, f.ControlPlaneContext), cpMedusaConfig)
+	require.NoError(err, fmt.Sprintf("failed to create MedusaConfiguration in control plane %s", f.ControlPlaneContext))
+
+	// create a 2-dc K8ssandraCluster with Medusa featuring the reference to the above MedusaConfiguration
+	kc := &api.K8ssandraCluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      clusterName,
+			Namespace: namespace,
+		},
+		Spec: api.K8ssandraClusterSpec{
+			Cassandra: &api.CassandraClusterTemplate{
+				Datacenters: []api.CassandraDatacenterTemplate{
+					dcTemplate("dc1", f.DataPlaneContexts[1]),
+					dcTemplate("dc2", f.DataPlaneContexts[2]),
+				},
+			},
+			Medusa: &medusaapi.MedusaClusterTemplate{
+				MedusaConfigurationRef: corev1.ObjectReference{
+					Namespace: namespace,
+					Name:      originalConfigName,
+				},
+				StorageProperties: medusaapi.Storage{
+					Prefix: "some-prefix",
+				},
+			},
+		},
+	}
+	err = f.Client.Create(ctx, kc)
+	require.NoError(err, "failed to create K8ssandraCluster")
+
+	verifySuperuserSecretCreated(ctx, t, f, kc)
+	verifyReplicatedSecretReconciled(ctx, t, f, kc)
+
+	reconcileMedusaStandaloneDeployment(ctx, t, f, kc, "dc1", f.DataPlaneContexts[1])
+
+	// crate the first DC
+	dc1Key := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}, K8sContext: f.DataPlaneContexts[1]}
+	require.Eventually(f.DatacenterExists(ctx, dc1Key), timeout, interval)
+
+	// mark the first DC as ready
+	t.Log("update dc1 status to ready")
+	err = f.SetDatacenterStatusReady(ctx, dc1Key)
+	require.NoError(err, "failed to update dc1 status to ready")
+
+	// create the second DC
+	reconcileMedusaStandaloneDeployment(ctx, t, f, kc, "dc2", f.DataPlaneContexts[2])
+	dc2Key := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc2"}, K8sContext: f.DataPlaneContexts[2]}
+	require.Eventually(f.DatacenterExists(ctx, dc2Key), timeout, interval)
+
+	// verify the copied secret is mounted in the pods
+	verifyBucketSecretMounted(ctx, t, f, dc1Key, clusterSecretName)
+	verifyBucketSecretMounted(ctx, t, f, dc2Key, clusterSecretName)
+
+	// verify the cluster's spec still contains the correct value
+	// which is empty because we used MedusaConfigRef
+	// merged it at runtime but never persisted to the k8ssandraCluster object
+	kc = &api.K8ssandraCluster{}
+	err = f.Client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: clusterName}, kc)
+	require.NoError(err, "failed to get K8ssandraCluster")
+	require.Equal("", kc.Spec.Medusa.StorageProperties.StorageSecretRef.Name)
+}
+
+func verifyBucketSecretMounted(ctx context.Context, t *testing.T, f *framework.Framework, dcKey framework.ClusterKey, clusterSecretName string) {
+	require := require.New(t)
+
+	// fetch the DC spec
+	dc := &cassdcapi.CassandraDatacenter{}
+	err := f.Get(ctx, dcKey, dc)
+	require.NoError(err, fmt.Sprintf("failed to get %s", dcKey.Name))
+
+	// fetch medusa container
+	containerIndex, found := cassandra.FindContainer(dc.Spec.PodTemplateSpec, "medusa")
+	require.True(found, fmt.Sprintf("%s doesn't have medusa container", dc.Name))
+	medusaContainer := dc.Spec.PodTemplateSpec.Spec.Containers[containerIndex]
+
+	// check its mount
+	assert.True(t, f.ContainerHasVolumeMount(medusaContainer, clusterSecretName, "/etc/medusa-secrets"), "Missing Volume Mount for Medusa bucket key")
+}

docs/content/en/tasks/backup-restore/_index.md

@@ -88,7 +88,7 @@ spec:
       #   size: 100Mi
 ```
 
-The definition above requires a secret named `medusa-bucket-key` to be created in the target namespace before the `K8ssandraCluster` object gets created. Use the following format for this secret: 
+The definition above requires a secret named `medusa-bucket-key` to be present in the target namespace before the `K8ssandraCluster` object gets created. Use the following format for this secret: 
 
 ```yaml
 apiVersion: v1
@@ -106,9 +106,11 @@ stringData:
 
 The file should always specify `credentials` as shown in the example above; in that section, provide the expected format and credential values that are expected by Medusa for the chosen storage backend. For more, refer to the [Medusa documentation](https://github.com/thelastpickle/cassandra-medusa/blob/master/docs/Installation.md) to know which file format should used for each supported storage backend.
 
-A successful deployment should inject a new init container named `medusa-restore` and a new container named `medusa` in the Cassandra StatefulSet pods.  
+If using a shared Medusa configuration (see below), this secret can be created in the same namespace as the `MedusaConfiguration` object. The K8ssandra operator will then make sure the secret is replicated to the namespaces hosting the Cassandra clusters.
 
-## Using shared medusa configuration properties
+A successful deployment should inject a new init container named `medusa-restore` and a new container named `medusa` in the Cassandra StatefulSet pods.
+
+## Using shared Medusa configuration properties
 
 Medusa configuration properties can be shared across multiple K8ssandraClusters by creating a `MedusaConfiguration` custom resource in the Control Plane K8ssandra cluster.
 Example:

test/e2e/cluster_scope_test.go

@@ -23,6 +23,7 @@ func multiDcMultiCluster(t *testing.T, ctx context.Context, klusterNamespace str
 
 	dc1Key := framework.ClusterKey{K8sContext: f.DataPlaneContexts[0], NamespacedName: types.NamespacedName{Namespace: dc1Namespace, Name: "dc1"}}
 	checkDatacenterReady(t, ctx, dc1Key, f)
+	checkBucketKeyPresent(t, f, ctx, dc1Namespace, dc1Key.K8sContext, k8ssandra)
 
 	t.Log("check k8ssandra cluster status")
 	require.Eventually(func() bool {
@@ -41,6 +42,7 @@ func multiDcMultiCluster(t *testing.T, ctx context.Context, klusterNamespace str
 
 	dc2Key := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: dc2Namespace, Name: "dc2"}}
 	checkDatacenterReady(t, ctx, dc2Key, f)
+	checkBucketKeyPresent(t, f, ctx, dc2Namespace, dc2Key.K8sContext, k8ssandra)
 
 	t.Log("check k8ssandra cluster status")
 	require.Eventually(func() bool {
@@ -65,6 +67,10 @@ func multiDcMultiCluster(t *testing.T, ctx context.Context, klusterNamespace str
 		return cassandraDatacenterReady(cassandraStatus)
 	}, polling.k8ssandraClusterStatus.timeout, polling.k8ssandraClusterStatus.interval, "timed out waiting for K8ssandraCluster status to get updated")
 
+	t.Log("check replicated secret mounted")
+	checkReplicatedSecretMounted(t, ctx, f, dc1Key, dc1Namespace, k8ssandra)
+	checkReplicatedSecretMounted(t, ctx, f, dc2Key, dc2Namespace, k8ssandra)
+
 	t.Log("retrieve database credentials")
 	username, password, err := f.RetrieveDatabaseCredentials(ctx, f.DataPlaneContexts[0], dc1Namespace, k8ssandra.SanitizedName())
 	require.NoError(err, "failed to retrieve database credentials")