Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🌱 Enhance E2E Tests to Filter Out Extra Provider-Specific Metadata #10715

Merged
merged 1 commit into from
Jul 4, 2024
Merged

🌱 Enhance E2E Tests to Filter Out Extra Provider-Specific Metadata #10715

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 48 additions & 25 deletions test/e2e/clusterclass_rollout.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ type ClusterClassRolloutSpecInput struct {
// Allows to inject a function to be run after test namespace is created.
// If not specified, this is a no-op.
PostNamespaceCreated func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string)

// FilterMetadataBeforeValidation allows filtering out labels and annotations of Machines, InfraMachines,
// BootstrapConfigs and Nodes before we validate them.
// This can be e.g. used to filter out additional infrastructure provider specific labels that would
// otherwise lead to a failed test.
FilterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta
}

// ClusterClassRolloutSpec implements a test that verifies the ClusterClass rollout behavior.
Expand Down Expand Up @@ -111,6 +117,14 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
Expect(input.E2EConfig.Variables).To(HaveValidVersion(input.E2EConfig.GetVariable(KubernetesVersion)))

// Set a default function to ensure that FilterMetadataBeforeValidation has a default behavior for
// filtering metadata if it is not specified by infrastructure provider.
if input.FilterMetadataBeforeValidation == nil {
input.FilterMetadataBeforeValidation = func(obj client.Object) clusterv1.ObjectMeta {
return clusterv1.ObjectMeta{Labels: obj.GetLabels(), Annotations: obj.GetAnnotations()}
}
}

// Set up a Namespace where to host objects for this spec and create a watcher for the namespace events.
namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated)
clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
Expand Down Expand Up @@ -142,7 +156,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
}, clusterResources)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)

By("Rolling out changes to control plane, MachineDeployments, and MachinePools (in-place)")
machinesBeforeUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
Expand Down Expand Up @@ -218,7 +232,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
g.Expect(machinesAfterUpgrade.Equal(machinesBeforeUpgrade)).To(BeTrue(), "Machines must not be replaced through in-place rollout")
}, 30*time.Second, 1*time.Second).Should(Succeed())
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)

By("Rolling out changes to control plane, MachineDeployments, and MachinePools (rollout)")
machinesBeforeUpgrade = getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
Expand Down Expand Up @@ -259,7 +273,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
g.Expect(machinesAfterUpgrade.HasAny(machinesBeforeUpgrade.UnsortedList()...)).To(BeFalse(), "All Machines must be replaced through rollout")
}, input.E2EConfig.GetIntervals(specName, "wait-control-plane")...).Should(Succeed())
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)

By("Rolling out control plane and MachineDeployment (rolloutAfter)")
machinesBeforeUpgrade = getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
Expand Down Expand Up @@ -287,7 +301,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
g.Expect(machinesAfterUpgrade.HasAny(machinesBeforeUpgrade.UnsortedList()...)).To(BeFalse(), "All Machines must be replaced through rollout with rolloutAfter")
}, input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")...).Should(Succeed())
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)

By("PASSED!")
})
Expand All @@ -299,7 +313,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
}

// assertClusterObjects asserts cluster objects by checking that all objects have the right labels, annotations and selectors.
func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterProxy, cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass) {
func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterProxy, cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) {
By("Checking cluster objects have the right labels, annotations and selectors")

Eventually(func(g Gomega) {
Expand All @@ -312,12 +326,12 @@ func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterPro

// ControlPlane
assertControlPlane(g, clusterClassObjects, clusterObjects, cluster, clusterClass)
assertControlPlaneMachines(g, clusterObjects, cluster)
assertControlPlaneMachines(g, clusterObjects, cluster, filterMetadataBeforeValidation)

// MachineDeployments
assertMachineDeployments(g, clusterClassObjects, clusterObjects, cluster, clusterClass)
assertMachineSets(g, clusterObjects, cluster)
assertMachineSetsMachines(g, clusterObjects, cluster)
assertMachineSetsMachines(g, clusterObjects, cluster, filterMetadataBeforeValidation)

// MachinePools
assertMachinePools(g, clusterClassObjects, clusterObjects, cluster, clusterClass)
Expand Down Expand Up @@ -430,13 +444,14 @@ func assertControlPlane(g Gomega, clusterClassObjects clusterClassObjects, clust
))
}

func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster) {
func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) {
controlPlaneMachineTemplateMetadata := mustMetadata(contract.ControlPlane().MachineTemplate().Metadata().Get(clusterObjects.ControlPlane))
controlPlaneInfrastructureMachineTemplateTemplateMetadata := mustMetadata(contract.InfrastructureMachineTemplate().Template().Metadata().Get(clusterObjects.ControlPlaneInfrastructureMachineTemplate))

for _, machine := range clusterObjects.ControlPlaneMachines {
// ControlPlane Machine.metadata
g.Expect(machine.Labels).To(BeEquivalentTo(
machineMetadata := filterMetadataBeforeValidation(machine)
g.Expect(machineMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -449,17 +464,18 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster
))
g.Expect(
union(
machine.Annotations,
machineMetadata.Annotations,
).without(g, controlplanev1.KubeadmClusterConfigurationAnnotation),
).To(BeEquivalentTo(
controlPlaneMachineTemplateMetadata.Annotations,
))

// ControlPlane Machine InfrastructureMachine.metadata
infrastructureMachine := clusterObjects.InfrastructureMachineByMachine[machine.Name]
infrastructureMachineMetadata := filterMetadataBeforeValidation(infrastructureMachine)
controlPlaneMachineTemplateInfrastructureRef, err := contract.ControlPlane().MachineTemplate().InfrastructureRef().Get(clusterObjects.ControlPlane)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(infrastructureMachine.GetLabels()).To(BeEquivalentTo(
g.Expect(infrastructureMachineMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -471,7 +487,7 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster
controlPlaneInfrastructureMachineTemplateTemplateMetadata.Labels,
),
))
g.Expect(infrastructureMachine.GetAnnotations()).To(BeEquivalentTo(
g.Expect(infrastructureMachineMetadata.Annotations).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(controlPlaneMachineTemplateInfrastructureRef),
Expand All @@ -484,7 +500,8 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster

// ControlPlane Machine BootstrapConfig.metadata
bootstrapConfig := clusterObjects.BootstrapConfigByMachine[machine.Name]
g.Expect(bootstrapConfig.GetLabels()).To(BeEquivalentTo(
bootstrapConfigMetadata := filterMetadataBeforeValidation(bootstrapConfig)
g.Expect(bootstrapConfigMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -497,16 +514,18 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster
))
g.Expect(
union(
bootstrapConfig.GetAnnotations(),
bootstrapConfigMetadata.Annotations,
).without(g, clusterv1.MachineCertificatesExpiryDateAnnotation),
).To(BeEquivalentTo(
controlPlaneMachineTemplateMetadata.Annotations,
))

// ControlPlane Machine Node.metadata
node := clusterObjects.NodesByMachine[machine.Name]
for k, v := range getManagedLabels(machine.Labels) {
g.Expect(node.GetLabels()).To(HaveKeyWithValue(k, v))
nodeMetadata := filterMetadataBeforeValidation(node)

for k, v := range getManagedLabels(machineMetadata.Labels) {
g.Expect(nodeMetadata.Labels).To(HaveKeyWithValue(k, v))
}
}
}
Expand Down Expand Up @@ -790,7 +809,7 @@ func assertMachineSets(g Gomega, clusterObjects clusterObjects, cluster *cluster
}
}

func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster) {
func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) {
for _, machineDeployment := range clusterObjects.MachineDeployments {
mdTopology := getMDTopology(cluster, machineDeployment)
infrastructureMachineTemplate := clusterObjects.InfrastructureMachineTemplateByMachineDeployment[machineDeployment.Name]
Expand All @@ -802,8 +821,9 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
machineTemplateHash := machineSet.Labels[clusterv1.MachineDeploymentUniqueLabel]

for _, machine := range clusterObjects.MachinesByMachineSet[machineSet.Name] {
machineMetadata := filterMetadataBeforeValidation(machine)
// MachineDeployment MachineSet Machine.metadata
g.Expect(machine.Labels).To(BeEquivalentTo(
g.Expect(machineMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -816,13 +836,14 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
machineSet.Spec.Template.Labels,
),
))
g.Expect(machine.Annotations).To(BeEquivalentTo(
g.Expect(machineMetadata.Annotations).To(BeEquivalentTo(
machineSet.Spec.Template.Annotations,
))

// MachineDeployment MachineSet Machine InfrastructureMachine.metadata
infrastructureMachine := clusterObjects.InfrastructureMachineByMachine[machine.Name]
g.Expect(infrastructureMachine.GetLabels()).To(BeEquivalentTo(
infrastructureMachineMetadata := filterMetadataBeforeValidation(infrastructureMachine)
g.Expect(infrastructureMachineMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -836,7 +857,7 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
infrastructureMachineTemplateTemplateMetadata.Labels,
),
))
g.Expect(infrastructureMachine.GetAnnotations()).To(BeEquivalentTo(
g.Expect(infrastructureMachineMetadata.Annotations).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(&machineSet.Spec.Template.Spec.InfrastructureRef),
Expand All @@ -849,7 +870,8 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster

// MachineDeployment MachineSet Machine BootstrapConfig.metadata
bootstrapConfig := clusterObjects.BootstrapConfigByMachine[machine.Name]
g.Expect(bootstrapConfig.GetLabels()).To(BeEquivalentTo(
bootstrapConfigMetadata := filterMetadataBeforeValidation(bootstrapConfig)
g.Expect(bootstrapConfigMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -863,7 +885,7 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
bootstrapConfigTemplateTemplateMetadata.Labels,
),
))
g.Expect(bootstrapConfig.GetAnnotations()).To(BeEquivalentTo(
g.Expect(bootstrapConfigMetadata.Annotations).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(machineSet.Spec.Template.Spec.Bootstrap.ConfigRef),
Expand All @@ -876,8 +898,9 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster

// MachineDeployment MachineSet Machine Node.metadata
node := clusterObjects.NodesByMachine[machine.Name]
for k, v := range getManagedLabels(machine.Labels) {
g.Expect(node.GetLabels()).To(HaveKeyWithValue(k, v))
nodeMetadata := filterMetadataBeforeValidation(node)
for k, v := range getManagedLabels(machineMetadata.Labels) {
g.Expect(nodeMetadata.Labels).To(HaveKeyWithValue(k, v))
}
}
}
Expand Down
Loading