Skip to content

Commit

Permalink
Test n-3 clusterctl upgrade
Browse files Browse the repository at this point in the history
Signed-off-by: Stefan Büringer buringerst@vmware.com
  • Loading branch information
sbueringer committed Feb 21, 2025
1 parent 0d4ceea commit 0a7ae63
Show file tree
Hide file tree
Showing 13 changed files with 1,067 additions and 9 deletions.
7 changes: 6 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -533,7 +533,7 @@ generate-doctoc:
TRACE=$(TRACE) ./hack/generate-doctoc.sh

.PHONY: generate-e2e-templates
generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v0.3 v0.4 v1.5 v1.6 v1.8 v1.9 main) ## Generate cluster templates for all versions
generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v0.3 v0.4 v1.5 v1.6 v1.7 v1.8 v1.9 main) ## Generate cluster templates for all versions

DOCKER_TEMPLATES := test/e2e/data/infrastructure-docker

Expand All @@ -555,6 +555,11 @@ generate-e2e-templates-v1.6: $(KUSTOMIZE)
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.6/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.6/cluster-template.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.6/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.6/cluster-template-topology.yaml

.PHONY: generate-e2e-templates-v1.7
generate-e2e-templates-v1.7: $(KUSTOMIZE)
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.7/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.7/cluster-template.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.7/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.7/cluster-template-topology.yaml

.PHONY: generate-e2e-templates-v1.8
generate-e2e-templates-v1.8: $(KUSTOMIZE)
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.8/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.8/cluster-template.yaml
Expand Down
11 changes: 6 additions & 5 deletions docs/book/src/clusterctl/commands/upgrade.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,13 @@ Cluster API only tests a subset of possible clusterctl upgrade paths as otherwis
Untested upgrade paths are not blocked by clusterctl and should work in general, but users
intending to perform an upgrade path not tested by us should do their own validation to ensure the operation works correctly.

The following is an example of the tested upgrade paths for v1.7:
The following is an example of the tested upgrade paths for v1.10:

| From | To | Note |
|------|------|------------------------------|
| v1.5 | v1.7 | n-2 --> n (v1.5 is v1.7 - 2) |
| v1.6 | v1.7 | n-1 --> n (v1.6 is v1.7 - 1) |
| From | To | Note |
|------|-------|-------------------------------|
| v1.7 | v1.10 | n-3 --> n (v1.7 is v1.10 - 3) |
| v1.8 | v1.10 | n-2 --> n (v1.8 is v1.10 - 2) |
| v1.9 | v1.10 | n-1 --> n (v1.9 is v1.10 - 1) |

</aside>

Expand Down
38 changes: 35 additions & 3 deletions test/e2e/clusterctl_upgrade_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,38 @@ var _ = Describe("When testing clusterctl upgrades (v0.4=>v1.6=>current)", Flake
})
})

// Note: This test should be changed during "prepare main branch", it should test n-3 => current.
var _ = Describe("When testing clusterctl upgrades using ClusterClass (v1.7=>current) [ClusterClass]", Label("ClusterClass"), func() {
// Get n-3 latest stable release
version := "1.7"
stableRelease, err := GetStableReleaseOfMinor(ctx, version)
Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", version)
ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
return ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, stableRelease),
// We have to pin the providers because with `InitWithProvidersContract` the test would
// use the latest version for the contract (which is the next minor for v1beta1).
InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, stableRelease),
InitWithBootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease)},
InitWithControlPlaneProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease)},
InitWithInfrastructureProviders: []string{fmt.Sprintf(providerDockerPrefix, stableRelease)},
InitWithProvidersContract: "v1beta1",
// Note: Both InitWithKubernetesVersion and WorkloadKubernetesVersion should be the highest mgmt cluster version supported by the source Cluster API version.
// When picking this version, please check also the list of versions known by the source Cluster API version.
InitWithKubernetesVersion: "v1.30.0",
WorkloadKubernetesVersion: "v1.30.0",
MgmtFlavor: "topology",
WorkloadFlavor: "topology",
UseKindForManagementCluster: true,
}
})
})

// Note: This test should be changed during "prepare main branch", it should test n-2 => current.
var _ = Describe("When testing clusterctl upgrades using ClusterClass (v1.8=>current) [ClusterClass]", Label("ClusterClass"), func() {
// Get n-2 latest stable release
Expand All @@ -201,7 +233,7 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (v1.8=>cur
InitWithInfrastructureProviders: []string{fmt.Sprintf(providerDockerPrefix, stableRelease)},
InitWithProvidersContract: "v1beta1",
// Note: Both InitWithKubernetesVersion and WorkloadKubernetesVersion should be the highest mgmt cluster version supported by the source Cluster API version.
// When picking this version, please check also the list of versions known by the source Cluster API version.
// When picking this version, please check also the list of versions known by the source Cluster API version.
InitWithKubernetesVersion: "v1.31.0",
WorkloadKubernetesVersion: "v1.31.0",
MgmtFlavor: "topology",
Expand All @@ -227,7 +259,7 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (v1.9=>cur
InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, stableRelease),
InitWithProvidersContract: "v1beta1",
// Note: Both InitWithKubernetesVersion and WorkloadKubernetesVersion should be the highest mgmt cluster version supported by the source Cluster API version.
// When picking this version, please check also the list of versions known by the source Cluster API version.
// When picking this version, please check also the list of versions known by the source Cluster API version.
InitWithKubernetesVersion: "v1.32.0",
WorkloadKubernetesVersion: "v1.32.0",
MgmtFlavor: "topology",
Expand Down Expand Up @@ -256,7 +288,7 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (v1.9=>cur
InitWithProvidersContract: "v1beta1",
// Note: InitWithKubernetesVersion should be the latest of the next supported kubernetes version by the target Cluster API version.
// Note: WorkloadKubernetesVersion should be the highest mgmt cluster version supported by the source Cluster API version.
// When picking this version, please check also the list of versions known by the source Cluster API version.
// When picking this version, please check also the list of versions known by the source Cluster API version.
InitWithKubernetesVersion: initKubernetesVersion,
WorkloadKubernetesVersion: "v1.32.0",
MgmtFlavor: "topology",
Expand Down
39 changes: 39 additions & 0 deletions test/e2e/config/docker.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,15 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.6/metadata.yaml"
- name: "{go://sigs.k8s.io/cluster-api@v1.7}"
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v1.7}/core-components.yaml"
type: "url"
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.7/metadata.yaml"
- name: "{go://sigs.k8s.io/cluster-api@v1.8}"
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v1.8}/core-components.yaml"
type: "url"
Expand Down Expand Up @@ -131,6 +140,15 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.6/metadata.yaml"
- name: "{go://sigs.k8s.io/cluster-api@v1.7}"
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v1.7}/bootstrap-components.yaml"
type: "url"
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.7/metadata.yaml"
- name: "{go://sigs.k8s.io/cluster-api@v1.8}"
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v1.8}/bootstrap-components.yaml"
type: "url"
Expand Down Expand Up @@ -193,6 +211,15 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.6/metadata.yaml"
- name: "{go://sigs.k8s.io/cluster-api@v1.7}"
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v1.7}/control-plane-components.yaml"
type: "url"
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.7/metadata.yaml"
- name: "{go://sigs.k8s.io/cluster-api@v1.8}"
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v1.8}/control-plane-components.yaml"
type: "url"
Expand Down Expand Up @@ -263,6 +290,18 @@ providers:
- sourcePath: "../data/infrastructure-docker/v1.6/cluster-template.yaml"
- sourcePath: "../data/infrastructure-docker/v1.6/cluster-template-topology.yaml"
- sourcePath: "../data/infrastructure-docker/v1.6/clusterclass-quick-start.yaml"
- name: "{go://sigs.k8s.io/cluster-api@v1.7}"
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v1.7}/infrastructure-components-development.yaml"
type: "url"
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.7/metadata.yaml"
- sourcePath: "../data/infrastructure-docker/v1.7/cluster-template.yaml"
- sourcePath: "../data/infrastructure-docker/v1.7/cluster-template-topology.yaml"
- sourcePath: "../data/infrastructure-docker/v1.7/clusterclass-quick-start.yaml"
- name: "{go://sigs.k8s.io/cluster-api@v1.8}"
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/cluster-api@v1.8}/infrastructure-components-development.yaml"
type: "url"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
---
# DockerCluster object referenced by the Cluster object
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerCluster
metadata:
name: '${CLUSTER_NAME}'
spec:
failureDomains:
fd1:
controlPlane: true
fd2:
controlPlane: true
fd3:
controlPlane: true
fd4:
controlPlane: false
fd5:
controlPlane: false
fd6:
controlPlane: false
fd7:
controlPlane: false
fd8:
controlPlane: false
---
# Cluster object with
# - Reference to the KubeadmControlPlane object
# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet.
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
name: '${CLUSTER_NAME}'
labels:
cni: "${CLUSTER_NAME}-crs-0"
spec:
clusterNetwork:
services:
cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
pods:
cidrBlocks: ['${DOCKER_POD_CIDRS}']
serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerCluster
name: '${CLUSTER_NAME}'
controlPlaneRef:
kind: KubeadmControlPlane
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
name: "${CLUSTER_NAME}-control-plane"
---
# DockerMachineTemplate object referenced by the KubeadmControlPlane object
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
metadata:
name: "${CLUSTER_NAME}-control-plane"
spec:
template:
spec:
extraMounts:
- containerPath: "/var/run/docker.sock"
hostPath: "/var/run/docker.sock"
# The DOCKER_PRELOAD_IMAGES variable gets set in self-hosted E2E tests to the list of images of the E2E configuration.
preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]}
---
# KubeadmControlPlane referenced by the Cluster object with
# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test.
kind: KubeadmControlPlane
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
metadata:
name: "${CLUSTER_NAME}-control-plane"
labels:
kcp-adoption.step2: ""
spec:
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
machineTemplate:
infrastructureRef:
kind: DockerMachineTemplate
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
name: "${CLUSTER_NAME}-control-plane"
kubeadmConfigSpec:
clusterConfiguration:
apiServer:
# host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
initConfiguration:
nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
joinConfiguration:
nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
version: "${KUBERNETES_VERSION}"
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
name: '${CLUSTER_NAME}'
namespace: default
labels:
cni: "${CLUSTER_NAME}-crs-0"
spec:
clusterNetwork:
services:
cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
pods:
cidrBlocks: ['${DOCKER_POD_CIDRS}']
serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
topology:
class: "quick-start"
version: "${KUBERNETES_VERSION}"
controlPlane:
metadata:
labels:
Cluster.topology.controlPlane.label: "Cluster.topology.controlPlane.labelValue"
# Note: this label is propagated to Nodes.
Cluster.topology.controlPlane.label.node.cluster.x-k8s.io: "Cluster.topology.controlPlane.nodeLabelValue"
annotations:
Cluster.topology.controlPlane.annotation: "Cluster.topology.controlPlane.annotationValue"
nodeDeletionTimeout: "30s"
nodeVolumeDetachTimeout: "5m"
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
workers:
machineDeployments:
- class: "default-worker"
name: "md-0"
metadata:
labels:
Cluster.topology.machineDeployment.label: "Cluster.topology.machineDeployment.labelValue"
# Note: this label is propagated to Nodes.
Cluster.topology.machineDeployment.label.node.cluster.x-k8s.io: "Cluster.topology.machineDeployment.nodeLabelValue"
annotations:
Cluster.topology.machineDeployment.annotation: "Cluster.topology.machineDeployment.annotationValue"
nodeDeletionTimeout: "30s"
nodeVolumeDetachTimeout: "5m"
minReadySeconds: 5
replicas: ${WORKER_MACHINE_COUNT}
failureDomain: fd4
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: "20%"
maxUnavailable: 0
machinePools:
- class: "default-worker"
name: "mp-0"
metadata:
labels:
Cluster.topology.machinePool.label: "Cluster.topology.machinePool.labelValue"
# Note: this label is propagated to Nodes.
Cluster.topology.machinePool.label.node.cluster.x-k8s.io: "Cluster.topology.machinePool.nodeLabelValue"
annotations:
Cluster.topology.machinePool.annotation: "Cluster.topology.machinePool.annotationValue"
nodeDeletionTimeout: "30s"
nodeVolumeDetachTimeout: "5m"
minReadySeconds: 5
replicas: ${WORKER_MACHINE_COUNT}
failureDomains:
- fd4
variables:
# We set an empty value to use the default tag kubeadm init is using.
- name: etcdImageTag
value: ""
# We set an empty value to use the default tag kubeadm init is using.
- name: coreDNSImageTag
value: ""
- name: preLoadImages
# The DOCKER_PRELOAD_IMAGES variable gets set in self-hosted E2E tests to the list of images of the E2E configuration.
value: ${DOCKER_PRELOAD_IMAGES:-[]}
24 changes: 24 additions & 0 deletions test/e2e/data/infrastructure-docker/v1.7/bases/crs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
---
# ConfigMap object referenced by the ClusterResourceSet object and with
# the CNI resource defined in the test config file
apiVersion: v1
kind: ConfigMap
metadata:
name: "cni-${CLUSTER_NAME}-crs-0"
data: ${CNI_RESOURCES}
binaryData:
---
# ClusterResourceSet object with
# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0
apiVersion: addons.cluster.x-k8s.io/v1beta1
kind: ClusterResourceSet
metadata:
name: "${CLUSTER_NAME}-crs-0"
spec:
strategy: ApplyOnce
clusterSelector:
matchLabels:
cni: "${CLUSTER_NAME}-crs-0"
resources:
- name: "cni-${CLUSTER_NAME}-crs-0"
kind: ConfigMap
Loading

0 comments on commit 0a7ae63

Please sign in to comment.