From c8df53200b0638b4683b6b45ecdd37a5fb879c05 Mon Sep 17 00:00:00 2001
From: Christian Schlotter <christian.schlotter@broadcom.com>
Date: Mon, 20 Jan 2025 16:35:51 +0100
Subject: [PATCH 1/2] Update ControlPlaneKubeletLocalMode test for the feature
 gate being disabled

---
 ...ontrol-plane-local-kubelet-mode-tasks.yaml | 39 ++++++++++---------
 .../control-plane-local-kubelet-mode.yaml     |  4 +-
 ...ntrol-plane-local-kubelet-mode-latest.yaml |  4 +-
 ...ontrol-plane-local-kubelet-mode-tasks.yaml | 39 ++++++++++---------
 4 files changed, 44 insertions(+), 42 deletions(-)

diff --git a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml
index a0b4f790..ac12e730 100644
--- a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml
+++ b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml
@@ -2,7 +2,7 @@
 version: 1
 summary: |
   This workflow implements a sequence of tasks used test the proper functioning
-  of the ControlPlaneKubeletLocalMode feature gate.
+  of having the ControlPlaneKubeletLocalMode feature gate set to false.
 vars:
   # vars defines default values for variable used by tasks in this workflow;
   # those values might be overridden when importing this files.
@@ -59,19 +59,19 @@ tasks:
   - --name={{ .vars.clusterName }}
   - --loglevel=debug
   - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
-  - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=true"
+  - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=false"
   - --copy-certs=auto
   timeout: 5m
 - name: post-init
   description: |
     Run commands after kubeadm init is called on a primary CP node to checks if
-    the kubelet's kubeconfig file points to the local apiserver.
+    the kubelet's kubeconfig file points to load balanced apiserver.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
 
       # Ensure kubelet.conf points to the local IP.
@@ -95,22 +95,23 @@ tasks:
 - name: post-join
   description: |
     Run commands after kubeadm join is called on all joined CP node to checks if
-    the kubelet's kubeconfig file points to the local apiserver.
+    the kubelet's kubeconfig file points to the remote apiserver.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
+
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)"
+
       CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
 
-      # Ensure kubelet.conf points to the local IP.
+      # Ensure kubelet.conf points to the remote IP.
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
 
-      # Ensure kubelet.conf points to the local IP.
+      # Ensure kubelet.conf points to the remote IP.
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
       # Ensure exit status of 0
@@ -119,23 +120,24 @@ tasks:
 - name: pre-upgrade
   description: |
     Run commands before kubeadm upgrade is called on all joined CP node to replace
-    the server in the kubelet's kubeconfig to point to the load balancer.
+    the server in the kubelet's kubeconfig to point to the control-plane IP.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
 
-      LOAD_BALANCER_IP_ADDRESS=$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)
-
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
-      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
 
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
-      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
 
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
-      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
 
       # Ensure exit status of 0
       exit 0
@@ -155,22 +157,21 @@ tasks:
 - name: post-upgrade
   description: |
     Run commands after kubeadm upgrade is called on all joined CP node to checks if
-    the kubelet's kubeconfig file points to the local apiserver.
+    the kubelet's kubeconfig file points to the remote apiserver.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)"
+
       CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
diff --git a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml
index a9d1038c..89d400f1 100644
--- a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml
+++ b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml
@@ -1,7 +1,7 @@
 version: 1
 summary: |
-  This workflow tests the proper functioning of the {{ .KubernetesVersion }} version of both kubeadm and Kubernetes using
-  the ControlPlaneKubeletLocalMode feature gate.
+  This workflow tests the proper functioning of the {{ .KubernetesVersion }} version of both kubeadm and Kubernetes having
+  the ControlPlaneKubeletLocalMode feature gate set to false.
   test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-control-plane-local-kubelet-mode{{ dashVer .KubernetesVersion }}
   config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }}
 vars:
diff --git a/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml b/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml
index 4ecc6dfb..e58c05ea 100644
--- a/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml
+++ b/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml
@@ -1,8 +1,8 @@
 # AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows
 version: 1
 summary: |
-  This workflow tests the proper functioning of the latest version of both kubeadm and Kubernetes using
-  the ControlPlaneKubeletLocalMode feature gate.
+  This workflow tests the proper functioning of the latest version of both kubeadm and Kubernetes having
+  the ControlPlaneKubeletLocalMode feature gate set to false.
   test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-control-plane-local-kubelet-modelatest
   config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/kubeadm-kinder-control-plane-local-kubelet-mode.yaml
 vars:
diff --git a/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml b/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml
index 620bf745..dae2cedb 100644
--- a/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml
+++ b/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml
@@ -3,7 +3,7 @@
 version: 1
 summary: |
   This workflow implements a sequence of tasks used test the proper functioning
-  of the ControlPlaneKubeletLocalMode feature gate.
+  of having the ControlPlaneKubeletLocalMode feature gate set to false.
 vars:
   # vars defines default values for variable used by tasks in this workflow;
   # those values might be overridden when importing this files.
@@ -60,19 +60,19 @@ tasks:
   - --name={{ .vars.clusterName }}
   - --loglevel=debug
   - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
-  - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=true"
+  - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=false"
   - --copy-certs=auto
   timeout: 5m
 - name: post-init
   description: |
     Run commands after kubeadm init is called on a primary CP node to checks if
-    the kubelet's kubeconfig file points to the local apiserver.
+    the kubelet's kubeconfig file points to load balanced apiserver.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
 
       # Ensure kubelet.conf points to the local IP.
@@ -96,22 +96,23 @@ tasks:
 - name: post-join
   description: |
     Run commands after kubeadm join is called on all joined CP node to checks if
-    the kubelet's kubeconfig file points to the local apiserver.
+    the kubelet's kubeconfig file points to the remote apiserver.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
+
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)"
+
       CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
 
-      # Ensure kubelet.conf points to the local IP.
+      # Ensure kubelet.conf points to the remote IP.
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
 
-      # Ensure kubelet.conf points to the local IP.
+      # Ensure kubelet.conf points to the remote IP.
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
       # Ensure exit status of 0
@@ -120,23 +121,24 @@ tasks:
 - name: pre-upgrade
   description: |
     Run commands before kubeadm upgrade is called on all joined CP node to replace
-    the server in the kubelet's kubeconfig to point to the load balancer.
+    the server in the kubelet's kubeconfig to point to the control-plane IP.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
 
-      LOAD_BALANCER_IP_ADDRESS=$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)
-
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
-      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
 
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
-      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
 
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
-      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
 
       # Ensure exit status of 0
       exit 0
@@ -156,22 +158,21 @@ tasks:
 - name: post-upgrade
   description: |
     Run commands after kubeadm upgrade is called on all joined CP node to checks if
-    the kubelet's kubeconfig file points to the local apiserver.
+    the kubelet's kubeconfig file points to the remote apiserver.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)"
+
       CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 

From e9f1704bfffd1b107f7520959c2b77c0b3731577 Mon Sep 17 00:00:00 2001
From: Christian Schlotter <christian.schlotter@broadcom.com>
Date: Tue, 4 Feb 2025 08:46:25 +0100
Subject: [PATCH 2/2] kinder: fix setup-external-ca to still generate valid
 worker kubeconfigs

---
 .../cluster/manager/actions/setup-external-ca.go   | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/kinder/pkg/cluster/manager/actions/setup-external-ca.go b/kinder/pkg/cluster/manager/actions/setup-external-ca.go
index 01237553..1c63d634 100644
--- a/kinder/pkg/cluster/manager/actions/setup-external-ca.go
+++ b/kinder/pkg/cluster/manager/actions/setup-external-ca.go
@@ -72,6 +72,18 @@ func SetupExternalCA(c *status.Cluster, vLevel int) error {
 		return nil
 	}
 
+	generateKubeletConfWorker := func(n *status.Node) error {
+		if err := n.Command(
+			"/bin/sh", "-c",
+			fmt.Sprintf("kubeadm init phase kubeconfig kubelet --control-plane-endpoint=%s --apiserver-advertise-address=%s --v=%d",
+				loadBalancerIP, loadBalancerIP,
+				vLevel),
+		).RunWithEcho(); err != nil {
+			return errors.Wrapf(err, "could not generate a kubelet.conf on node: %s", n.Name())
+		}
+		return nil
+	}
+
 	// iterate secondary CP nodes
 	for _, n := range c.SecondaryControlPlanes() {
 		// copy the shared kubeconfig files
@@ -108,7 +120,7 @@ func SetupExternalCA(c *status.Cluster, vLevel int) error {
 		}
 
 		// generate kubelet.conf
-		if err := generateKubeletConf(n); err != nil {
+		if err := generateKubeletConfWorker(n); err != nil {
 			return err
 		}
 	}