upgrade to K8s v1.10.x (#10)

* removed tls-cert-file setting as it is deprecated (see kubernetes/kubernetes#58968)

* remove deprecated kube-apiserver parameter insecure-bind-address / added variable+parameter secure-port for kube-apiserver

* added kube-controller-manager certificate files to k8s_certificates list

* update README

* added kube-scheduler certificate files to k8s_certificates list

* updated README

* added kube-controller-manager-sa certificate files to k8s_certificates list

* removed unneeded files from k8s_certificates list

* change K8s release to v1.10.3

* added kubeconfig for admin user, controller manager, scheduler / introduce k8s_scheduler_conf_dir, k8s_controller_manager_conf_dir / k8s_release to v1.10.3 / update README

* replaced kube-apiserver parameter admission-control with enable-admission-plugins

* new service-account-key-file value for kube-apiserver

* changes in k8s_controller_manager_settings: removed master parameter, added kubeconfig, new value for service-account-private-key-file, new parameter use-service-account-credentials

* update to K8s v1.10.4

* update README

* implemented needed changes for kube-scheduler

* kubectl now uses --kubeconfig

* fix wrong directory destinations

* added k8s-controller-base tag / fixed wrong tags

Author: githubixx

README.md

@@ -16,6 +16,19 @@ This role requires that you already created some certificates for Kubernetes API
 Changelog
 ---------
 
+**r4.0.0_v1.10.4**
+
+- update `k8s_release` to `1.10.4`
+- removed deprecated kube-apiserver parameter `insecure-bind-address` (see: [#59018](https://github.com/kubernetes/kubernetes/pull/59018))
+- added variable `k8s_apiserver_secure_port: 6443`
+- added parameter `secure-port` to `k8s_apiserver_settings` parameter list
+- added `kube-controller-manager-ca` certificate files to `k8s_certificates` list
+- added variable `k8s_controller_manager_conf_dir` / added kubeconfig for kube-controller-manager
+- added variable `k8s_scheduler_conf_dir` / added kubeconfig for kube-scheduler / settings for kube-scheduler now in ` templates/var/lib/kube-scheduler/kube-scheduler.yaml.j2`
+- added kubeconfig for `admin` user (located by default in `k8s_conf_dir`). This `admin.kubeconfig` will be needed for `kubectl`
+- new `service-account-key-file` value for kube-apiserver
+- changes in `k8s_controller_manager_settings`: removed `master` parameter, added `kubeconfig`, new value for `service-account-private-key-file`, new parameter `use-service-account-credentials`
+
 **r3.0.0_v1.9.8**
 
 - update `k8s_release` to `1.9.8`
@@ -61,7 +74,7 @@ k8s_conf_dir: "/var/lib/kubernetes"
 # The directory to store the K8s binaries
 k8s_bin_dir: "/usr/local/bin"
 # K8s release
-k8s_release: "1.9.8"
+k8s_release: "1.10.4"
 # The interface on which the K8s services should listen on. As all cluster
 # communication should use the PeerVPN interface the interface name is
 # normally "tap0" or "peervpn0".
@@ -91,14 +104,18 @@ k8s_certificates:
   - ca-k8s-apiserver-key.pem
   - cert-k8s-apiserver.pem
   - cert-k8s-apiserver-key.pem
+  - cert-k8s-controller-manager-sa.pem
+  - cert-k8s-controller-manager-sa-key.pem
+
+k8s_apiserver_secure_port: "6443"
 
 # kube-apiserver settings (can be overriden or additional added by defining
 # "k8s_apiserver_settings_user" - see text below)
 k8s_apiserver_settings:
   "advertise-address": "hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address"
   "bind-address": "hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address"
-  "insecure-bind-address": "hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address"
-  "admission-control": "Initializers,NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
+  "secure-port": "{{k8s_apiserver_secure_port}}"
+  "enable-admission-plugins": "Initializers,NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
   "allow-privileged": "true"
   "apiserver-count": "3"
   "authorization-mode": "Node,RBAC"
@@ -121,35 +138,36 @@ k8s_apiserver_settings:
   "kubelet-certificate-authority": "{{k8s_conf_dir}}/ca-k8s-apiserver.pem"
   "kubelet-client-certificate": "{{k8s_conf_dir}}/cert-k8s-apiserver.pem"
   "kubelet-client-key": "{{k8s_conf_dir}}/cert-k8s-apiserver-key.pem"
-  "service-account-key-file": "{{k8s_conf_dir}}/cert-k8s-apiserver-key.pem"
+  "service-account-key-file": "{{k8s_conf_dir}}/cert-k8s-controller-manager-sa.pem"
   "tls-ca-file": "{{k8s_conf_dir}}/ca-k8s-apiserver.pem"
   "tls-cert-file": "{{k8s_conf_dir}}/cert-k8s-apiserver.pem"
   "tls-private-key-file": "{{k8s_conf_dir}}/cert-k8s-apiserver-key.pem"
 
 # The directory to store controller manager configuration.
-k8s_controller_manager_conf_dir: "{{k8s_conf_dir}}"
+k8s_controller_manager_conf_dir: "/var/lib/kube-controller-manager"
 
 # kube-controller-manager settings (can be overriden or additional added by defining
 # "k8s_controller_manager_settings_user" - see text below)
 k8s_controller_manager_settings:
   "address": "{{hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address}}"
-  "master": "{{'http://' + hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address + ':8080'}}"
   "cluster-cidr": "10.200.0.0/16"
   "cluster-name": "kubernetes"
+  "kubeconfig": "{{k8s_controller_manager_conf_dir}}/kube-controller-manager.kubeconfig"
   "leader-elect": "true"
   "service-cluster-ip-range": "10.32.0.0/16"
-  "cluster-signing-cert-file": "{{k8s_controller_manager_conf_dir}}/ca-k8s-apiserver.pem"
-  "cluster-signing-key-file": "{{k8s_controller_manager_conf_dir}}/cert-k8s-apiserver-key.pem"
-  "root-ca-file": "{{k8s_controller_manager_conf_dir}}/ca-k8s-apiserver.pem"
-  "cluster-signing-cert-file": "{{k8s_controller_manager_conf_dir}}/ca-k8s-apiserver.pem"
-  "service-account-private-key-file": "{{k8s_controller_manager_conf_dir}}/cert-k8s-apiserver-key.pem"
-
-# kube-scheduler settings (can be overriden or additional added by defining
-# "k8s_scheduler_settings_user" - see text below)
+  "cluster-signing-cert-file": "{{k8s_conf_dir}}/ca-k8s-apiserver.pem"
+  "cluster-signing-key-file": "{{k8s_conf_dir}}/cert-k8s-apiserver-key.pem"
+  "root-ca-file": "{{k8s_conf_dir}}/ca-k8s-apiserver.pem"
+  "service-account-private-key-file": "{{k8s_conf_dir}}/cert-k8s-controller-manager-sa-key.pem"
+  "use-service-account-credentials": "true"
+
+# The directory to store scheduler configuration.
+k8s_scheduler_conf_dir: "/var/lib/kube-scheduler"
+
+# kube-scheduler settings (only --config left,
+# see https://github.com/kubernetes/kubernetes/pull/62515, remaining parameter deprecated)
 k8s_scheduler_settings:
-  "address": "{{hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address}}"
-  "master": "{{'http://' + hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address + ':8080'}}"
-  "leader-elect": "true"
+  "config": "{{k8s_scheduler_conf_dir}}/kube-scheduler.yaml"
 
 # The port the control plane componentes should connect to etcd cluster
 etcd_client_port: "2379"

defaults/main.yml

@@ -4,7 +4,7 @@ k8s_conf_dir: "/var/lib/kubernetes"
 # The directory to store the K8s binaries
 k8s_bin_dir: "/usr/local/bin"
 # K8s release
-k8s_release: "1.9.8"
+k8s_release: "1.10.4"
 # The interface on which the K8s services should listen on. As all cluster
 # communication should use the PeerVPN interface the interface name is
 # normally "tap0" or "peervpn0".
@@ -28,20 +28,24 @@ k8s_controller_binaries:
   - kube-scheduler
   - kubectl
 
-# K8s API daemon certificates
+# K8s kube-(apiserver|controller-manager-sa) certificates
 k8s_certificates:
   - ca-k8s-apiserver.pem
   - ca-k8s-apiserver-key.pem
   - cert-k8s-apiserver.pem
   - cert-k8s-apiserver-key.pem
+  - cert-k8s-controller-manager-sa.pem
+  - cert-k8s-controller-manager-sa-key.pem
+
+k8s_apiserver_secure_port: "6443"
 
 # K8s API daemon settings (can be overriden or additional added by defining
 # "k8s_apiserver_settings_user")
 k8s_apiserver_settings:
   "advertise-address": "{{hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address}}"
   "bind-address": "{{hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address}}"
-  "insecure-bind-address": "{{hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address}}"
-  "admission-control": "Initializers,NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
+  "secure-port": "{{k8s_apiserver_secure_port}}"
+  "enable-admission-plugins": "Initializers,NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
   "allow-privileged": "true"
   "apiserver-count": "3"
   "authorization-mode": "Node,RBAC"
@@ -66,32 +70,33 @@ k8s_apiserver_settings:
   "kubelet-certificate-authority": "{{k8s_conf_dir}}/ca-k8s-apiserver.pem"
   "kubelet-client-certificate": "{{k8s_conf_dir}}/cert-k8s-apiserver.pem"
   "kubelet-client-key": "{{k8s_conf_dir}}/cert-k8s-apiserver-key.pem"
-  "service-account-key-file": "{{k8s_conf_dir}}/cert-k8s-apiserver-key.pem"
-  "tls-ca-file": "{{k8s_conf_dir}}/ca-k8s-apiserver.pem"
+  "service-account-key-file": "{{k8s_conf_dir}}/cert-k8s-controller-manager-sa.pem"
   "tls-cert-file": "{{k8s_conf_dir}}/cert-k8s-apiserver.pem"
   "tls-private-key-file": "{{k8s_conf_dir}}/cert-k8s-apiserver-key.pem"
 
 # The directory to store controller manager configuration.
-k8s_controller_manager_conf_dir: "{{k8s_conf_dir}}"
+k8s_controller_manager_conf_dir: "/var/lib/kube-controller-manager"
 # K8s controller manager settings (can be overriden or additional added by defining
 # "k8s_controller_manager_settings_user")
 k8s_controller_manager_settings:
   "address": "{{hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address}}"
-  "master": "{{'http://' + hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address + ':8080'}}"
   "cluster-cidr": "10.200.0.0/16"
   "cluster-name": "kubernetes"
+  "kubeconfig": "{{k8s_controller_manager_conf_dir}}/kube-controller-manager.kubeconfig"
   "leader-elect": "true"
   "service-cluster-ip-range": "10.32.0.0/16"
-  "cluster-signing-cert-file": "{{k8s_controller_manager_conf_dir}}/ca-k8s-apiserver.pem"
-  "cluster-signing-key-file": "{{k8s_controller_manager_conf_dir}}/cert-k8s-apiserver-key.pem"
-  "root-ca-file": "{{k8s_controller_manager_conf_dir}}/ca-k8s-apiserver.pem"
-  "service-account-private-key-file": "{{k8s_controller_manager_conf_dir}}/cert-k8s-apiserver-key.pem"
+  "cluster-signing-cert-file": "{{k8s_conf_dir}}/ca-k8s-apiserver.pem"
+  "cluster-signing-key-file": "{{k8s_conf_dir}}/cert-k8s-apiserver-key.pem"
+  "root-ca-file": "{{k8s_conf_dir}}/ca-k8s-apiserver.pem"
+  "service-account-private-key-file": "{{k8s_conf_dir}}/cert-k8s-controller-manager-sa-key.pem"
+  "use-service-account-credentials": "true"
 
-# kube-scheduler settings
+# The directory to store scheduler configuration.
+k8s_scheduler_conf_dir: "/var/lib/kube-scheduler"
+# kube-scheduler settings (only --config left,
+# see https://github.com/kubernetes/kubernetes/pull/62515)
 k8s_scheduler_settings:
-  "address": "{{hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address}}"
-  "master": "{{'http://' + hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address + ':8080'}}"
-  "leader-elect": "true"
+  "config": "{{k8s_scheduler_conf_dir}}/kube-scheduler.yaml"
 
 # The port the control plane componentes should connect to etcd cluster
 etcd_client_port: "2379"

tasks/main.yml

@@ -1,5 +1,5 @@
 ---
-- name: Create Kubernetes config directory
+- name: Create Kubernetes/kube-apiserver config directory
   file:
     path: "{{k8s_conf_dir}}"
     state: directory
@@ -8,6 +8,73 @@
     group: root
   tags:
     - k8s-controller
+    - k8s-controller-base
+
+- name: Create kube-controller-manager config directory
+  file:
+    path: "{{k8s_controller_manager_conf_dir}}"
+    state: directory
+    mode: 0700
+    owner: root
+    group: root
+  tags:
+    - k8s-controller
+    - k8s-controller-base
+
+- name: Create kube-controller-manager kubeconfig
+  template:
+    src: "{{k8s_config_directory}}/kube-controller-manager.kubeconfig"
+    dest: "{{k8s_controller_manager_conf_dir}}/kube-controller-manager.kubeconfig"
+    owner: root
+    group: root
+    mode: 0644
+  tags:
+    - k8s-worker
+    - k8s-controller-base
+
+- name: Create scheduler config directory
+  file:
+    path: "{{k8s_scheduler_conf_dir}}"
+    state: directory
+    mode: 0700
+    owner: root
+    group: root
+  tags:
+    - k8s-controller
+    - k8s-controller-base
+
+- name: Create kube-scheduler kubeconfig
+  template:
+    src: "{{k8s_config_directory}}/kube-scheduler.kubeconfig"
+    dest: "{{k8s_scheduler_conf_dir}}/kube-scheduler.kubeconfig"
+    owner: root
+    group: root
+    mode: 0644
+  tags:
+    - k8s-controller
+    - k8s-controller-base
+
+- name: Create kube-scheduler.yaml
+  template:
+    src: "templates/var/lib/kube-scheduler/kube-scheduler.yaml.j2"
+    dest: "{{k8s_scheduler_conf_dir}}/kube-scheduler.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  tags:
+    - k8s-controller
+    - k8s-controller-base
+
+- name: Create kubeconfig for admin user
+  template:
+    src: "{{k8s_config_directory}}/admin.kubeconfig"
+    dest: "{{k8s_conf_dir}}/admin.kubeconfig"
+    owner: root
+    group: root
+    mode: 0644
+  tags:
+    - k8s-controller
+    - k8s-controller-base
 
 - name: Copy etcd certificates
   copy:
@@ -20,6 +87,7 @@
     - "{{etcd_certificates}}"
   tags:
     - k8s-controller
+    - k8s-controller-base
 
 - name: Copy Kubernetes certificates
   copy:
@@ -32,6 +100,7 @@
     - "{{k8s_certificates}}"
   tags:
     - k8s-controller
+    - k8s-controller-base
 
 - name: Downloading official Kubernetes binaries
   get_url:
@@ -56,6 +125,7 @@
     group: root
   tags:
     - k8s-controller
+    - k8s-controller-base
 
 - name: Combine k8s_apiserver_settings and k8s_apiserver_settings_user (if defined)
   set_fact:
@@ -174,15 +244,15 @@
   delegate_to: "{{groups.k8s_controller|first}}"
 
 - name: Apply kube-apiserver-to-kubelet ClusterRole
-  shell: "kubectl --server={{hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address}}:8080 apply -f /tmp/kube-apiserver-to-kubelet_cluster_role.yaml"
+  shell: "kubectl apply --kubeconfig {{k8s_conf_dir}}/admin.kubeconfig -f /tmp/kube-apiserver-to-kubelet_cluster_role.yaml"
   register: kube_apiserver_to_kubelet_cluster_role
   run_once: true
   delegate_to: "{{groups.k8s_controller|first}}"
   tags:
     - k8s-controller
 
 - name: Apply kube-apiserver-to-kubelet ClusterRoleBinding
-  shell: "kubectl --server={{hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address}}:8080 apply -f /tmp/kube-apiserver-to-kubelet_cluster_role_binding.yaml"
+  shell: "kubectl apply --kubeconfig {{k8s_conf_dir}}/admin.kubeconfig -f /tmp/kube-apiserver-to-kubelet_cluster_role_binding.yaml"
   register: kube_apiserver_to_kubelet_cluster_role_binding
   run_once: true
   delegate_to: "{{groups.k8s_controller|first}}"

templates/var/lib/kube-scheduler/kube-scheduler.yaml.j2

@@ -0,0 +1,9 @@
+#jinja2: trim_blocks:False
+apiVersion: componentconfig/v1alpha1
+kind: KubeSchedulerConfiguration
+clientConnection:
+  kubeconfig: "{{k8s_scheduler_conf_dir}}/kube-scheduler.kubeconfig"
+healthzBindAddress: {{hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address}}:10251
+leaderElection:
+  leaderElect: true
+metricsBindAddress: {{hostvars[inventory_hostname]['ansible_' + k8s_interface].ipv4.address}}:10251