Use secrets instead of file mounts

Signed-off-by: Chuck Ha <chuckh@vmware.com>

Dockerfile.capk

@@ -6,7 +6,8 @@ RUN go mod download
 RUN  curl -L https://dl.k8s.io/v1.14.3/kubernetes-client-linux-amd64.tar.gz | tar xvz
 ADD cmd cmd
 ADD capkactuators capkactuators
-ADD pkg/kind pkg/kind
+ADD kind kind
+ADD execer execer
 
 RUN go install -v ./cmd/capk-manager
 RUN GO111MODULE="on" go get sigs.k8s.io/kind@v0.3.0

README.md

@@ -2,6 +2,10 @@
 
 A temporary home for CAPK
 
+# Development
+
+Please make an issue to discuss before large changes occur. 
+
 # Manager Container Image
 
 A sample is built and hosted at `gcr.io/kubernetes1-226021/capk-manager:latest` 

capkactuators/actuators.go

@@ -7,11 +7,15 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"time"
 
-	"k8s.io/apimachinery/pkg/types"
-
+	"github.com/pkg/errors"
 	"gitlab.com/chuckh/cluster-api-provider-kind/kind/actions"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
 	"sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1"
 	capierror "sigs.k8s.io/cluster-api/pkg/controller/error"
@@ -22,14 +26,14 @@ import (
 )
 
 type Machine struct {
-	ClusterAPI     v1alpha1.ClusterV1alpha1Interface
-	KubeconfigsDir string
+	Core       corev1.CoreV1Interface
+	ClusterAPI v1alpha1.ClusterV1alpha1Interface
 }
 
-func NewMachineActuator(kubeconfigs string, clusterapi v1alpha1.ClusterV1alpha1Interface) *Machine {
+func NewMachineActuator(clusterapi v1alpha1.ClusterV1alpha1Interface, core corev1.CoreV1Interface) *Machine {
 	return &Machine{
-		ClusterAPI:     clusterapi,
-		KubeconfigsDir: kubeconfigs,
+		Core:       core,
+		ClusterAPI: clusterapi,
 	}
 }
 
@@ -75,7 +79,21 @@ func (m *Machine) Create(ctx context.Context, c *clusterv1.Cluster, machine *clu
 			return err
 		}
 		setKindName(machine, controlPlaneNode.Name())
-		return m.save(old, machine)
+		if err := m.save(old, machine); err != nil {
+			fmt.Printf("%+v", err)
+			return err
+		}
+		s, err := kubeconfigToSecret(c.Name, c.Namespace)
+		if err != nil {
+			fmt.Printf("%+v", err)
+			return err
+		}
+		// Save the secret to the management cluster
+		if _, err := m.Core.Secrets(machine.GetNamespace()).Create(s); err != nil {
+			fmt.Printf("%+v", err)
+			return err
+		}
+		return nil
 	}
 
 	// If there are no control plane then we should hold off on joining workers
@@ -103,14 +121,22 @@ func (m *Machine) Update(ctx context.Context, cluster *clusterv1.Cluster, machin
 }
 
 func (m *Machine) Exists(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine) (bool, error) {
+	if getKindName(machine) == "" {
+		return false, nil
+	}
 	fmt.Println("Looking for a docker container named", getKindName(machine))
 	role := getRole(machine)
-	nodeList, err := nodes.List(fmt.Sprintf("label=%s=%s", constants.NodeRoleKey, role),
+	labels := []string{
+		fmt.Sprintf("label=%s=%s", constants.NodeRoleKey, role),
 		fmt.Sprintf("label=%s=%s", constants.ClusterLabelKey, cluster.Name),
-		fmt.Sprintf("name=%s", getKindName(machine)))
+		fmt.Sprintf("name=^%s$", getKindName(machine)),
+	}
+	fmt.Printf("using labels: %v\n", labels)
+	nodeList, err := nodes.List(labels...)
 	if err != nil {
-		return true, err
+		return false, err
 	}
+	fmt.Printf("found nodes: %v\n", nodeList)
 	return len(nodeList) >= 1, nil
 }
 
@@ -138,7 +164,9 @@ func (m *Machine) save(old, new *clusterv1.Machine) error {
 }
 
 func setKindName(machine *clusterv1.Machine, name string) {
-	machine.SetAnnotations(map[string]string{"name": name})
+	a := machine.GetAnnotations()
+	a["name"] = name
+	machine.SetAnnotations(a)
 }
 
 func getKindName(machine *clusterv1.Machine) string {
@@ -185,3 +213,22 @@ func (c *Cluster) Delete(cluster *clusterv1.Cluster) error {
 	fmt.Println("Cluster delete is not implemented.")
 	return nil
 }
+
+func kubeconfigToSecret(clusterName, namespace string) (*v1.Secret, error) {
+	// open kubeconfig file
+	data, err := ioutil.ReadFile(actions.KubeConfigPath(clusterName))
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	// write it to a secret
+	return &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("kubeconfig-%s", clusterName),
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			"kubeconfig": data,
+		},
+	}, nil
+}

cmd/capk-manager/main.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"gitlab.com/chuckh/cluster-api-provider-kind/capkactuators"
-	"k8s.io/klog"
+	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/cluster-api/pkg/apis"
 	"sigs.k8s.io/cluster-api/pkg/apis/cluster/common"
 	"sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset"
@@ -35,13 +35,17 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
+	k8sclientset, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		panic(err)
+	}
 	cs, err := clientset.NewForConfig(cfg)
 	if err != nil {
-		klog.Fatalf("Failed to create client from configuration: %v", err)
+		panic(err)
 	}
 
 	clusterActuator := capkactuators.NewClusterActuator()
-	machineActuator := capkactuators.NewMachineActuator("/kubeconfigs", cs.ClusterV1alpha1())
+	machineActuator := capkactuators.NewMachineActuator(cs.ClusterV1alpha1(), k8sclientset.CoreV1())
 
 	// Register our cluster deployer (the interface is in clusterctl and we define the Deployer interface on the actuator)
 	common.RegisterClusterProvisioner("aws", clusterActuator)

cmd/capkctl/main.go

@@ -19,15 +19,57 @@ func main() {
 		printCRDs()
 	case "capk":
 		printClusterAPIPlane()
+	case "control-plane":
+		fmt.Fprintf(os.Stdout, machineYAML(os.Args[2], os.Args[3], os.Args[4], "control-plane"))
+	case "worker":
+		fmt.Fprintf(os.Stdout, machineYAML(os.Args[2], os.Args[3], os.Args[4], "worker"))
+	case "cluster":
+		fmt.Fprintf(os.Stdout, clusterYAML(os.Args[2], os.Args[3]))
 	default:
 		fmt.Fprint(os.Stderr, "unknown command", os.Args[1])
 		os.Exit(2)
 	}
 }
 
+func clusterYAML(name, namespace string) string {
+	return fmt.Sprintf(`apiVersion: "cluster.k8s.io/v1alpha1"
+kind: Cluster
+metadata:
+  name: %s
+  namespace: %s
+spec:
+  clusterNetwork:
+    services:
+      cidrBlocks: ["10.96.0.0/12"]
+    pods:
+      cidrBlocks: ["192.168.0.0/16"]
+    serviceDomain: "cluster.local"
+  providerSpec: {}`, name, namespace)
+}
+
+func machineYAML(name, namespace, cluster, set string) string {
+	return fmt.Sprintf(`apiVersion: "cluster.k8s.io/v1alpha1"
+kind: MachineList
+items:
+  - apiVersion: "cluster.k8s.io/v1alpha1"
+    kind: Machine
+    metadata:
+      name: %s
+      namespace: %s
+      labels:
+        cluster.k8s.io/cluster-name: %s
+      annotations:
+        set: %s
+    spec:
+      versions:
+        kubelet: v1.13.6
+        controlPlane: v1.13.6
+      providerSpec: {}`, name, namespace, cluster, set)
+}
+
 func makeManagementCluster() {
-	// start kind with docker mount
 	kind := execer.NewClient("kind")
+	// start kind with docker mount
 	kindConfig, err := kindConfigFile()
 	if err != nil {
 		panic(err)
@@ -45,8 +87,7 @@ nodes:
   extraMounts:
   - containerPath: /var/run/docker.sock
     hostPath: /var/run/docker.sock
-  - containerPath: /kubeconfigs
-    hostPath: /kubeconfigs`
+`
 
 	f, err := ioutil.TempFile("", "*-kind-config.yaml")
 	if err != nil {
@@ -109,8 +150,6 @@ spec:
           name: dockersock
         - mountPath: /var/lib/docker
           name: dockerlib
-        - mountPath: /kubeconfigs
-          name: kubeconfigs
         securityContext:
           privileged: true
       volumes:
@@ -121,9 +160,6 @@ spec:
       - name: dockerlib
         hostPath:
           path: /var/lib/docker
-      - name: kubeconfigs
-        hostPath:
-          path: /kubeconfigs
       tolerations:
       - effect: NoSchedule
         key: node-role.kubernetes.io/master
@@ -1116,6 +1152,7 @@ rules:
   resources:
   - nodes
   - events
+  - secrets
   verbs:
   - get
   - list

go.mod

@@ -20,8 +20,10 @@ require (
 	go.uber.org/zap v1.10.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
+	k8s.io/api v0.0.0-20181213150558-05914d821849
 	k8s.io/apiextensions-apiserver v0.0.0-20181213153335-0fe22c71c476 // indirect
 	k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93
+	k8s.io/client-go v10.0.0+incompatible
 	k8s.io/klog v0.3.2
 	sigs.k8s.io/cluster-api v0.0.0-20190607141803-aacb0c613ffb
 	sigs.k8s.io/controller-runtime v0.1.10

kind/actions/kind.go

@@ -13,15 +13,14 @@ import (
 	"regexp"
 	"strings"
 
-	"sigs.k8s.io/kind/pkg/container/cri"
-
 	"github.com/pkg/errors"
 	"gitlab.com/chuckh/cluster-api-provider-kind/kind/kubeadm"
 	"gitlab.com/chuckh/cluster-api-provider-kind/kind/loadbalancer"
 	"sigs.k8s.io/kind/pkg/cluster/config"
 	"sigs.k8s.io/kind/pkg/cluster/config/defaults"
 	"sigs.k8s.io/kind/pkg/cluster/constants"
 	"sigs.k8s.io/kind/pkg/cluster/nodes"
+	"sigs.k8s.io/kind/pkg/container/cri"
 	"sigs.k8s.io/kind/pkg/exec"
 	"sigs.k8s.io/kind/pkg/kustomize"
 )
@@ -31,7 +30,7 @@ import (
 // KubeConfigPath returns the path to the kubeconfig file for the given cluster name.
 func KubeConfigPath(clusterName string) string {
 	// configDir matches the standard directory expected by kubectl etc
-	configDir := "/kubeconfigs"
+	configDir := filepath.Join(os.Getenv("HOME"), ".kube")
 	// note that the file name however does not, we do not want to overwrite
 	// the standard config, though in the future we may (?) merge them
 	fileName := fmt.Sprintf("kind-config-%s", clusterName)
@@ -90,12 +89,7 @@ func CreateControlPlane(clusterName string) (*nodes.Node, error) {
 		clusterLabel,
 		"127.0.0.1",
 		0,
-		[]cri.Mount{
-			{
-				ContainerPath: "/root/.kube",
-				HostPath:      "/kubeconfigs",
-			},
-		},
+		[]cri.Mount{},
 	)
 	if err != nil {
 		return nil, err