Merge pull request #283 from kmathur2/model_connections

KubedirectorCluster Connections

deploy/kubedirector/kubedirector.hpe.com_kubedirectorclusters_crd.yaml

@@ -36,6 +36,17 @@ spec:
             appCatalog:
               type: string
               pattern: '^local$|^system$'
+            connections:
+              type: object
+              properties:
+                clusters:
+                  type: array
+                  items:
+                    type: string
+                configmaps:
+                  type: array
+                  items:
+                    type: string  
             serviceType:
               type: string
               pattern: '^ClusterIP$|^NodePort$|^LoadBalancer$'
@@ -56,6 +67,8 @@ spec:
                   maximum: 511
                 readOnly:
                   type: boolean
+            connectionsGenerationToProcess:
+              type: integer      
             roles:
               type: array
               items:
@@ -197,6 +210,8 @@ spec:
                   type: boolean
             generationUID:
               type: string
+            lastConnectionsGeneration:
+              type: integer  
             specGenerationToProcess:
               type: integer
             clusterService:

pkg/apis/kubedirector/v1beta1/kubedirectorapp_types.go

@@ -42,8 +42,7 @@ type KubeDirectorAppSpec struct {
 type KubeDirectorApp struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
-
-	Spec KubeDirectorAppSpec `json:"spec,omitempty"`
+	Spec              KubeDirectorAppSpec `json:"spec,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -82,9 +81,10 @@ type SetupPackageURL struct {
 // access, and/or identified for other use by API clients or consumers
 // internal to the virtual cluster (e.g. app setup packages).
 type Service struct {
-	ID       string          `json:"id"`
-	Label    Label           `json:"label,omitempty"`
-	Endpoint ServiceEndpoint `json:"endpoint,omitempty"`
+	ID              string          `json:"id"`
+	Label           Label           `json:"label,omitempty"`
+	Endpoint        ServiceEndpoint `json:"endpoint,omitempty"`
+	ExportedService string          `json:"exported_service,omitempty"`
 }
 
 // ServiceEndpoint describes the service network address and protocol, and

pkg/apis/kubedirector/v1beta1/kubedirectorcluster_types.go

@@ -25,11 +25,20 @@ import (
 // requested cluster roles, each of which will be implemented (by KubeDirector)
 // using a StatefulSet.
 type KubeDirectorClusterSpec struct {
-	AppID         string    `json:"app"`
-	AppCatalog    *string   `json:"appCatalog,omitempty"`
-	ServiceType   *string   `json:"serviceType,omitempty"`
-	Roles         []Role    `json:"roles"`
-	DefaultSecret *KDSecret `json:"defaultSecret,omitempty"`
+	AppID                   string      `json:"app"`
+	AppCatalog              *string     `json:"appCatalog,omitempty"`
+	ServiceType             *string     `json:"serviceType,omitempty"`
+	Roles                   []Role      `json:"roles"`
+	DefaultSecret           *KDSecret   `json:"defaultSecret,omitempty"`
+	ConnectionsGenToProcess int64       `json:"connectionsGenerationToProcess"`
+	Connections             Connections `json:"connections"`
+}
+
+// Connections specifies list of cluster objects and configmaps objects that has
+// be connected to the cluster.
+type Connections struct {
+	Clusters   []string `json:"clusters,omitempty"`
+	ConfigMaps []string `json:"configmaps,omitempty"`
 }
 
 // KubeDirectorClusterStatus defines the observed state of KubeDirectorCluster.
@@ -43,6 +52,7 @@ type KubeDirectorClusterStatus struct {
 	ClusterService          string       `json:"clusterService"`
 	LastNodeID              int64        `json:"lastNodeID"`
 	Roles                   []RoleStatus `json:"roles"`
+	LastConnectionGen       int64        `json:"lastConnectionsGeneration"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -57,10 +67,9 @@ type KubeDirectorClusterStatus struct {
 type KubeDirectorCluster struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
-
-	Spec    KubeDirectorClusterSpec    `json:"spec,omitempty"`
-	Status  *KubeDirectorClusterStatus `json:"status,omitempty"`
-	AppSpec *KubeDirectorApp           `json:"-"`
+	Spec              KubeDirectorClusterSpec    `json:"spec,omitempty"`
+	Status            *KubeDirectorClusterStatus `json:"status,omitempty"`
+	AppSpec           *KubeDirectorApp           `json:"-"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/catalog/configmeta.go

@@ -20,22 +20,35 @@ import (
 	"sync"
 
 	kdv1 "github.com/bluek8s/kubedirector/pkg/apis/kubedirector/v1beta1"
+	"github.com/bluek8s/kubedirector/pkg/observer"
 	"github.com/bluek8s/kubedirector/pkg/shared"
 	v1 "k8s.io/api/core/v1"
 )
 
+const (
+	// ConfigMapType is a label placed on desired comfig maps that
+	// we want to watch and propogate inside containers
+	configMapType = shared.KdDomainBase + "/cmType"
+)
+
 // allServiceRefkeys is a subroutine of getServices, used to generate a
 // description of a service's associated roles in the format expected by the
 // app setup Python packages.
 func allServiceRefkeys(
 	roleNames []string,
 	serviceName string,
+	connectedClusterName string,
 ) refkeysMap {
 
 	result := make(refkeysMap)
 	for _, r := range roleNames {
+		var refKeyList []string
+		if connectedClusterName != "" {
+			refKeyList = []string{"connections", "clusters", connectedClusterName}
+		}
+		refKeyList = append(refKeyList, "nodegroups", "1", "roles", r, "services", serviceName)
 		result[r] = refkeys{
-			BdvlibRefKey: []string{"nodegroups", "1", "roles", r, "services", serviceName},
+			BdvlibRefKey: refKeyList,
 		}
 	}
 	return result
@@ -47,6 +60,7 @@ func allServiceRefkeys(
 func getServices(
 	appCR *kdv1.KubeDirectorApp,
 	membersForRole map[string][]*kdv1.MemberStatus,
+	connectedClusterName string,
 ) map[string]ngRefkeysMap {
 
 	result := make(map[string]ngRefkeysMap)
@@ -62,7 +76,7 @@ func getServices(
 		}
 		if len(activeRoleNames) > 0 {
 			result[service.ID] = ngRefkeysMap{
-				"1": allServiceRefkeys(activeRoleNames, service.ID),
+				"1": allServiceRefkeys(activeRoleNames, service.ID, connectedClusterName),
 			}
 		}
 	}
@@ -76,6 +90,8 @@ func servicesForRole(
 	appCR *kdv1.KubeDirectorApp,
 	roleName string,
 	members []*kdv1.MemberStatus,
+	connectedClusterName string,
+	domain string,
 ) map[string]service {
 
 	result := make(map[string]service)
@@ -89,7 +105,7 @@ func servicesForRole(
 					for _, m := range members {
 						nodeName := m.Pod
 						endpoint := serviceDef.Endpoint.URLScheme
-						endpoint += "://" + nodeName
+						endpoint += "://" + nodeName + "." + domain
 						endpoint += ":" + strconv.Itoa(int(*(serviceDef.Endpoint.Port)))
 						endpoints = append(endpoints, endpoint)
 					}
@@ -105,9 +121,19 @@ func servicesForRole(
 					FQDNs: refkeys{
 						BdvlibRefKey: []string{"nodegroups", "1", "roles", roleName, "fqdns"},
 					},
-					ExportedService: "", // currently, always empty
+					ExportedService: serviceDef.ExportedService,
 					Endpoints:       endpoints,
 				}
+				if connectedClusterName != "" {
+					s.Hostnames.BdvlibRefKey = append(
+						[]string{"connections", "clusters", connectedClusterName},
+						s.Hostnames.BdvlibRefKey...,
+					)
+					s.FQDNs.BdvlibRefKey = append(
+						[]string{"connections", "clusters", connectedClusterName},
+						s.FQDNs.BdvlibRefKey...,
+					)
+				}
 				result[serviceDef.ID] = s
 			}
 		}
@@ -116,6 +142,85 @@ func servicesForRole(
 	return result
 }
 
+// genconfigConnections will look at the cluster spec
+// and generates a map of configmap type and corresponding
+// configmaps to be connected to the given cluster
+func genconfigConnections(
+	cr *kdv1.KubeDirectorCluster,
+) (map[string]map[string]map[string]string, error) {
+
+	cmMap := make(map[string]map[string]string)
+	kdcm := make(map[string]map[string]map[string]string)
+	for _, connectedCmName := range cr.Spec.Connections.ConfigMaps {
+		cm, err := observer.GetConfigMap(cr.Namespace, connectedCmName)
+		if kdConfigMapType, ok := cm.Labels[configMapType]; ok {
+			cmMap[connectedCmName] = cm.Data
+			kdcm[kdConfigMapType] = cmMap
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	return kdcm, nil
+}
+
+// genClusterConnections generates a map of running clusters that are to be connected
+// to this cluster.
+func genClusterConnections(
+	cr *kdv1.KubeDirectorCluster,
+) (map[string]configmeta, error) {
+
+	toConnectMeta := make(map[string]configmeta)
+	for _, clusterName := range cr.Spec.Connections.Clusters {
+		// Fetch the cluster object
+		clusterToConnect, connectedErr := observer.GetCluster(cr.Namespace, clusterName)
+		if connectedErr != nil {
+			return nil, connectedErr
+		}
+		appForclusterToConnect, connectedAppErr := observer.GetApp(clusterToConnect.Namespace, clusterToConnect.Spec.AppID)
+		if connectedAppErr != nil {
+			return nil, connectedAppErr
+		}
+		domain := clusterToConnect.Status.ClusterService + "." + clusterToConnect.Namespace + shared.GetSvcClusterDomainBase()
+		membersForRole := make(map[string][]*kdv1.MemberStatus)
+		for _, roleInfo := range clusterToConnect.Status.Roles {
+			var membersStatus []*kdv1.MemberStatus
+			for _, members := range roleInfo.Members {
+				membersStatus = append(
+					membersStatus,
+					&members,
+				)
+			}
+			membersForRole[roleInfo.Name] = membersStatus
+		}
+
+		toConnectMeta[clusterName] = configmeta{
+			Version:    strconv.Itoa(appForclusterToConnect.Spec.SchemaVersion),
+			Services:   getServices(appForclusterToConnect, membersForRole, clusterName),
+			Nodegroups: nodegroups(clusterToConnect, appForclusterToConnect, membersForRole, domain),
+			Distros: map[string]refkeysMap{
+				appForclusterToConnect.Spec.DistroID: refkeysMap{
+					"1": refkeys{
+						BdvlibRefKey: []string{"connections", "clusters", clusterName, "nodegroups", "1"},
+					},
+				},
+			},
+			Cluster: cluster{
+				Name:     clusterName,
+				Isolated: false, // currently, always false
+				ID:       string(clusterToConnect.UID),
+				ConfigMeta: map[string]refkeys{
+					"1": refkeys{
+						BdvlibRefKey: []string{"nodegroups", "1", "config_metadata"},
+					},
+				},
+			},
+		}
+	}
+
+	return toConnectMeta, nil
+}
+
 // nodegroups generates a map of nodegroup ID to internal nodegroup
 // representation. Note that KubeDirector currently only allows/manages one
 // nodegroup per virtual cluster, so this will always be a map that has a
@@ -157,7 +262,7 @@ func nodegroups(
 			Cores:       strconv.FormatInt(coresQuant.Value(), 10), // rounds up
 		}
 		roles[roleName] = role{
-			Services:     servicesForRole(appCR, roleName, members),
+			Services:     servicesForRole(appCR, roleName, members, "", domain),
 			NodeIDs:      nodeIds,
 			Hostnames:    fqdns,
 			FQDNs:        fqdns,
@@ -182,11 +287,21 @@ func clusterBaseConfig(
 	appCR *kdv1.KubeDirectorApp,
 	membersForRole map[string][]*kdv1.MemberStatus,
 	domain string,
-) *configmeta {
+) (*configmeta, error) {
+
+	clustersMeta, connErr := genClusterConnections(cr)
+	kdConfigMaps, cmErr := genconfigConnections(cr)
+
+	if cmErr != nil {
+		return nil, cmErr
+	}
+	if connErr != nil {
+		return nil, connErr
+	}
 
 	return &configmeta{
 		Version:    strconv.Itoa(appCR.Spec.SchemaVersion),
-		Services:   getServices(appCR, membersForRole),
+		Services:   getServices(appCR, membersForRole, ""),
 		Nodegroups: nodegroups(cr, appCR, membersForRole, domain),
 		Distros: map[string]refkeysMap{
 			appCR.Spec.DistroID: refkeysMap{
@@ -205,7 +320,11 @@ func clusterBaseConfig(
 				},
 			},
 		},
-	}
+		Connections: connections{
+			Clusters:   clustersMeta,
+			ConfigMaps: kdConfigMaps,
+		},
+	}, nil
 }
 
 // ConfigmetaGenerator returns a function that generates metadata which will be
@@ -230,7 +349,7 @@ func ConfigmetaGenerator(
 	// would be generated.
 	domain := cr.Status.ClusterService + "." + cr.Namespace + shared.GetSvcClusterDomainBase()
 	perNodeConfig := make(map[string]*node)
-	c := clusterBaseConfig(cr, appCR, membersForRole, domain)
+	c, _ := clusterBaseConfig(cr, appCR, membersForRole, domain)
 	for roleName, members := range membersForRole {
 		for _, member := range members {
 			memberName := member.Pod

pkg/catalog/types.go

@@ -19,12 +19,13 @@ package catalog
 // CR. It is arranged in a format to be consumed by the app setup Python
 // packages.
 type configmeta struct {
-	Version    string                  `json:"version"`
-	Services   map[string]ngRefkeysMap `json:"services"`
-	Nodegroups map[string]nodegroup    `json:"nodegroups"`
-	Distros    map[string]refkeysMap   `json:"distros"`
-	Cluster    cluster                 `json:"cluster"`
-	Node       *node                   `json:"node"`
+	Version     string                  `json:"version"`
+	Services    map[string]ngRefkeysMap `json:"services"`
+	Nodegroups  map[string]nodegroup    `json:"nodegroups"`
+	Distros     map[string]refkeysMap   `json:"distros"`
+	Cluster     cluster                 `json:"cluster"`
+	Node        *node                   `json:"node"`
+	Connections connections             `json:"connections"`
 }
 
 type ngRefkeysMap map[string]refkeysMap
@@ -42,6 +43,11 @@ type nodegroup struct {
 	ConfigMeta          map[string]string `json:"config_metadata"`
 }
 
+type connections struct {
+	Clusters   map[string]configmeta                   `json:"clusters"`
+	ConfigMaps map[string]map[string]map[string]string `json:"configmaps"`
+}
+
 type cluster struct {
 	Name       string             `json:"name"`
 	Isolated   bool               `json:"isolated"`

pkg/controller/add_configmap.go

@@ -0,0 +1,11 @@
+package controller
+
+import (
+	"github.com/bluek8s/kubedirector/pkg/controller/configmap"
+)
+
+func init() {
+
+	// AddToManagerFuncs is a list of functions to create controllers and add them to a manager.
+	AddToManagerFuncs = append(AddToManagerFuncs, configmap.Add)
+}