internal/scaffold/olm-catalog: populate CSV customresourcedefinitions from Go type annotations

internal/pkg/scaffold/olm-catalog/csv_updaters.go

@@ -22,9 +22,11 @@ import (
 	"github.com/ghodss/yaml"
 	olmapiv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
 	olminstall "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
+	log "github.com/sirupsen/logrus"
 	appsv1 "k8s.io/api/apps/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
 // CSVUpdater is an interface for any data that can be in a CSV, which will be
@@ -200,12 +202,18 @@ func (store *updaterStore) AddOwnedCRD(yamlDoc []byte) error {
 	if err := yaml.Unmarshal(yamlDoc, crd); err != nil {
 		return err
 	}
-	store.crds.Owned = append(store.crds.Owned, olmapiv1alpha1.CRDDescription{
+	crdDesc := olmapiv1alpha1.CRDDescription{
 		Name:    crd.ObjectMeta.Name,
 		Version: crd.Spec.Version,
 		Kind:    crd.Spec.Names.Kind,
-	})
-	store.crds.crKinds[crd.Spec.Names.Kind] = struct{}{}
+	}
+	store.crds.crKinds[crdDesc.Kind] = struct{}{}
+	// Parse {spec,status}Descriptor's from source code comments.
+	gv := schema.GroupVersion{Group: crd.Spec.Group, Version: crd.Spec.Version}
+	if err := setCRDDescriptorsForGV(&crdDesc, gv); err != nil {
+		return err
+	}
+	store.crds.Owned = append(store.crds.Owned, crdDesc)
 	return nil
 }
 
@@ -222,6 +230,45 @@ func (u *CustomResourceDefinitionsUpdate) Apply(csv *olmapiv1alpha1.ClusterServi
 			csvDesc.Name = uDesc.Name
 			csvDesc.Version = uDesc.Version
 			csvDesc.Kind = uDesc.Kind
+			// Description comes from code comments so should always be updated.
+			csvDesc.Description = uDesc.Description
+			if csvDesc.DisplayName == "" {
+				csvDesc.DisplayName = uDesc.DisplayName
+			}
+			// Suggest status and spec descriptor paths and x-descriptors for any
+			// descriptor not in the CSV or that does not have a value.
+			specMap := make(map[string]olmapiv1alpha1.SpecDescriptor)
+			for _, cd := range csvDesc.SpecDescriptors {
+				specMap[cd.DisplayName] = cd
+			}
+			for _, ud := range uDesc.SpecDescriptors {
+				cd, ok := specMap[ud.DisplayName]
+				if ud.Path != "" && (!ok || cd.Path == "") {
+					log.Infof("Suggestion: owned CRD specDescriptor '%s' should have path '%s'.", ud.DisplayName, ud.Path)
+				}
+				if len(ud.XDescriptors) != 0 && (!ok || len(cd.XDescriptors) == 0) {
+					log.Infof("Suggestion: owned CRD specDescriptor '%s' should have x-descriptors '%s'.", ud.DisplayName, ud.XDescriptors)
+				}
+			}
+			if len(csvDesc.SpecDescriptors) == 0 {
+				csvDesc.SpecDescriptors = uDesc.SpecDescriptors
+			}
+			statusMap := make(map[string]olmapiv1alpha1.StatusDescriptor)
+			for _, cd := range csvDesc.StatusDescriptors {
+				statusMap[cd.DisplayName] = cd
+			}
+			for _, ud := range uDesc.StatusDescriptors {
+				cd, ok := statusMap[ud.DisplayName]
+				if ud.Path != "" && (!ok || cd.Path == "") {
+					log.Infof("Suggestion: owned CRD statusDescriptor '%s' should have path '%s'.", ud.DisplayName, ud.Path)
+				}
+				if len(ud.XDescriptors) != 0 && (!ok || len(cd.XDescriptors) == 0) {
+					log.Infof("Suggestion: owned CRD statusDescriptor '%s' should have x-descriptors '%s'.", ud.DisplayName, ud.XDescriptors)
+				}
+			}
+			if len(csvDesc.StatusDescriptors) == 0 {
+				csvDesc.StatusDescriptors = uDesc.StatusDescriptors
+			}
 			du.Owned[i] = csvDesc
 		}
 	}

internal/pkg/scaffold/olm-catalog/descriptor.go

@@ -0,0 +1,216 @@
+// Copyright 2019 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package catalog
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/printer"
+	"go/token"
+	"go/types"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/operator-framework/operator-sdk/internal/pkg/scaffold"
+
+	olmapiv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
+	log "github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type fieldVals struct {
+	name, typ, tag, comments string
+}
+
+// setCRDDescriptorsForGV parses document and type declaration comments on
+// CRD types to populate a csv's 'crds.owned[].{spec,status}Descriptors' for
+// a given group and version.
+func setCRDDescriptorsForGV(crdDesc *olmapiv1alpha1.CRDDescription, gv schema.GroupVersion) error {
+	fset := token.NewFileSet()
+	ff := func(info os.FileInfo) bool {
+		return strings.HasSuffix(info.Name(), "_types.go")
+	}
+	dir := filepath.Join(scaffold.ApisDir, strings.Split(gv.Group, ".")[0], gv.Version)
+	pkgs, err := parser.ParseDir(fset, dir, ff, parser.ParseComments)
+	if err != nil {
+		// Don't return in error as other CSV components can still be generated.
+		if os.IsNotExist(err) {
+			return nil
+		}
+		log.Fatal(err)
+	}
+
+	allTypes := make(map[string][]fieldVals)
+	for _, pkg := range pkgs {
+		ast.Inspect(pkg, func(n ast.Node) bool {
+			switch x := n.(type) {
+			case *ast.GenDecl:
+				for _, spec := range x.Specs {
+					if spec, ok := spec.(*ast.TypeSpec); ok {
+						specName := spec.Name.Name
+						if specName == crdDesc.Kind {
+							crdDesc.Description = processComments(x.Doc.Text())
+						}
+						vals, ok := allTypes[specName]
+						if !ok {
+							vals = make([]fieldVals, 0)
+						}
+						switch st := spec.Type.(type) {
+						case *ast.StructType:
+							for _, field := range st.Fields.List {
+								typ := processType(fset, field.Type)
+								comments := processComments(field.Doc.Text())
+								names := field.Names
+								if len(names) == 0 {
+									names = []*ast.Ident{{Name: typ}}
+								}
+								for _, name := range names {
+									// Is exported, primitive, or imported.
+									if name.IsExported() || strings.Contains(name.Name, ".") {
+										vals = append(vals, fieldVals{
+											name:     getDisplayName(name.Name),
+											typ:      typ,
+											tag:      field.Tag.Value,
+											comments: comments,
+										})
+									}
+								}
+							}
+						}
+						allTypes[specName] = vals
+					}
+				}
+			}
+			return true
+		})
+	}
+	kindTypes, ok := allTypes[crdDesc.Kind]
+	if !ok {
+		return fmt.Errorf("no type found for kind %s", crdDesc.Kind)
+	}
+	for _, kt := range kindTypes {
+		for _, fields := range allTypes[kt.typ] {
+			if strings.HasSuffix(kt.name, "Spec") {
+				path, xd := guessPathAndXDescriptorFromTag(fields.tag, true)
+				crdDesc.SpecDescriptors = append(crdDesc.SpecDescriptors, olmapiv1alpha1.SpecDescriptor{
+					Description:  fields.comments,
+					DisplayName:  fields.name,
+					Path:         path,
+					XDescriptors: xd,
+				})
+			} else if strings.HasSuffix(kt.name, "Status") {
+				path, xd := guessPathAndXDescriptorFromTag(fields.tag, false)
+				crdDesc.StatusDescriptors = append(crdDesc.StatusDescriptors, olmapiv1alpha1.StatusDescriptor{
+					Description:  fields.comments,
+					DisplayName:  fields.name,
+					Path:         path,
+					XDescriptors: xd,
+				})
+			}
+		}
+	}
+	return nil
+}
+
+// processComments joins comment strings into one line, removing any tool
+// directives.
+func processComments(comments string) string {
+	lines := make([]string, 0)
+	scanner := bufio.NewScanner(strings.NewReader(comments))
+	for scanner.Scan() {
+		l := strings.TrimSpace(scanner.Text())
+		if l == "" || strings.Contains(l, "+k8s:") || strings.Contains(l, "+kubebuilder:") {
+			continue
+		}
+		lines = append(lines, l)
+	}
+	if err := scanner.Err(); err != nil {
+		log.Error(err)
+	}
+	return strings.Join(lines, " ")
+}
+
+var mapRe = regexp.MustCompile(`map\[.+\]`)
+
+// processType cleans and returns the string of a type expression within fset.
+func processType(fset *token.FileSet, e ast.Expr) (t string) {
+	tbuf := &bytes.Buffer{}
+	if err := printer.Fprint(tbuf, fset, e); err != nil {
+		log.Fatal(err)
+	}
+	t = tbuf.String()
+	// Clean slice, map, and pointer syntax.
+	tt := strings.Replace(t, "[]", "", -1)
+	tt = strings.Replace(tt, "*", "", -1)
+	tt = mapRe.ReplaceAllString(tt, "")
+	// Only return non-primitive types without extra syntax.
+	if types.Universe.Lookup(tt) == nil {
+		t = tt
+	}
+	return t
+}
+
+// From https://github.com/openshift/console/blob/master/frontend/public/components/operator-lifecycle-manager/descriptors/types.ts#L5-L14
+var specXDescriptors = map[string][]string{
+	"size":                 {"size", "urn:alm:descriptor:com.tectonic.ui:podCount"},
+	"endpoints":            {"endpointList", "urn:alm:descriptor:com.tectonic.ui:endpointList"},
+	"label":                {"label", "urn:alm:descriptor:com.tectonic.ui:label"},
+	"resourceRequirements": {"resourceRequirements", "urn:alm:descriptor:com.tectonic.ui:resourceRequirements"},
+	"selector":             {"selector", "urn:alm:descriptor:com.tectonic.ui:selector:"},
+	"namespaceSelector":    {"namespaceSelector", "urn:alm:descriptor:com.tectonic.ui:namespaceSelector"},
+	"booleanSwitch":        {"booleanSwitch", "urn:alm:descriptor:com.tectonic.ui:booleanSwitch"},
+}
+
+// From https://github.com/openshift/console/blob/master/frontend/public/components/operator-lifecycle-manager/descriptors/types.ts#L16-L27
+var statusXDescriptors = map[string][]string{
+	"size":               {"size", "urn:alm:descriptor:com.tectonic.ui:podCount"},
+	"podStatuses":        {"podStatuses", "urn:alm:descriptor:com.tectonic.ui:podStatuses"},
+	"links":              {"w3Link", "urn:alm:descriptor:org.w3:link"},
+	"conditions":         {"conditions", "urn:alm:descriptor:io.kubernetes.conditions"},
+	"text":               {"text", "urn:alm:descriptor:text"},
+	"prometheusEndpoint": {"prometheusEndpoint", "urn:alm:descriptor:prometheusEndpoint"},
+	"status":             {"phase", "urn:alm:descriptor:io.kubernetes.phase"},
+	"reason":             {"reason", "urn:alm:descriptor:io.kubernetes.phase:reason"},
+}
+
+var jsonTagRe = regexp.MustCompile("`json:\"([^,]+),?.*\"`")
+
+// guessPathAndXDescriptorFromTag uses json field tags to guess which path
+// and x-descriptor a CRD should have. This choice is a guess based on tag
+// identifier.
+func guessPathAndXDescriptorFromTag(tag string, isSpec bool) (path string, xd []string) {
+	tagMatches := jsonTagRe.FindStringSubmatch(tag)
+	if len(tagMatches) == 2 {
+		path = tagMatches[1]
+	}
+	var (
+		pathAndXD []string
+		ok        bool
+	)
+	if isSpec {
+		pathAndXD, ok = specXDescriptors[path]
+	} else {
+		pathAndXD, ok = statusXDescriptors[path]
+	}
+	if ok {
+		return pathAndXD[0], []string{pathAndXD[1]}
+	}
+	return path, xd
+}

internal/pkg/scaffold/olm-catalog/testdata/deploy/olm-catalog/app-operator/0.1.0/app-operator.v0.1.0.clusterserviceversion.yaml

@@ -14,7 +14,7 @@ spec:
       kind: App
       name: apps1.example.com
       version: v1alpha1
-    - description: some description
+    - displayName: App
       kind: App
       name: apps2.example.com
       version: v1alpha2