Layering engine for kyma modules (#1322)

* Layering engine for kyma modules

Module root folders are now inspected and specific subfolders and files are added into dedicated layers

Signed-off-by: Borja Clemente <borja.clemente.castanera@sap.com>

* Adapt to new charts module format and inspection

Charts now live in their own subflder and inspection needs to be adapted.

Signed-off-by: Borja Clemente <borja.clemente.castanera@sap.com>

Co-authored-by: Johannes Veicht <johannes.veicht@sap.com>

cmd/kyma/alpha/create/module/module.go

@@ -71,9 +71,6 @@ func (c *command) Run(args []string) error {
 	}
 
 	/* -- CREATE ARCHIVE -- */
-
-	// prepend the module root to the paths
-	c.opts.ResourcePaths = append([]string{args[2]}, c.opts.ResourcePaths...)
 	fs := osfs.New()
 
 	c.NewStep(fmt.Sprintf("Creating module archive at %q", c.opts.ModPath))
@@ -88,7 +85,13 @@ func (c *command) Run(args []string) error {
 
 	c.NewStep("Adding resources...")
 
-	defs := []module.ResourceDef{}
+	// Create base resource defs with module root and its sub-layers
+	defs, err := module.Inspect(args[2], l)
+	if err != nil {
+		return err
+	}
+
+	// add extra resources
 	for _, p := range c.opts.ResourcePaths {
 		rd, err := module.ResourceDefFromString(p)
 		if err != nil {

pkg/module/def.go

@@ -12,9 +12,10 @@ const (
 
 // ResourceDef represents a resource definition that can be added to a module as a layer
 type ResourceDef struct {
-	name         string
-	resourceType string
-	path         string
+	name          string
+	resourceType  string
+	path          string
+	excludedFiles []string
 }
 
 // ResourceDefFromString creates a resource definition from a string with format NAME:TYPE@PATH

pkg/module/resources.go

@@ -3,6 +3,7 @@ package module
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 
 	cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2"
@@ -106,6 +107,7 @@ func generateResources(log *zap.SugaredLogger, version string, defs ...ResourceD
 			r.Input.Type = "dir"
 			compress := true
 			r.Input.CompressWithGzip = &compress
+			r.Input.ExcludeFiles = d.excludedFiles
 		} else {
 			r.Input.Type = "file"
 		}
@@ -147,3 +149,85 @@ func (rd ResourceDescriptor) String() string {
 	}
 	return string(y)
 }
+
+// builtInResources contains a set of resources that will automatically be created if found inspecting a module path
+var builtInResources = map[string]ResourceDef{
+	"crds": {
+		name:         "crds",
+		resourceType: "crds",
+	},
+	"operator": {
+		name:         "operator",
+		resourceType: "operator",
+	},
+	"profiles": {
+		name:         "profiles",
+		resourceType: "profiles",
+	},
+	"channels": {
+		name:         "channels",
+		resourceType: "channels",
+	},
+	"config.yaml": {
+		name:         "config",
+		resourceType: "yaml",
+	},
+}
+
+// Inspect analyzes the contents of a module and creates resource definitions for each separate layer the module should be split into
+func Inspect(path string, log *zap.SugaredLogger) ([]ResourceDef, error) {
+	log.Debugf("Inspecting module contents at [%s]:", path)
+	// check path subfolders
+	infos, err := os.ReadDir(path)
+	if err != nil {
+		return nil, fmt.Errorf("could not Inspect module folder %q: %w", path, err)
+	}
+
+	defs := []ResourceDef{}
+	// for each element in the mod path [charts, crds, operator, profiles, channels, config.yaml] create a def and validate it
+	for _, i := range infos {
+		// special case for charts folder
+		if i.Name() == "charts" {
+			charts, err := os.ReadDir(filepath.Join(path, "charts"))
+			if err != nil {
+				return nil, err
+			}
+
+			for _, i := range charts {
+				log.Debugf("Found chart %q", i.Name())
+				r := ResourceDef{
+					name:         i.Name(),
+					path:         filepath.Join(path, "charts", i.Name()),
+					resourceType: "helm-chart",
+				}
+				if defs, err = appendDefIfValid(defs, r, i, log); err != nil {
+					return nil, err
+				}
+
+			}
+		} else if r, ok := builtInResources[i.Name()]; ok {
+			log.Debugf("Found built in layer %q", i.Name())
+
+			r.path = filepath.Join(path, i.Name())
+			if defs, err = appendDefIfValid(defs, r, i, log); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return defs, nil
+}
+
+func appendDefIfValid(defs []ResourceDef, r ResourceDef, i os.DirEntry, log *zap.SugaredLogger) ([]ResourceDef, error) {
+	if i.IsDir() {
+		empty, err := files.IsDirEmpty(r.path)
+		if err != nil {
+			return nil, fmt.Errorf("could not determine if directory %q is empty: %w", r.path, err)
+		}
+		if empty {
+			log.Debugf("Resource %q has no content, skipping", i.Name())
+			return defs, nil
+		}
+	}
+	log.Debugf("Added layer %+v", r)
+	return append(defs, r), nil
+}

pkg/module/test/test-module/charts/dummy/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+description: Dummy chart
+name: dummy-chart
+version: 0.0.999
+home: https://kyma-project.io
+icon: https://github.com/kyma-project/kyma/blob/main/logo.png?raw=true

pkg/module/test/test-module/charts/test-chart/values.yaml

@@ -0,0 +1,11 @@
+image:
+  pullPolicy: IfNotPresent
+disableConcurrency: false
+restartPolicy: Never
+resources:
+  requests:
+    memory: 32Mi
+    cpu: 10m
+  limits:
+    memory: 64Mi
+    cpu: 200m

pkg/module/test/test-module/crds/crds.yaml

@@ -0,0 +1,40 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  # name must match the spec fields below, and be in the form: <plural>.<group>
+  name: something.stable.example.com
+spec:
+  # group name to use for REST API: /apis/<group>/<version>
+  group: stable.example.com
+  # list of versions supported by this CustomResourceDefinition
+  versions:
+    - name: v1
+      # Each version can be enabled/disabled by Served flag.
+      served: true
+      # One and only one version must be marked as the storage version.
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                cronSpec:
+                  type: string
+                image:
+                  type: string
+                replicas:
+                  type: integer
+  # either Namespaced or Cluster
+  scope: Namespaced
+  names:
+    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
+    plural: somethings
+    # singular name to be used as an alias on the CLI and for display
+    singular: something
+    # kind is normally the CamelCased singular type. Your resource manifests use this.
+    kind: Something
+    # shortNames allow shorter string to match your resource on the CLI
+    shortNames:
+      - st