Feature: Add only filters for local OS and cluster architecture component filtering (zarf-dev#591)

* add Only filters for components
* start describing the schema better
* Add only filter to docs
* add compose debug messages and handle bad component name
* Add debug message to component compatibility test

Author: jeff-mccoy

.tool-versions

@@ -1,3 +1,2 @@
 golang 1.18
-terraform 1.1.4
 adr-tools 3.0.0

docs/4-user-guide/2-zarf-packages/1-zarf-packages.md

@@ -4,9 +4,9 @@ sidebar_position: 1
 
 # Understanding Zarf Packages
 
-A Zarf package is a single binary Tarball that contains everything you need to deploy a system or capability while fully disconnected. Zarf packages are defined by a `zarf.yaml` file.
+A Zarf package is a single tarball archive that contains everything you need to deploy a system or capability while fully disconnected. Zarf packages are defined by a `zarf.yaml` file.
 
-Zarf packages are built while 'online' and connected to whatever is hosting the dependencies your package definition defined. When being built, all these defined dependencies are pulled from the internet and stored within the tarball package. Because all the dependencies are now within the tarball, the package can be deployed on to disconnected systems that don't have connection to the outside world.
+Zarf packages are built while 'online' and connected to whatever is hosting the dependencies your package definition defined. When being built, all these defined dependencies are downloaded and stored within the archive. Because all the dependencies are now within the tarball, the package can be deployed on to disconnected systems that don't have connection to the outside world.
 
 The `zarf.yaml` file, that the package builds from, defines declarative instructions on how capabilities of the package should be deployed. The declarative nature of the package means everything is represented by code and automatically runs as it is configured, instead of having to give manual steps that might not be reproducible on all systems.
 

docs/4-user-guide/2-zarf-packages/2-zarf-components.md

@@ -69,6 +69,12 @@ components:
 
     required: <BOOLEAN> # If required, this component will always be deployed with the package
 
+    only: <OJB> # Filter components on package create / deploy to control when it is available 
+      localOS: <STRING> # Only present on package deploy if the OS matches 
+      cluster: <OBJ>
+        architecture: <STRING> # Filter on both package deploy & create for matching cluster architecture
+        distros: <STRING LIST> # Future use
+
     secretName: <STRING> # The secret Zarf will use for the registry; default is 'zarf-registry'>
                          # The secret lives in the 'zarf' namespace.
 

go.mod

@@ -14,6 +14,7 @@ replace (
 
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.5
+	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b
 	github.com/derailed/k9s v0.25.21
 	github.com/distribution/distribution/v3 v3.0.0-20220612151901-b5e2f3f33dbc
@@ -44,8 +45,6 @@ require (
 	sigs.k8s.io/yaml v1.3.0
 )
 
-require github.com/Masterminds/semver/v3 v3.1.1
-
 require (
 	atomicgo.dev/cursor v0.1.1 // indirect
 	atomicgo.dev/keyboard v0.2.8 // indirect

go.sum

@@ -3359,8 +3359,6 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/klog/v2 v2.70.0 h1:GMmmjoFOrNepPN0ZeGCzvD2Gh5IKRwdFx8W5PBxVTQU=
-k8s.io/klog/v2 v2.70.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ=
 k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=

packages/big-bang-core/zarf.yaml

@@ -9,7 +9,7 @@ components:
   - name: flux
     required: true
     import:
-      path: ../../packages/flux-iron-bank
+      path: ../flux-iron-bank
 
   - name: big-bang-core-assets
     description: "Git repositories and OCI images used by Big Bang Core"

packages/distros/k3s/assets/empty-file packages/distros/k3s/common/empty-file

@@ -0,0 +1,39 @@
+kind: ZarfInitConfig
+metadata:
+  name: "distro-k3s"
+
+components:
+  - name: k3s
+    only:
+      localOS: linux
+    description: >
+      *** REQUIRES ROOT ***
+      Install K3s, certified Kubernetes distribution built for IoT & Edge computing.
+      K3s provides the cluster need for Zarf running in Appliance Mode as well as can
+      host a low-resource Gitops Service if not using an existing Kubernetes platform.
+    scripts:
+      retry: true
+      before:
+        # If running RHEL variant, disable firewalld
+        # https://rancher.com/docs/k3s/latest/en/advanced/#additional-preparation-for-red-hat-centos-enterprise-linux
+        # NOTE: The empty echo prevents infinite retry loops on non-RHEL systems where the exit code would be an error
+        - "[ -e /etc/redhat-release ] && systemctl disable firewalld --now || echo ''"
+      after:
+        # Configure K3s systemd service
+        - "systemctl daemon-reload"
+        - "systemctl enable --now k3s"
+    files:
+      # K3s removal script
+      - source: zarf-clean-k3s.sh
+        target: /opt/zarf/zarf-clean-k3s.sh
+        executable: true
+      # The K3s systemd service definition
+      - source: k3s.service
+        target: /etc/systemd/system/k3s.service
+        symlinks:
+          - /etc/systemd/system/multi-user.target.wants/k3s.service
+      # Mock file for creating the kube config symlink
+      - source: empty-file
+        target: /etc/rancher/k3s/k3s.yaml
+        symlinks:
+          - /root/.kube/config

packages/distros/k3s/assets/k3s.service packages/distros/k3s/common/k3s.service

@@ -2,26 +2,16 @@ kind: ZarfInitConfig
 metadata:
   name: "distro-k3s"
   description: "Used to establish a new Zarf cluster"
-  architecture: amd64
 
 components:
+  # AMD-64 version of the K3s stack
   - name: k3s
-    description: >
-      *** REQUIRES ROOT ***
-      Install K3s, certified Kubernetes distribution built for IoT & Edge computing.
-      K3s provides the cluster need for Zarf running in Appliance Mode as well as can
-      host a low-resource Gitops Service if not using an existing Kubernetes platform.
-    scripts:
-      retry: true
-      before:
-        # If running RHEL variant, disable firewalld
-        # https://rancher.com/docs/k3s/latest/en/advanced/#additional-preparation-for-red-hat-centos-enterprise-linux
-        # NOTE: The empty echo prevents infinite retry loops on non-RHEL systems where the exit code would be an error
-        - "[ -e /etc/redhat-release ] && systemctl disable firewalld --now || echo ''"
-      after:
-        # Configure K3s systemd service
-        - "systemctl daemon-reload"
-        - "systemctl enable --now k3s"
+    import:
+      path: common
+      name: k3s
+    only:
+      cluster:
+        architecture: amd64
     files:
       # Include the actual K3s binary
       - source: https://github.com/k3s-io/k3s/releases/download/v1.24.1+k3s1/k3s
@@ -37,17 +27,27 @@ components:
       - source: https://github.com/k3s-io/k3s/releases/download/v1.24.1+k3s1/k3s-airgap-images-amd64.tar.zst
         shasum: 6736f9fa4d5754d60b0508bafb2f888170cb99a2d93a3a1617a919ca4ee74034
         target: /var/lib/rancher/k3s/agent/images/k3s.tar.zst
-      # K3s removal script
-      - source: assets/zarf-clean-k3s.sh
-        target: /opt/zarf/zarf-clean-k3s.sh
+
+  # ARM-64 version of the K3s stack
+  - name: k3s
+    import:
+      path: common
+      name: k3s
+    only:
+      cluster:
+        architecture: arm64
+    files:
+      # Include the actual K3s binary
+      - source: https://github.com/k3s-io/k3s/releases/download/v1.24.1+k3s1/k3s-arm64
+        shasum: bd8b87d215f7a073d0139a0ab70e3fbeaa76e1b9ce6c00cd8d471cb44ba80687
+        target: /usr/sbin/k3s
         executable: true
-      # The K3s systemd service definition
-      - source: assets/k3s.service
-        target: /etc/systemd/system/k3s.service
-        symlinks:
-          - /etc/systemd/system/multi-user.target.wants/k3s.service
-      # Mock file for creating the kube config symlink
-      - source: assets/empty-file
-        target: /etc/rancher/k3s/k3s.yaml
+        # K3s magic provides these tools when symlinking
         symlinks:
-          - /root/.kube/config
+          - /usr/sbin/kubectl
+          - /usr/sbin/ctr
+          - /usr/sbin/crictl
+      # Transfer the K3s images for containerd to pick them up
+      - source: https://github.com/k3s-io/k3s/releases/download/v1.24.1+k3s1/k3s-airgap-images-arm64.tar.zst
+        shasum: 12029e4bbfecfa16942345aeac798f4790e568a7202c2b85ee068d7b4756ba04
+        target: /var/lib/rancher/k3s/agent/images/k3s.tar.zst

packages/distros/k3s/assets/zarf-clean-k3s.sh packages/distros/k3s/common/zarf-clean-k3s.sh

@@ -182,8 +182,28 @@ func GetRegistry() string {
 	return fmt.Sprintf("%s:%s", IPV4Localhost, state.NodePort)
 }
 
-func LoadConfig(path string) error {
-	return utils.ReadYaml(path, &active)
+// LoadConfig loads the config from the given path and removes
+// components not matching the current OS if filterByOS is set.
+func LoadConfig(path string, filterByOS bool) error {
+	if err := utils.ReadYaml(path, &active); err != nil {
+		return err
+	}
+
+	// Filter each component to only compatible platforms
+	filteredComponents := []types.ZarfComponent{}
+	for _, component := range active.Components {
+		if isCompatibleComponent(component, filterByOS) {
+			filteredComponents = append(filteredComponents, component)
+		}
+	}
+	// Update the active package with the filtered components
+	active.Components = filteredComponents
+
+	return nil
+}
+
+func GetActiveConfig() types.ZarfPackage {
+	return active
 }
 
 func BuildConfig(path string) error {
@@ -231,3 +251,28 @@ func GetImageCachePath() string {
 
 	return strings.Replace(CreateOptions.ImageCachePath, "~", homePath, 1)
 }
+
+func isCompatibleComponent(component types.ZarfComponent, filterByOS bool) bool {
+	message.Debugf("config.isCompatibleComponent(%s, %v)", component.Name, filterByOS)
+
+	// Ignore only filters that are empty
+	var validArch, validOS bool
+
+	targetArch := GetArch()
+
+	// Test for valid architecture
+	if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == targetArch {
+		validArch = true
+	} else {
+		message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, targetArch)
+	}
+
+	// Test for a valid OS
+	if !filterByOS || component.Only.LocalOS == "" || component.Only.LocalOS == runtime.GOOS {
+		validOS = true
+	} else {
+		message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.LocalOS, runtime.GOOS)
+	}
+
+	return validArch && validOS
+}

packages/distros/k3s/common/zarf.yaml

@@ -6,7 +6,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -15,6 +14,7 @@ import (
 
 	"github.com/defenseunicorns/zarf/src/types"
 	"github.com/pterm/pterm"
+	"gopkg.in/yaml.v2"
 
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/defenseunicorns/zarf/src/config"
@@ -75,8 +75,10 @@ func createComponentPaths(basePath string, component types.ZarfComponent) compon
 	}
 }
 
-func confirmAction(configPath, userMessage string, sbomViewFiles []string) bool {
-	content, err := ioutil.ReadFile(configPath)
+func confirmAction(userMessage string, sbomViewFiles []string) bool {
+	active := config.GetActiveConfig()
+
+	content, err := yaml.Marshal(active)
 	if err != nil {
 		message.Fatal(err, "Unable to open the package config file")
 	}

packages/distros/k3s/zarf.yaml

@@ -11,6 +11,8 @@ import (
 )
 
 func GetComponents() (components []types.ZarfComponent) {
+	message.Debugf("packager.GetComponents()")
+
 	for _, component := range config.GetComponents() {
 		if component.Import.Path == "" {
 			components = append(components, component)
@@ -27,6 +29,8 @@ func GetComponents() (components []types.ZarfComponent) {
 }
 
 func GetComposedComponent(childComponent types.ZarfComponent) types.ZarfComponent {
+	message.Debugf("packager.GetComposedComponent(%+v)", childComponent)
+
 	// Make sure the component we're trying to import cant be accessed
 	validateOrBail(&childComponent)
 
@@ -44,6 +48,8 @@ func GetComposedComponent(childComponent types.ZarfComponent) types.ZarfComponen
 }
 
 func getParentComponent(childComponent types.ZarfComponent, everGrowingComposePath string) (parentComponent types.ZarfComponent) {
+	message.Debugf("packager.getParentComponent(%+v, %s)", childComponent, everGrowingComposePath)
+
 	importedPackage, err := getSubPackage(filepath.Join(everGrowingComposePath, childComponent.Import.Path))
 	if err != nil {
 		message.Fatal(err, "Unable to get the package that we're importing a component from")
@@ -56,14 +62,30 @@ func getParentComponent(childComponent types.ZarfComponent, everGrowingComposePa
 		parentComponentName = childComponent.Name
 	}
 
+	targetArch := config.GetArch()
 	// Find the parent component from the imported package that matches our arch
 	for _, importedComponent := range importedPackage.Components {
 		if importedComponent.Name == parentComponentName {
-			parentComponent = importedComponent
-			break
+			filterArch := importedComponent.Only.Cluster.Architecture
+
+			// Override the filter if it is set by the child component
+			if childComponent.Only.Cluster.Architecture != "" {
+				filterArch = childComponent.Only.Cluster.Architecture
+			}
+
+			// Only add this component if it is valid for the target architecture.
+			if filterArch == "" || filterArch == targetArch {
+				parentComponent = importedComponent
+				break
+			}
 		}
 	}
 
+	// If we didn't find a parent component, bail
+	if parentComponent.Name == "" {
+		message.Fatalf(nil, "Unable to find the component %s in the imported package", parentComponentName)
+	}
+
 	// Check if we need to get more of the parents!!!
 	if parentComponent.Import.Path != "" {
 		// Set a temporary composePath so we can get future parents/grandparents from our current location
@@ -86,6 +108,8 @@ func getParentComponent(childComponent types.ZarfComponent, everGrowingComposePa
 }
 
 func fixComposedFilepaths(parentComponent, childComponent types.ZarfComponent) types.ZarfComponent {
+	message.Debugf("packager.fixComposedFilepaths(%+v, %+v)", parentComponent, childComponent)
+
 	// Prefix composed component file paths.
 	for fileIdx, file := range parentComponent.Files {
 		parentComponent.Files[fileIdx].Source = getComposedFilePath(file.Source, childComponent.Import.Path)
@@ -117,13 +141,17 @@ func fixComposedFilepaths(parentComponent, childComponent types.ZarfComponent) t
 
 // Validates the sub component, exits program if validation fails.
 func validateOrBail(component *types.ZarfComponent) {
+	message.Debugf("packager.validateOrBail(%+v)", component)
+
 	if err := validate.ValidateImportPackage(component); err != nil {
 		message.Fatalf(err, "Invalid import definition in the %s component: %s", component.Name, err)
 	}
 }
 
 // Sets Name, Default, Required and Description to the original components values
 func mergeComponentOverrides(target *types.ZarfComponent, override types.ZarfComponent) {
+	message.Debugf("packager.mergeComponentOverrides(%+v, %+v)", target, override)
+
 	target.Name = override.Name
 	target.Default = override.Default
 	target.Required = override.Required
@@ -157,23 +185,38 @@ func mergeComponentOverrides(target *types.ZarfComponent, override types.ZarfCom
 	target.Scripts.After = append(target.Scripts.After, override.Scripts.After...)
 	target.Scripts.ShowOutput = override.Scripts.ShowOutput
 	if override.Scripts.Retry {
-		target.Scripts.Retry = override.Scripts.Retry
+		target.Scripts.Retry = true
+	}
+	if override.Scripts.ShowOutput {
+		target.Scripts.ShowOutput = true
 	}
-
 	if override.Scripts.TimeoutSeconds > 0 {
 		target.Scripts.TimeoutSeconds = override.Scripts.TimeoutSeconds
 	}
+
+	// Merge Only filters
+	target.Only.Cluster.Distros = append(target.Only.Cluster.Distros, override.Only.Cluster.Distros...)
+	if override.Only.Cluster.Architecture != "" {
+		target.Only.Cluster.Architecture = override.Only.Cluster.Architecture
+	}
+	if override.Only.LocalOS != "" {
+		target.Only.LocalOS = override.Only.LocalOS
+	}
 }
 
 // Reads the locally imported zarf.yaml
 func getSubPackage(packagePath string) (importedPackage types.ZarfPackage, err error) {
+	message.Debugf("packager.getSubPackage(%s)", packagePath)
+
 	path := filepath.Join(packagePath, config.ZarfYAML)
 	err = utils.ReadYaml(path, &importedPackage)
 	return importedPackage, err
 }
 
 // Prefix file path with importPath if original file path is not a url.
 func getComposedFilePath(originalPath string, pathPrefix string) string {
+	message.Debugf("packager.getComposedFilePath(%s, %s)", originalPath, pathPrefix)
+
 	// Return original if it is a remote file.
 	if utils.IsUrl(originalPath) {
 		return originalPath