Big Bang Core Example

.editorconfig

@@ -11,4 +11,7 @@ max_line_length = 120
 tab_width = 4
 
 [*.md]
-trim_trailing_whitespace = false
+trim_trailing_whitespace = false
+
+[Makefile]
+indent_style = tab

Makefile

@@ -16,7 +16,7 @@ endif
 
 # remove all zarf packages recursively
 remove-packages:
-	find . -type f -name 'zarf-package*' -delete
+	find . -type f -name 'zarf-package-*' -delete
 
 # usage: make test OS=ubuntu
 test:
@@ -27,21 +27,21 @@ test:
 test-close:
 	vagrant destroy -f
 
-package:
+init-package:
 	$(ZARF_BIN) package create --confirm
-	mv zarf*.tar.zst build
+	mv zarf-init.tar.zst build
 
-	cd build && sha256sum -b zarf* > zarf.sha256	
+	cd build && sha256sum -b zarf* > zarf.sha256
 	ls -lh build
 
 build-cli:
 	rm -fr build
 	cd cli && $(MAKE) build
 	cd cli && $(MAKE) build-mac
 
-build-test: build-cli package
+build-test: build-cli init-package
 
-ci-release: package
+ci-release: init-package
 
 # automatically package all example directories and add the tarballs to the build directory
 package-examples:

Vagrantfile

@@ -1,5 +1,13 @@
 Vagrant.configure("2") do |config|
 
+  config.vm.provider "virtualbox" do |vb|
+      vb.check_guest_additions = false
+      vb.cpus = 6
+      vb.memory = 8192
+    end
+
+  config.vm.disk :disk, size: "20GB", primary: true
+
   config.vm.define "rhel7" do |target|
     target.vm.box = "generic/rhel7"
   end
@@ -35,20 +43,10 @@ Vagrant.configure("2") do |config|
   config.vm.network "forwarded_port", guest: 80, host: 8080
   config.vm.network "forwarded_port", guest: 443, host: 8443
 
-  config.vm.disk :disk, size: "20GB", primary: true
   config.ssh.insert_key = false
 
-  config.vm.provider "virtualbox" do |vb|
-    vb.check_guest_additions = false
-    vb.cpus = 6
-    vb.memory = 8192
-  end
-
   config.vm.provision "shell", inline: <<-SHELL
-    cd /opt/zarf
     # Airgap images please
-    echo "0.0.0.0 charts.helm.sh repo1.dso.mil github.com registry.dso.mil registry1.dso.mil index.docker.io auth.docker.io registry-1.docker.io dseasb33srnrn.cloudfront.net production.cloudflare.docker.com registry.opensource.zalan.do" >> /etc/hosts
-    # ./zarf init --confirm --host=localhost
-  SHELL
-
+    echo "0.0.0.0 registry.hub.docker.com hub.docker.com charts.helm.sh repo1.dso.mil github.com registry.dso.mil registry1.dso.mil docker.io index.docker.io auth.docker.io registry-1.docker.io dseasb33srnrn.cloudfront.net production.cloudflare.docker.com registry.opensource.zalan.do" >> /etc/hosts
+    SHELL
 end

examples/.gitignore

@@ -0,0 +1 @@
+sync/

examples/Makefile

@@ -1,5 +1,5 @@
 # Figure out which Zarf binary we should use based on the operating system we are on
-ZARF_BIN := ../../build/zarf
+ZARF_BIN := ../sync/zarf
 UNAME_S := $(shell uname -s)
 UNAME_P := $(shell uname -p)
 ifneq ($(UNAME_S),Linux)
@@ -14,10 +14,73 @@ ifneq ($(UNAME_S),Linux)
 	endif
 endif
 
-package-examples:
-	cd appliance               && $(ZARF_BIN) package create --confirm && mv zarf*.tar.zst ../../build
-	cd data-injection          && $(ZARF_BIN) package create --confirm && mv zarf*.tar     ../../build
-	cd game                    && $(ZARF_BIN) package create --confirm && mv zarf*.tar.zst ../../build
-	cd single-big-bang-package && $(ZARF_BIN) package create --confirm && mv zarf*.tar.zst ../../build
-	cd tiny-kafka              && $(ZARF_BIN) package create --confirm && mv zarf*.tar.zst ../../build
+# Download zarf, build all packages and launch a basic VM with the assets
+.PHONY: all
+all: fetch-release package-examples vm-init
+
+# Same as target 'all', but build the binaries using the current codebase rather than downloading the latest version from the internet
+.PHONY: all-dev
+all-dev: build-release package-examples vm-init
+
+# Clean the sync dir
+.PHONY: clean
+clean:
+	@rm -fr sync && mkdir -p sync
+
+# Grab the latest release as an alternative to needing to build the binaries
+.PHONY: fetch-release
+fetch-release: clean
+	@# This probably isn't the cleanest way to get a release, but since we're moving to github, not worth adding the code until post-migration
+	@curl -fL "https://zarf-public.s3-us-gov-west-1.amazonaws.com/release/$$(git describe --tags --abbrev=0)/{zarf,zarf-mac-intel,zarf-mac-apple,zarf-init.tar.zst}" -o "sync/#1"
+	@chmod +x sync/*
+
+# Build the binaries as an alternative to downloading the latest release
+.PHONY: build-release
+build-release: clean
+	@cd .. && $(MAKE) build-test
+	@cp -R ../build/* sync
+
+# Stripped-down vagrant box to reduce friction for basic user testing
+# Note the need to perform disk resizing for some examples
+.PHONY: vm-init
+vm-init: vm-destroy
+	@VAGRANT_EXPERIMENTAL="disks" vagrant up --no-color
+	@echo -e "\n\n\n\033[1;93m  ✅ VM READY.  Logging in now, run \"sudo su\" once the prompt appears.\n\n\n\033[0m"
+	@vagrant ssh
+
+# Cleanup plz
+.PHONY: vm-destroy
+vm-destroy:
+	@vagrant destroy -f
+
+# Create zarf packages from all examples
+.PHONY: package-examples
+package-examples: package-example-big-bang package-example-appliance package-example-data-injection package-example-game package-example-single-big-bang-package package-example-tiny-kafka
+
+.PHONY: package-example-big-bang
+package-example-big-bang:
+	cd big-bang && kustomize build template/bigbang > manifests/bigbang_generated.yaml && kustomize build template/flux > manifests/flux_generated.yaml && $(ZARF_BIN) package create --confirm && mv zarf-package-* ../sync/
+
+.PHONY: package-example-appliance
+package-example-appliance:
+	cd appliance && $(ZARF_BIN) package create --confirm && mv zarf-package-* ../sync/
+
+.PHONY: package-example-data-injection
+package-example-data-injection:
+	cd data-injection && $(ZARF_BIN) package create --confirm && mv zarf-package-* ../sync/
+
+.PHONY: package-example-game
+package-example-game:
+	cd game && $(ZARF_BIN) package create --confirm && mv zarf-package-* ../sync/
+
+.PHONY: package-example-single-big-bang-package
+package-example-single-big-bang-package:
+	cd single-big-bang-package && $(ZARF_BIN) package create --confirm && mv zarf-package-* ../sync/
+
+.PHONY: package-example-tiny-kafka
+package-example-tiny-kafka:
+	cd tiny-kafka && $(ZARF_BIN) package create --confirm && mv zarf-package-* ../sync/
+
+.PHONY: package-example-postgres-operator
+package-example-postgres-operator:
 	cd postgres-operator       && $(ZARF_BIN) package create --confirm && mv zarf*.tar.zst ../../build

examples/README.md

@@ -1 +1,5 @@
+# Zarf Examples
+
 The Zarf examples demonstrate different ways to utility Zarf in your environment.  All of these examples follow the same general release pattern and assume an offline / air-gapped deployment target.  
+
+To test create a virtual area to test all examples, you can run `make all` or `make vm-init` if you've already run the examples before. Run `make vm-destroy` to clean up.  

examples/Vagrantfile

@@ -0,0 +1,32 @@
+Vagrant.configure("2") do |config|
+  config.vm.provider "virtualbox" do |vb|
+    vb.check_guest_additions = false
+    vb.cpus = 6
+    vb.memory = 20000
+  end
+
+  config.vm.disk :disk, size: "100GB", primary: true
+  config.vm.box = "boxomatic/ubuntu-20.04"
+
+  config.vm.hostname = "zarf-examples"
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+  config.vm.synced_folder './sync/', '/home/vagrant/zarf-examples', SharedFoldersEnableSymlinksCreate: false
+
+  config.vm.network "forwarded_port", guest: 80, host: 8080
+  config.vm.network "forwarded_port", guest: 443, host: 8443
+  config.vm.network "forwarded_port", guest: 8080, host: 9080
+  config.vm.network "forwarded_port", guest: 8443, host: 9443
+
+  config.ssh.insert_key = false
+
+  config.vm.provision "shell", inline: <<-SHELL
+    # The partition is 100GB but the filesystem isn't yet
+    growpart /dev/sda 1 && resize2fs /dev/sda1
+
+    # Elasticsearch needs this
+    sysctl -w vm.max_map_count=262144
+
+    # Airgap images please
+    echo "0.0.0.0 registry.hub.docker.com hub.docker.com charts.helm.sh repo1.dso.mil github.com registry.dso.mil registry1.dso.mil docker.io index.docker.io auth.docker.io registry-1.docker.io dseasb33srnrn.cloudfront.net production.cloudflare.docker.com" >> /etc/hosts
+    SHELL
+end

examples/big-bang/README.md

@@ -0,0 +1,26 @@
+# Example: Big Bang Core All-In-One
+
+This example deploys Big Bang Core to a Utility Cluster. This is not normally the method that will be used in production but for a demo it works great.
+
+Because the same cluster will be running both Traefik and Istio, Istio's VirtualServices will be available on port 9443
+
+## Prerequisites
+
+1. Install [Vagrant](https://www.vagrantup.com/)
+2. Install `make` and `kustomize`
+
+## Instructions
+
+1. From within the examples directory, Run: `make all`, which will download the latest built binaries, build all of the example packages, and launch a basic VM to run in. Alternatively, run `make all-dev` if you want to build the binaries using the current codebase instead of downloading them.
+5. Run: `sudo su` - Change user to root
+6. Run: `cd zarf-examples` - Change to the directory where the examples folder is mounted
+7. Run: `./zarf init --confirm --features management,utility-cluster --host localhost` - Initialize Zarf, telling it to install the management feature and utility cluster and skip logging feature (since BB has logging already) and tells Zarf to use `localhost` as the domain
+8. Wait a bit, run `k9s` to see pods come up. Don't move on until everything is running
+9. Run: `./zarf package deploy zarf-package-big-bang-core-demo.tar.zst --confirm` - Deploy Big Bang Core
+10. Wait several minutes. Run `k9s` to watch progress
+11. Use a browser to visit the various services, available at https://*.bigbang.dev:9443
+12. When you're done, run `make vm-destroy` to bring everything down
+
+## To-Do
+
+1. Re-enable the NetworkPolicies - They got disabled to resolve an issue connecting to the k8s cluster API server, which is fine for a demo but unacceptable in production

examples/big-bang/manifests/.gitignore

@@ -0,0 +1 @@
+*_generated.yaml

examples/big-bang/manifests/other_manifests.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: zarf-git-secret
+  namespace: bigbang
+stringData:
+  username: "zarf-git-user"
+  password: "###ZARF_SECRET###"

examples/big-bang/template/bigbang/kustomization.yaml

@@ -0,0 +1,21 @@
+bases:
+  - git::https://repo1.dso.mil/platform-one/big-bang/bigbang.git/base?ref=1.14.1
+
+configMapGenerator:
+  - name: common
+    namespace: bigbang
+    behavior: merge
+    files:
+      - values.yaml
+
+patchesStrategicMerge:
+- |-
+  apiVersion: source.toolkit.fluxcd.io/v1beta1
+  kind: GitRepository
+  metadata:
+    name: bigbang
+    namespace: bigbang
+  spec:
+    url: http://stuart-gitea-http.git.svc.cluster.local:3000/zarf-git-user/mirror__repo1.dso.mil__platform-one__big-bang__bigbang.git
+    secretRef:
+      name: zarf-git-secret

examples/big-bang/template/flux/kustomization.yaml

@@ -0,0 +1,2 @@
+bases:
+  - git::https://repo1.dso.mil/platform-one/big-bang/bigbang.git/base/flux?ref=tags/1.14.1

examples/big-bang/zarf.yaml

@@ -0,0 +1,65 @@
+kind: ZarfPackageConfig
+metadata:
+  name: big-bang-core-demo
+  description: "Demo Zarf basic deployment of Big Bang core"
+
+local:
+  manifests: manifests
+
+  images:
+    # Flux images
+    - registry1.dso.mil/ironbank/fluxcd/helm-controller:v0.11.0
+    - registry1.dso.mil/ironbank/fluxcd/kustomize-controller:v0.13.0
+    - registry1.dso.mil/ironbank/fluxcd/notification-controller:v0.15.0
+    - registry1.dso.mil/ironbank/fluxcd/source-controller:v0.14.0
+
+remote:
+  # 1. helm template bigbang ./chart |  yq e '. | select(.kind == "GitRepository") | "- " + .spec.url + "@" + .spec.ref.tag' -
+  # 2. Add the actual bigbang repo as well
+  # https://repo1.dso.mil/platform-one/big-bang/bigbang/-/tags/1.14.1
+  repos:
+    - https://repo1.dso.mil/platform-one/big-bang/bigbang.git@1.14.1
+    - https://repo1.dso.mil/platform-one/big-bang/apps/core/cluster-auditor.git@0.3.0-bb.5
+    - https://repo1.dso.mil/platform-one/big-bang/apps/core/policy.git@3.5.1-bb.8
+    - https://repo1.dso.mil/platform-one/big-bang/apps/core/istio-controlplane.git@1.8.4-bb.6
+    - https://repo1.dso.mil/platform-one/big-bang/apps/core/istio-operator.git@1.8.4-bb.2
+    - https://repo1.dso.mil/platform-one/big-bang/apps/core/jaeger.git@2.23.0-bb.1
+    - https://repo1.dso.mil/platform-one/big-bang/apps/core/kiali.git@1.37.0-bb.0
+    - https://repo1.dso.mil/platform-one/big-bang/apps/core/eck-operator.git@1.6.0-bb.2
+    - https://repo1.dso.mil/platform-one/big-bang/apps/core/elasticsearch-kibana.git@0.1.18-bb.0
+    - https://repo1.dso.mil/platform-one/big-bang/apps/core/fluentbit.git@0.16.1-bb.0
+    - https://repo1.dso.mil/platform-one/big-bang/apps/core/monitoring.git@14.0.0-bb.3
+    - https://repo1.dso.mil/platform-one/big-bang/apps/security-tools/twistlock.git@0.0.6-bb.1
+
+  images:
+    # TODO: Figure out a better way to derive this list.
+    # 1. Deploy Big Bang Core using some other method like https://repo1.dso.mil/platform-one/quick-start/big-bang
+    # 2. kubectl get pods --all-namespaces -o json | jq '.items[].spec.containers[].image' | jq -s 'unique' | yq e -P
+    # 3. Move all 'registry1.dso.mil/ironbank/fluxcd' images to the 'local.images' section
+    # 4. Add 'docker.io/' to any images that aren't fully qualified (example: rancher/metrics-server -> docker.io/rancher/metrics-server
+    - registry1.dso.mil/ironbank/cluster-auditor/opa-collector:0.3.2
+    - registry1.dso.mil/ironbank/elastic/eck-operator/eck-operator:1.6.0
+    - registry1.dso.mil/ironbank/elastic/elasticsearch/elasticsearch:7.13.4
+    - registry1.dso.mil/ironbank/elastic/kibana/kibana:7.12.0
+    - registry1.dso.mil/ironbank/kiwigrid/k8s-sidecar:1.10.6
+    - registry1.dso.mil/ironbank/opensource/coreos/kube-state-metrics:v1.9.8
+    - registry1.dso.mil/ironbank/opensource/fluent/fluent-bit:1.8.1
+    - registry1.dso.mil/ironbank/opensource/grafana/grafana:7.5.2
+    - registry1.dso.mil/ironbank/opensource/istio-1.8/operator:1.8.4
+    - registry1.dso.mil/ironbank/opensource/istio-1.8/pilot:1.8.4
+    - registry1.dso.mil/ironbank/opensource/istio-1.8/proxyv2:1.8.4
+    - registry1.dso.mil/ironbank/opensource/jaegertracing/all-in-one:1.24.0
+    - registry1.dso.mil/ironbank/opensource/jaegertracing/jaeger-operator:1.24.0
+    - registry1.dso.mil/ironbank/opensource/jet/kube-webhook-certgen:v1.5.1
+    - registry1.dso.mil/ironbank/opensource/kiali/kiali-operator:v1.37.0
+    - registry1.dso.mil/ironbank/opensource/kiali/kiali:v1.37.0
+    - registry1.dso.mil/ironbank/opensource/kubernetes-1.21/kubectl:v1.21.1
+    - registry1.dso.mil/ironbank/opensource/openpolicyagent/gatekeeper:v3.5.1
+    - registry1.dso.mil/ironbank/opensource/prometheus-operator/prometheus-config-reloader:v0.46.0
+    - registry1.dso.mil/ironbank/opensource/prometheus-operator/prometheus-operator:v0.46.0
+    - registry1.dso.mil/ironbank/opensource/prometheus/alertmanager:v0.21.0
+    - registry1.dso.mil/ironbank/opensource/prometheus/node-exporter:v1.0.1
+    - registry1.dso.mil/ironbank/opensource/prometheus/prometheus:v2.25.0
+    - registry1.dso.mil/ironbank/twistlock/console/console:21.04.412
+
+