ci: publish multi-arch image manifest lists

This change adds linux/arm64 binaries to the release. It also publishes an arm64
container image for all variants (standard, debug, rootless, static) and releases
(dev, edge, latest).

The build and push process uses buildx in order to push the individual
images by digest (i.e. untagged) and reference them in a single, tagged manifest
list. This avoids cluttering Docker Hub's tag list with `<tag>-<arch>` tags.

Fixes #2233

Signed-off-by: Nick Graef <1031317+ngraef@users.noreply.github.com>

.github/workflows/post-merge.yaml

@@ -90,6 +90,13 @@ jobs:
         env:
           TELEMETRY_URL: ${{ secrets.TELEMETRY_URL }}
 
+      - name: Build Linux arm64
+        run: make ci-go-ci-build-linux ci-go-ci-build-linux-static
+        timeout-minutes: 30
+        env:
+          GOARCH: arm64
+          TELEMETRY_URL: ${{ secrets.TELEMETRY_URL }}
+
       - name: Upload binaries
         uses: actions/upload-artifact@v2
         if: always()
@@ -145,6 +152,9 @@ jobs:
           name: binaries
           path: _release
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
       - name: Deploy OPA Edge
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USER }}

.github/workflows/post-tag.yaml

@@ -32,6 +32,13 @@ jobs:
         env:
           TELEMETRY_URL: ${{ secrets.TELEMETRY_URL }}
 
+      - name: Build Linux arm64
+        run: make ci-go-ci-build-linux ci-go-ci-build-linux-static
+        timeout-minutes: 30
+        env:
+          GOARCH: arm64
+          TELEMETRY_URL: ${{ secrets.TELEMETRY_URL }}
+
       - name: Upload binaries
         uses: actions/upload-artifact@v2
         if: always()
@@ -87,6 +94,9 @@ jobs:
           name: binaries
           path: _release
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
       - name: Build and Deploy OPA Docker Images
         id: build-and-deploy
         env:

.github/workflows/pull-request.yaml

@@ -24,7 +24,7 @@ jobs:
             capabilities.json
 
   go-build:
-    name: Go Build (${{ matrix.os }})
+    name: Go Build (${{ matrix.os }}${{ matrix.arch && format(' {0}', matrix.arch) || '' }})
     runs-on: ${{ matrix.run }}
     needs: generate
     strategy:
@@ -34,6 +34,11 @@ jobs:
           - os: linux
             run: ubuntu-18.04
             targets: ci-go-ci-build-linux ci-go-ci-build-linux-static
+            arch: amd64
+          - os: linux
+            run: ubuntu-18.04
+            targets: ci-go-ci-build-linux ci-go-ci-build-linux-static
+            arch: arm64
           - os: windows
             run: ubuntu-18.04
             targets: ci-go-ci-build-windows
@@ -61,6 +66,8 @@ jobs:
 
       - name: Build
         run: make ${{ matrix.targets }}
+        env:
+          GOARCH: ${{ matrix.arch }}
         timeout-minutes: 30
 
       - name: Upload binaries
@@ -188,15 +195,25 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v2
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64
+
       - name: Download release binaries
         uses: actions/download-artifact@v2
         with:
           name: binaries
           path: _release
 
-      - name: Test images
+      - name: Test amd64 images
         run: make ci-image-smoke-test
 
+      - name: Test arm64 images
+        run: make ci-image-smoke-test
+        env:
+          GOARCH: arm64
+
   smoke-test-binaries:
     runs-on: ${{ matrix.os }}
     needs: go-build
@@ -296,4 +313,3 @@ jobs:
           | opa eval --bundle build/policy/ --format values --stdin-input --fail-defined 'data.files.deny[message]'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-

Dockerfile

@@ -6,20 +6,22 @@ ARG BASE
 
 FROM ${BASE}
 
+LABEL org.opencontainers.image.authors="Torin Sandall <torinsandall@gmail.com>"
+
 # Any non-zero number will do, and unfortunately a named user will not, as k8s
 # pod securityContext runAsNonRoot can't resolve the user ID:
 # https://github.com/kubernetes/kubernetes/issues/40958. Make root (uid 0) when
 # not specified.
 ARG USER=0
-
-MAINTAINER Torin Sandall <torinsandall@gmail.com>
-
-# Hack.. https://github.com/moby/moby/issues/37965
-# _Something_ needs to be between the two COPY steps.
 USER ${USER}
 
-ARG BIN=./opa_linux_amd64
-COPY ${BIN} /opa
+# TARGETOS and TARGETARCH are automatic platform args injected by BuildKit
+# https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope
+ARG TARGETOS
+ARG TARGETARCH
+ARG BIN_DIR=.
+ARG BIN_SUFFIX=
+COPY ${BIN_DIR}/opa_${TARGETOS}_${TARGETARCH}${BIN_SUFFIX} /opa
 
 ENTRYPOINT ["/opa"]
 CMD ["run"]

Makefile

@@ -41,6 +41,12 @@ endif
 
 DOCKER := docker
 
+# BuildKit is required for automatic platform arg injection (see Dockerfile)
+export DOCKER_BUILDKIT := 1
+
+# Supported platforms to include in image manifest lists
+DOCKER_PLATFORMS := linux/amd64,linux/arm64
+
 BIN := opa_$(GOOS)_$(GOARCH)
 
 # Optional external configuration useful for forks of OPA
@@ -238,6 +244,7 @@ CI_GOLANG_DOCKER_MAKE := $(DOCKER) run \
 	-v $(PWD):/src \
 	-w /src \
 	-e GOCACHE=/src/.go/cache \
+	-e GOARCH=$(GOARCH) \
 	-e CGO_ENABLED=$(CGO_ENABLED) \
 	-e WASM_ENABLED=$(WASM_ENABLED) \
 	-e FUZZ_TIME=$(FUZZ_TIME) \
@@ -261,7 +268,7 @@ ci-check-working-copy: generate
 ci-wasm: wasm-test
 
 .PHONY: ci-build-linux
-ci-build-linux: ensure-release-dir
+ci-build-linux: ensure-release-dir ensure-linux-toolchain
 	@$(MAKE) build GOOS=linux
 	chmod +x opa_linux_$(GOARCH)
 	mv opa_linux_$(GOARCH) $(RELEASE_DIR)/
@@ -301,107 +308,130 @@ ci-build-windows: ensure-release-dir
 ensure-release-dir:
 	mkdir -p $(RELEASE_DIR)
 
+.PHONY: ensure-executable-bin
+ensure-executable-bin:
+	find $(RELEASE_DIR) -type f ! -name "*.sha256" | xargs chmod +x
+
+.PHONY: ensure-linux-toolchain
+ensure-linux-toolchain:
+ifeq ($(CGO_ENABLED),1)
+	$(eval export CC = $(shell GOARCH=$(GOARCH) build/ensure-linux-toolchain.sh))
+else
+	@echo "CGO_ENABLED=$(CGO_ENABLED). No need to check gcc toolchain."
+endif
+
 .PHONY: build-all-platforms
 build-all-platforms: ci-build-linux ci-build-linux-static ci-build-darwin ci-build-darwin-arm64-static ci-build-windows
 
 .PHONY: image-quick
-image-quick:
-	chmod +x $(RELEASE_DIR)/opa_linux_$(GOARCH)*
+image-quick: image-quick-$(GOARCH)
+
+# % = arch
+.PHONY: image-quick-%
+image-quick-%: ensure-executable-bin
 	$(DOCKER) build \
 		-t $(DOCKER_IMAGE):$(VERSION) \
 		--build-arg BASE=gcr.io/distroless/cc \
-		--build-arg BIN=$(RELEASE_DIR)/opa_linux_$(GOARCH) \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--platform linux/$* \
 		.
 	$(DOCKER) build \
 		-t $(DOCKER_IMAGE):$(VERSION)-debug \
 		--build-arg BASE=gcr.io/distroless/cc:debug \
-		--build-arg BIN=$(RELEASE_DIR)/opa_linux_$(GOARCH) \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--platform linux/$* \
 		.
 	$(DOCKER) build \
 		-t $(DOCKER_IMAGE):$(VERSION)-rootless \
 		--build-arg USER=1000 \
 		--build-arg BASE=gcr.io/distroless/cc \
-		--build-arg BIN=$(RELEASE_DIR)/opa_linux_$(GOARCH) \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--platform linux/$* \
 		.
 	$(DOCKER) build \
 		-t $(DOCKER_IMAGE):$(VERSION)-static \
 		--build-arg BASE=gcr.io/distroless/static \
-		--build-arg BIN=$(RELEASE_DIR)/opa_linux_$(GOARCH)_static \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--build-arg BIN_SUFFIX=_static \
+		--platform linux/$* \
+		.
+
+# % = base tag
+.PHONY: push-manifest-list-%
+push-manifest-list-%: ensure-executable-bin
+	$(DOCKER) buildx build \
+		--tag $(DOCKER_IMAGE):$* \
+		--build-arg BASE=gcr.io/distroless/cc \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--platform $(DOCKER_PLATFORMS) \
+		--push \
+		.
+	$(DOCKER) buildx build \
+		--tag $(DOCKER_IMAGE):$*-debug \
+		--build-arg BASE=gcr.io/distroless/cc:debug \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--platform $(DOCKER_PLATFORMS) \
+		--push \
+		.
+	$(DOCKER) buildx build \
+		--tag $(DOCKER_IMAGE):$*-rootless \
+		--build-arg USER=1000 \
+		--build-arg BASE=gcr.io/distroless/cc \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--platform $(DOCKER_PLATFORMS) \
+		--push \
+		.
+	$(DOCKER) buildx build \
+		--tag $(DOCKER_IMAGE):$*-static \
+		--build-arg BASE=gcr.io/distroless/static \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--build-arg BIN_SUFFIX=_static \
+		--platform $(DOCKER_PLATFORMS) \
+		--push \
 		.
 
 .PHONY: ci-image-smoke-test
-ci-image-smoke-test: image-quick
-	$(DOCKER) run $(DOCKER_IMAGE):$(VERSION) version
-	$(DOCKER) run $(DOCKER_IMAGE):$(VERSION)-debug version
-	$(DOCKER) run $(DOCKER_IMAGE):$(VERSION)-rootless version
-	$(DOCKER) run $(DOCKER_IMAGE):$(VERSION)-static version
+ci-image-smoke-test: ci-image-smoke-test-$(GOARCH)
+
+# % = arch
+.PHONY: ci-image-smoke-test-%
+ci-image-smoke-test-%: image-quick-%
+	$(DOCKER) run --platform linux/$* $(DOCKER_IMAGE):$(VERSION) version
+	$(DOCKER) run --platform linux/$* $(DOCKER_IMAGE):$(VERSION)-debug version
+	$(DOCKER) run --platform linux/$* $(DOCKER_IMAGE):$(VERSION)-rootless version
+	$(DOCKER) run --platform linux/$* $(DOCKER_IMAGE):$(VERSION)-static version
 
 .PHONY: ci-binary-smoke-test-%
 ci-binary-smoke-test-%:
 	chmod +x "$(RELEASE_DIR)/$(BINARY)"
 	"$(RELEASE_DIR)/$(BINARY)" eval -t "$*" 'time.now_ns()'
 
-.PHONY: push
-push:
-	$(DOCKER) push $(DOCKER_IMAGE):$(VERSION)
-	$(DOCKER) push $(DOCKER_IMAGE):$(VERSION)-debug
-	$(DOCKER) push $(DOCKER_IMAGE):$(VERSION)-rootless
-	$(DOCKER) push $(DOCKER_IMAGE):$(VERSION)-static
-
-.PHONY: tag-latest
-tag-latest:
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION)-debug $(DOCKER_IMAGE):latest-debug
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION)-rootless $(DOCKER_IMAGE):latest-rootless
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION)-static $(DOCKER_IMAGE):latest-static
-
-.PHONY: push-latest
-push-latest:
-	$(DOCKER) push $(DOCKER_IMAGE):latest
-	$(DOCKER) push $(DOCKER_IMAGE):latest-debug
-	$(DOCKER) push $(DOCKER_IMAGE):latest-rootless
-	$(DOCKER) push $(DOCKER_IMAGE):latest-static
-
 .PHONY: push-binary-edge
 push-binary-edge:
 	aws s3 sync $(RELEASE_DIR) s3://$(S3_RELEASE_BUCKET)/edge/ --delete
 
-.PHONY: tag-edge
-tag-edge:
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):edge
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION)-debug $(DOCKER_IMAGE):edge-debug
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION)-rootless $(DOCKER_IMAGE):edge-rootless
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION)-static $(DOCKER_IMAGE):edge-static
-
-.PHONY: push-edge
-push-edge:
-	$(DOCKER) push $(DOCKER_IMAGE):edge
-	$(DOCKER) push $(DOCKER_IMAGE):edge-debug
-	$(DOCKER) push $(DOCKER_IMAGE):edge-rootless
-	$(DOCKER) push $(DOCKER_IMAGE):edge-static
-
 .PHONY: docker-login
 docker-login:
 	@echo "Docker Login..."
 	@echo ${DOCKER_PASSWORD} | $(DOCKER) login -u ${DOCKER_USER} --password-stdin
 
 .PHONY: push-image
-push-image: docker-login image-quick push
+push-image: docker-login push-manifest-list-$(VERSION)
 
 .PHONY: push-wasm-builder-image
 push-wasm-builder-image: docker-login
 	$(MAKE) -C wasm push-builder
 
 .PHONY: deploy-ci
-deploy-ci: push-image tag-edge push-edge push-binary-edge
+deploy-ci: push-image push-manifest-list-edge push-binary-edge
 
 .PHONY: release-ci
 # Don't tag and push "latest" image tags if the version is a release candidate or a bugfix branch
 # where the changes don't exist in main
 ifneq (,$(or $(findstring rc,$(VERSION)), $(findstring release-,$(shell git branch --contains HEAD))))
 release-ci: push-image
 else
-release-ci: push-image tag-latest push-latest
+release-ci: push-image push-manifest-list-latest
 endif
 
 .PHONY: netlify-prod