Automate conformance testing on PR using zot

.github/workflows/build-pr.yml

@@ -23,8 +23,29 @@ jobs:
           go-version: ${{ matrix.go }}
 
       - name: run tests
+        id: tests
         run: |
           export PATH="$(go env GOPATH)/bin:${PATH}"
           make install.tools
           make .gitvalidation
           make docs conformance
+
+          set +e
+          make conformance-ci
+          CONFORMANCE_RC="$?"
+          set -e
+          if [[ -f report.html ]]; then
+            echo "Found report.html."
+            echo "has-report=true" >> $GITHUB_OUTPUT
+          fi
+          echo "Conformance return code: ${CONFORMANCE_RC}"
+          exit ${CONFORMANCE_RC}
+
+      - name: Upload OCI conformance results as build artifact
+        if: always() && steps.tests.outputs.has-report == 'true'
+        uses: actions/upload-artifact@v3
+        with:
+          name: oci-conformance-results-${{ matrix.go }}
+          path: |
+            ./report.html
+            ./junit.xml

.github/workflows/build.yml

@@ -23,8 +23,29 @@ jobs:
           go-version: ${{ matrix.go }}
 
       - name: run tests
+        id: tests
         run: |
           export PATH="$(go env GOPATH)/bin:${PATH}"
           make install.tools
           make .gitvalidation
           make docs conformance
+
+          set +e
+          make conformance-ci
+          CONFORMANCE_RC="$?"
+          set -e
+          if [[ -f report.html ]]; then
+            echo "Found report.html."
+            echo "has-report=true" >> $GITHUB_OUTPUT
+          fi
+          echo "Conformance return code: ${CONFORMANCE_RC}"
+          exit ${CONFORMANCE_RC}
+
+      - name: Upload OCI conformance results as build artifact
+        if: always() && steps.tests.outputs.has-report == 'true'
+        uses: actions/upload-artifact@v3
+        with:
+          name: oci-conformance-results-${{ matrix.go }}
+          path: |
+            ./report.html
+            ./junit.xml

.gitignore

@@ -5,3 +5,5 @@ header.html
 tags
 go.mod
 go.sum
+junit.xml
+report.html

Makefile

@@ -91,6 +91,24 @@ conformance-test:
 
 conformance-binary: $(OUTPUT_DIRNAME)/conformance.test
 
+TEST_REGISTRY_CONTAINER ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.0.0-rc3
+conformance-ci:
+	docker rm -f oci-conformance && \
+		echo '{"distSpecVersion":"1.1.0-dev","storage":{"rootDirectory":"/tmp/zot","gc":false,"dedupe":false},"http":{"address":"0.0.0.0","port":"5000"},"log":{"level":"debug"}}' > $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json
+		docker run -d \
+			-v $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json:/etc/zot/config.json \
+			--name=oci-conformance \
+			-p 5000:5000 \
+			$(TEST_REGISTRY_CONTAINER) && \
+		export OCI_ROOT_URL="http://localhost:5000" && \
+		export OCI_NAMESPACE="myorg/myrepo" && \
+		export OCI_TEST_PULL=1 && \
+		export OCI_TEST_PUSH=1 && \
+		export OCI_TEST_CONTENT_DISCOVERY=1 && \
+		export OCI_TEST_CONTENT_MANAGEMENT=1 && \
+		sleep 5 && \
+		$(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test
+
 $(OUTPUT_DIRNAME)/conformance.test:
 	cd conformance && \
 		CGO_ENABLED=0 go test -c -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test \

conformance/02_push_test.go

@@ -188,7 +188,6 @@ var test02Push = func() {
 				location := resp.Header().Get("Location")
 				Expect(location).ToNot(BeEmpty())
 				prevResponse = resp
-
 				req = client.NewRequest(reggie.PATCH, resp.GetRelativeLocation()).
 					SetHeader("Content-Type", "application/octet-stream").
 					SetHeader("Content-Length", testBlobBChunk1Length).
@@ -220,7 +219,7 @@ var test02Push = func() {
 				Expect(err).To(BeNil())
 				Expect(resp.StatusCode()).To(Equal(http.StatusNoContent))
 				Expect(resp.Header().Get("Location")).ToNot(BeEmpty())
-				Expect(resp.Header().Get("Range")).To(Equal(fmt.Sprintf("bytes=%s", testBlobBChunk1Range)))
+				Expect(resp.Header().Get("Range")).To(Equal(testBlobBChunk1Range))
 				lastResponse = resp
 			})
 
@@ -277,15 +276,13 @@ var test02Push = func() {
 					Equal(http.StatusCreated),
 					Equal(http.StatusAccepted),
 				))
-				Expect(resp.GetRelativeLocation()).To(Equal(fmt.Sprintf("/v2/%s/blobs/%s", crossmountNamespace, testBlobADigest)))
-
 				lastResponse = resp
 			})
 
 			g.Specify("GET request to test digest within cross-mount namespace should return 200", func() {
 				SkipIfDisabled(push)
 				RunOnlyIf(lastResponse.StatusCode() == http.StatusCreated)
-
+				Expect(lastResponse.GetRelativeLocation()).To(Equal(fmt.Sprintf("/v2/%s/blobs/%s", crossmountNamespace, testBlobADigest)))
 				req := client.NewRequest(reggie.GET, lastResponse.GetRelativeLocation())
 				resp, err := client.Do(req)
 				Expect(err).To(BeNil())
@@ -295,7 +292,6 @@ var test02Push = func() {
 			g.Specify("Cross-mounting of nonexistent blob should yield session id", func() {
 				SkipIfDisabled(push)
 				RunOnlyIf(lastResponse.StatusCode() == http.StatusAccepted)
-
 				Expect(lastResponse.GetRelativeLocation()).To(HavePrefix(fmt.Sprintf("/v2/%s/blobs/uploads/", crossmountNamespace)))
 			})
 
@@ -304,7 +300,6 @@ var test02Push = func() {
 				RunOnlyIf(runAutomaticCrossmountTest)
 				RunOnlyIf(lastResponse.StatusCode() == http.StatusCreated)
 				RunOnlyIf(automaticCrossmountEnabled)
-
 				req := client.NewRequest(reggie.POST, "/v2/<name>/blobs/uploads/",
 					reggie.WithName(crossmountNamespace)).
 					SetQueryParam("mount", testBlobADigest)
@@ -318,7 +313,6 @@ var test02Push = func() {
 				RunOnlyIf(runAutomaticCrossmountTest)
 				RunOnlyIf(lastResponse.StatusCode() == http.StatusCreated)
 				RunOnlyIfNot(automaticCrossmountEnabled)
-
 				req := client.NewRequest(reggie.POST, "/v2/<name>/blobs/uploads/",
 					reggie.WithName(crossmountNamespace)).
 					SetQueryParam("mount", testBlobADigest)
@@ -442,7 +436,6 @@ var test02Push = func() {
 					))
 				})
 			}
-
 		})
 	})
 }