workflow for publishing artifacts for cloud (#14199)

airbytehq · Jun 28, 2022 · adc04a2 · adc04a2
1 parent 4072d46
commit adc04a2
Show file tree

Hide file tree

Showing 2 changed files with 110 additions and 21 deletions.
diff --git a/.github/workflows/publish-oss-for-cloud.yml b/.github/workflows/publish-oss-for-cloud.yml
@@ -1,6 +1,6 @@
 name: Publish OSS Artifacts for Cloud
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ inputs.oss_ref || github.sha }}
 
 on:
   workflow_dispatch:
@@ -46,7 +46,7 @@ jobs:
           github-token: ${{ needs.find_valid_pat.outputs.pat }}
 
   generate-tags:
-    name: "Generate Tags"
+    name: "Generate Dev and Master Tags"
     runs-on: ubuntu-latest
     outputs:
       dev_tag: ${{ steps.set-outputs.outputs.dev_tag }}
@@ -69,16 +69,17 @@ jobs:
           echo "::set-output name=dev_tag::dev-${commit_sha}"
           
           # If this commit is on the master branch, also set master_tag
-          if test 0 -eq $(git merge-base --is-ancestor "${commit_sha}" master); then
+          if $(git merge-base --is-ancestor "${commit_sha}" master); then
             echo "::set-output name=master_tag::${commit_sha}"
           fi
 
   oss-branch-build:
-    name: "Build and Push Images from Branch"
+    name: "Gradle Build and Publish"
     needs:
       - start-runner
       - generate-tags
     runs-on: ${{ needs.start-runner.outputs.label }}
+    environment: more-secrets
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2
@@ -90,37 +91,55 @@ jobs:
         with:
           branch_version_tag: ${{ needs.generate-tags.outputs.dev_tag }}
 
+      - name: Publish Dev Jars
+        env:
+          CLOUDREPO_USER: ${{ secrets.CLOUDREPO_USER }}
+          CLOUDREPO_PASSWORD: ${{ secrets.CLOUDREPO_PASSWORD }}
+        run: VERSION=${{ needs.generate-tags.outputs.dev_tag }} SUB_BUILD=PLATFORM ./gradlew publish
+        shell: bash
+
+      - name: Publish Master Jars
+        if: needs.generate-tags.outputs.master_tag != ''
+        env:
+          CLOUDREPO_USER: ${{ secrets.CLOUDREPO_USER }}
+          CLOUDREPO_PASSWORD: ${{ secrets.CLOUDREPO_PASSWORD }}
+        run: VERSION=${{ needs.generate-tags.outputs.master_tag }} SUB_BUILD=PLATFORM ./gradlew publish
+        shell: bash
+
+  docker-push:
+    name: "Push Docker Images"
+    needs:
+      - start-runner
+      - generate-tags
+      - oss-branch-build
+    runs-on: ${{ needs.start-runner.outputs.label }}
+    steps:
       - name: Login to Docker (on Master)
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_PASSWORD }}
 
-      - name: Push Dev Docker Images
+      - name: Prepare Docker buildx
         run: |
-          GIT_REVISION=$(git rev-parse HEAD)
-          [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1
+          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
           docker buildx create --name oss-buildx --driver docker-container --use
-          VERSION=${{ needs.generate-tags.outputs.dev_tag }}
-          VERSION=$VERSION GIT_REVISION=$GIT_REVISION docker buildx bake --platform=linux/amd64,linux/arm64 -f docker-compose-cloud.build.yaml --push
-          docker buildx rm oss-buildx
         shell: bash
 
-      - name: Push Master Docker Images
-        if: needs.generate-tags.outputs.master_tag != ""
+      - name: Set Git Revision
         run: |
           GIT_REVISION=$(git rev-parse HEAD)
           [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1
-          docker buildx create --name oss-buildx --driver docker-container --use
-          VERSION=${{ needs.generate-tags.outputs.master_tag }}
-          VERSION=$VERSION GIT_REVISION=$GIT_REVISION docker buildx bake --platform=linux/amd64,linux/arm64 -f docker-compose-cloud.build.yaml --push
-          docker buildx rm oss-buildx
+          echo "GIT_REVISION=${GIT_REVISION}" >> $GITHUB_ENV
         shell: bash
 
-      - name: Publish Dev Jars
+      - name: Push Docker Images
+        env:
+          VERSION: ${{ needs.generate-tags.outputs.dev_tag }}
+          ALT_TAG: ${{ needs.generate-tags.outputs.master_tag }}
+        run: GIT_REVISION=$GIT_REVISION docker buildx bake -f docker-compose-cloud.buildx.yaml --push
         shell: bash
-        run: VERSION=${{ needs.generate-tags.outputs.dev_tag }} SUB_BUILD=PLATFORM ./gradlew publish
-      - name: Publish Master Jars
-        if: needs.generate-tags.outputs.master_tag != ""
+
+      - name: Cleanup Docker buildx
+        run: docker buildx rm oss-buildx
         shell: bash
-        run: VERSION=${{ needs.generate-tags.outputs.master_tag }} SUB_BUILD=PLATFORM ./gradlew publish
diff --git a/docker-compose-cloud.buildx.yaml b/docker-compose-cloud.buildx.yaml
@@ -0,0 +1,70 @@
+# Defines the minimum set of images needed to run Cloud.
+# Used to push OSS images that Cloud depends on.
+
+version: "3.7"
+
+services:
+  worker:
+    image: airbyte/worker:${VERSION}
+    build:
+      dockerfile: Dockerfile
+      context: airbyte-workers/build/docker
+      labels:
+        io.airbyte.git-revision: ${GIT_REVISION}
+      args:
+        VERSION: ${VERSION}
+      x-bake:
+        tags:
+          - airbyte/worker:${VERSION}
+          - airbyte/worker:${ALT_TAG:-${VERSION}}
+        platforms:
+          - linux/amd64
+          - linux/arm64
+  webapp:
+    image: airbyte/webapp:${VERSION}
+    build:
+      dockerfile: Dockerfile
+      context: airbyte-webapp/build/docker
+      labels:
+        io.airbyte.git-revision: ${GIT_REVISION}
+      args:
+        VERSION: ${VERSION}
+      x-bake:
+        tags:
+          - airbyte/webapp:${VERSION}
+          - airbyte/webapp:${ALT_TAG:-${VERSION}}
+        platforms:
+          - linux/amd64
+          - linux/arm64
+  metric-reporter:
+    image: airbyte/metrics-reporter:${VERSION}
+    build:
+      dockerfile: Dockerfile
+      context: airbyte-metrics/reporter/build/docker
+      labels:
+        io.airbyte.git-revision: ${GIT_REVISION}
+      args:
+        VERSION: ${VERSION}
+      x-bake:
+        tags:
+          - airbyte/metrics-reporter:${VERSION}
+          - airbyte/metrics-reporter:${ALT_TAG:-${VERSION}}
+        platforms:
+          - linux/amd64
+          - linux/arm64
+  container-orchestrator:
+    image: airbyte/container-orchestrator:${VERSION}
+    build:
+      dockerfile: Dockerfile
+      context: airbyte-container-orchestrator/build/docker
+      labels:
+        io.airbyte.git-revision: ${GIT_REVISION}
+      args:
+        VERSION: ${VERSION}
+      x-bake:
+        tags:
+          - airbyte/container-orchestrator:${VERSION}
+          - airbyte/container-orchestrator:${ALT_TAG:-${VERSION}}
+        platforms:
+          - linux/amd64
+          - linux/arm64