publish oss for cloud (#13978)

workflow to publish oss artifacts that cloud needs to build against
use docker buildx to create arm images for local development

.github/workflows/publish-oss-for-cloud.yml

@@ -0,0 +1,126 @@
+name: Publish OSS Artifacts for Cloud
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      oss_ref:
+        description: "Publish artifacts for the following git ref (if unspecified, uses the latest commit for the current branch):"
+        required: false
+jobs:
+  find_valid_pat:
+    name: "Find a PAT with room for actions"
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    outputs:
+      pat: ${{ steps.variables.outputs.pat }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: Check PAT rate limits
+        id: variables
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.AIRBYTEIO_PAT }} \
+            ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
+            ${{ secrets.SUPERTOPHER_PAT }} \
+            ${{ secrets.DAVINCHIA_PAT }}
+  start-runner:
+    name: "Start Runner on AWS"
+    needs: find_valid_pat
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: Start AWS Runner
+        id: start-ec2-runner
+        uses: ./.github/actions/start-aws-runner
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
+
+  generate-tags:
+    name: "Generate Tags"
+    runs-on: ubuntu-latest
+    outputs:
+      dev_tag: ${{ steps.set-outputs.outputs.dev_tag }}
+      master_tag: ${{ steps.set-outputs.outputs.master_tag }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.inputs.oss_ref || github.ref }}
+      - name: Generate Outputs
+        id: set-outputs
+        shell: bash
+        run: |-
+          set -x
+          
+          commit_sha=$(git rev-parse --short HEAD)
+          
+          # set dev_tag
+          # AirbyteVersion.java allows versions that have a prefix of 'dev'
+          echo "::set-output name=dev_tag::dev-${commit_sha}"
+          
+          # If this commit is on the master branch, also set master_tag
+          if test 0 -eq $(git merge-base --is-ancestor "${commit_sha}" master); then
+            echo "::set-output name=master_tag::${commit_sha}"
+          fi
+
+  oss-branch-build:
+    name: "Build and Push Images from Branch"
+    needs:
+      - start-runner
+      - generate-tags
+    runs-on: ${{ needs.start-runner.outputs.label }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.inputs.oss_ref || github.ref }}
+
+      - name: Build Branch
+        uses: ./.github/actions/build-branch
+        with:
+          branch_version_tag: ${{ needs.generate-tags.outputs.dev_tag }}
+
+      - name: Login to Docker (on Master)
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: Push Dev Docker Images
+        run: |
+          GIT_REVISION=$(git rev-parse HEAD)
+          [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1
+          docker buildx create --name oss-buildx --driver docker-container --use
+          VERSION=${{ needs.generate-tags.outputs.dev_tag }}
+          VERSION=$VERSION GIT_REVISION=$GIT_REVISION docker buildx bake --platform=linux/amd64,linux/arm64 -f docker-compose-cloud.build.yaml --push
+          docker buildx rm oss-buildx
+        shell: bash
+
+      - name: Push Master Docker Images
+        if: needs.generate-tags.outputs.master_tag != ""
+        run: |
+          GIT_REVISION=$(git rev-parse HEAD)
+          [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1
+          docker buildx create --name oss-buildx --driver docker-container --use
+          VERSION=${{ needs.generate-tags.outputs.master_tag }}
+          VERSION=$VERSION GIT_REVISION=$GIT_REVISION docker buildx bake --platform=linux/amd64,linux/arm64 -f docker-compose-cloud.build.yaml --push
+          docker buildx rm oss-buildx
+        shell: bash
+
+      - name: Publish Dev Jars
+        shell: bash
+        run: VERSION=${{ needs.generate-tags.outputs.dev_tag }} SUB_BUILD=PLATFORM ./gradlew publish
+      - name: Publish Master Jars
+        if: needs.generate-tags.outputs.master_tag != ""
+        shell: bash
+        run: VERSION=${{ needs.generate-tags.outputs.master_tag }} SUB_BUILD=PLATFORM ./gradlew publish