From 5e136005eafa6cb9536587017431f4d3270bc6ef Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 6 Dec 2024 11:25:00 -0800 Subject: [PATCH 1/4] ci: delete old workflow --- .github/workflows/publish_sdm_connector.yml | 178 -------------------- 1 file changed, 178 deletions(-) delete mode 100644 .github/workflows/publish_sdm_connector.yml diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml deleted file mode 100644 index 3dcb86dea..000000000 --- a/.github/workflows/publish_sdm_connector.yml +++ /dev/null @@ -1,178 +0,0 @@ -# This flow publishes the Source-Declarative-Manifest (SDM) -# connector to DockerHub as a Docker image. -# TODO: Delete this workflow file once the unified publish flow is implemented and proven stable. - -name: Publish SDM Connector - -on: - workflow_dispatch: - inputs: - version: - description: The version to publish, ie 1.0.0 or 1.0.0-dev1. - If omitted, and if run from a release branch, the version will be - inferred from the git tag. - If omitted, and if run from a non-release branch, then only a SHA-based - Docker tag will be created. - required: false - dry_run: - description: If true, the workflow will not push to DockerHub. - type: boolean - required: false - default: false - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Detect Release Tag Version - if: startsWith(github.ref, 'refs/tags/v') - run: | - DETECTED_VERSION=${{ github.ref_name }} - echo "Version ref set to '${DETECTED_VERSION}'" - # Remove the 'v' prefix if it exists - DETECTED_VERSION="${DETECTED_VERSION#v}" - echo "Setting version to '$DETECTED_VERSION'" - echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV - - - name: Validate and set VERSION from tag ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }}) - id: set_version - if: github.event_name == 'workflow_dispatch' - run: | - INPUT_VERSION=${{ github.event.inputs.version }} - echo "Version input set to '${INPUT_VERSION}'" - # Exit with success if both detected and input versions are empty - if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then - echo "No version detected or input. Will publish to SHA tag instead." - echo 'VERSION=' >> $GITHUB_ENV - exit 0 - fi - # Remove the 'v' prefix if it exists - INPUT_VERSION="${INPUT_VERSION#v}" - # Fail if detected version is non-empty and different from the input version - if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then - echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'." - exit 1 - fi - # Set the version to the input version if non-empty, otherwise the detected version - VERSION="${INPUT_VERSION:-$DETECTED_VERSION}" - # Fail if the version is still empty - if [ -z "$VERSION" ]; then - echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." - exit 1 - fi - echo "Setting version to '$VERSION'" - echo "VERSION=${VERSION}" >> $GITHUB_ENV - echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT - # Check if version is a prerelease version (will not tag 'latest') - if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "IS_PRERELEASE=false" >> $GITHUB_ENV - echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT - else - echo "IS_PRERELEASE=true" >> $GITHUB_ENV - echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT - fi - - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: hynek/build-and-inspect-python-package@v2 - name: Build package with version ref '${{ env.VERSION || '0.0.0dev0' }}' - env: - # Pass in the evaluated version from the previous step - # More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables - POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION || '0.0.0dev0'}} - - - uses: actions/upload-artifact@v4 - with: - name: Packages-${{ github.run_id }} - path: | - /tmp/baipp/dist/*.whl - /tmp/baipp/dist/*.tar.gz - outputs: - VERSION: ${{ steps.set_version.outputs.VERSION }} - IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }} - - publish_sdm: - name: Publish SDM to DockerHub - if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - needs: [build] - environment: - name: DockerHub - url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags - env: - VERSION: ${{ needs.build.outputs.VERSION }} - IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # We need to download the build artifact again because the previous job was on a different runner - - name: Download Build Artifact - uses: actions/download-artifact@v4 - with: - name: Packages-${{ github.run_id }} - path: dist - - - name: Set up QEMU for multi-platform builds - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - - name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )" - if: env.VERSION != '' - run: | - tag="airbyte/source-declarative-manifest:${{ env.VERSION }}" - if [ -z "$tag" ]; then - echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." - exit 1 - fi - echo "Checking if tag '$tag' exists on DockerHub..." - if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then - echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite." - exit 1 - fi - echo "No existing tag '$tag' found. Proceeding with publish." - - - name: Build and push (sha tag) - # Only run if the version is not set - if: env.VERSION == '' && github.event.inputs.dry_run == 'false' - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: | - airbyte/source-declarative-manifest:${{ github.sha }} - - - name: "Build and push (version tag: ${{ env.VERSION || 'none'}})" - # Only run if the version is set - if: env.VERSION != '' && github.event.inputs.dry_run == 'false' - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: | - airbyte/source-declarative-manifest:${{ env.VERSION }} - - - name: Build and push ('latest' tag) - # Only run if version is set and IS_PRERELEASE is false - if: env.VERSION != '' && env.IS_PRERELEASE == 'false' && github.event.inputs.dry_run == 'false' - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: | - airbyte/source-declarative-manifest:latest From 7186f4af28f2aac942d01bd28410f8bc3195afb2 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 6 Dec 2024 11:25:24 -0800 Subject: [PATCH 2/4] ci: update release workflow defaults and help description --- .github/workflows/pypi_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 2bfd7cb2f..eb095c6c5 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -14,7 +14,7 @@ on: workflow_dispatch: inputs: version: - description: "Version. The version to publish, ie 1.0.0 or 1.0.0-dev1. In most cases, you can leave this blank. If run from a release tag (recommended), the version number will be inferred from the git tag." + description: "Version. For public-facing stable releases, please use the GitHub Releases workflow instead: https://github.com/airbytehq/airbyte-python-cdk/blob/main/docs/RELEASES.md. If running this workflow again main or from a dev branch, the desired version number is required to be entered here, e.g. `v1.2.3dev0` or `v1.2.3rc1`." required: false publish_to_pypi: description: "Publish to PyPI. If true, the workflow will publish to PyPI." @@ -30,7 +30,7 @@ on: description: "Update Connector Builder. If true, the workflow will create a PR to bump the CDK version used by Connector Builder." type: boolean required: true - default: true + default: false jobs: build: From adc62fa41beaccf93a57fbcdc1e60a45caadbc59 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 6 Dec 2024 11:32:44 -0800 Subject: [PATCH 3/4] docs: update release instructions --- docs/RELEASES.md | 36 +++++++++--------------------------- 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/docs/RELEASES.md b/docs/RELEASES.md index c51ebd6bf..5a92e090b 100644 --- a/docs/RELEASES.md +++ b/docs/RELEASES.md @@ -1,6 +1,6 @@ # Airbyte Python CDK - Release Management Guide -## Publishing stable releases of the CDK +## Publishing stable releases of the CDK and SDM A few seconds after any PR is merged to `main` , a release draft will be created or updated on the releases page here: https://github.com/airbytehq/airbyte-python-cdk/releases. Here are the steps to publish a CDK release: @@ -13,34 +13,16 @@ A few seconds after any PR is merged to `main` , a release draft will be created - *Only maintainers can see release drafts. Non-maintainers will only see published releases.* - If you create a tag on accident that you need to remove, contact a maintainer to delete the tag and the release. -- You can monitor the PyPi release process here in the GitHub Actions view: https://github.com/airbytehq/airbyte-python-cdk/actions/workflows/pypi_publish.yml +- You can monitor the PyPI release process here in the GitHub Actions view: https://github.com/airbytehq/airbyte-python-cdk/actions/workflows/pypi_publish.yml - **_[▶️ Loom Walkthrough](https://www.loom.com/share/ceddbbfc625141e382fd41c4f609dc51?sid=78e13ef7-16c8-478a-af47-4978b3ff3fad)_** -## Publishing Pre-Release Versions of the CDK +## Publishing Pre-Release Versions of the CDK and/or SDM (Internal) -Publishing a pre-release version is similar to publishing a stable version. However, instead of using the auto-generated release draft, you’ll create a new release draft. +This process is slightly different from the above, since we don't necessarily want public release notes to be published for internal testing releases. The same underlying workflow will be run, but we'll kick it off directly: -1. Navigate to the releases page: https://github.com/airbytehq/airbyte-python-cdk/releases -2. Click “Draft a new release”. -3. In the tag selector, type the version number of the prerelease you’d like to create and copy-past the same into the Release name box. - - The release should be like `vX.Y.Zsuffix` where `suffix` is something like `dev0`, `dev1` , `alpha0`, `beta1`, etc. - -## Publishing new versions of SDM (source-declarative-manifest) - -Prereqs: - -1. The SDM publish process assumes you have already published the CDK. If you have not already done so, you’ll want to first publish the CDK using the steps above. While this prereq is not technically *required*, it is highly recommended. - -Publish steps: - -1. Navigate to the GitHub action page here: https://github.com/airbytehq/airbyte-python-cdk/actions/workflows/publish_sdm_connector.yml -2. Click “Run workflow” to start the process of invoking a new manual workflow. -3. Click the drop-down for “Run workflow from” and then select the “tags” tab to browse already-created tags. Select the tag of the published CDK version you want to use for the SDM publish process. Notes: - 1. Optionally you can type part of the version number to filter down the list. - 2. You can ignore the version prompt box (aka leave blank) when publishing from a release tag. The version will be detected from the git tag. - 3. You can optionally click the box for “Dry run” if you want to observe the process before running the real thing. The dry run option will perform all steps *except* for the DockerHub publish step. -4. Without changing any other options, you can click “Run workflow” to run the workflow. -5. Watch the GitHub Action run. If successful, you should see it publish to DockerHub and a URL will appear on the “Summary” view once it has completed. - -- **_[▶️ Loom Walkthrough](https://www.loom.com/share/bc8ddffba9384fcfacaf535608360ee1)_** +1. Navigate to the "Packaging and Publishing" workflow in GitHub Actions. +2. Type the version number - including a valid pre-release suffix. Examples: `1.2.3dev0`, `1.2.3rc1`, `1.2.3b0`, etc. +3. Select `main` or your dev branch from the "Use workflow from" dropdown. +4. Select your options and click "Run workflow". +5. Monitor the workflow to ensure the process has succeeded. From 36e06a18aa0daa6989580b45656c7f8b551d0d89 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 6 Dec 2024 11:35:45 -0800 Subject: [PATCH 4/4] update instructions text --- .github/workflows/pypi_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index eb095c6c5..8baf6765e 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -14,7 +14,7 @@ on: workflow_dispatch: inputs: version: - description: "Version. For public-facing stable releases, please use the GitHub Releases workflow instead: https://github.com/airbytehq/airbyte-python-cdk/blob/main/docs/RELEASES.md. If running this workflow again main or from a dev branch, the desired version number is required to be entered here, e.g. `v1.2.3dev0` or `v1.2.3rc1`." + description: "Note that this workflow is intended for prereleases. For public-facing stable releases, please use the GitHub Releases workflow instead: https://github.com/airbytehq/airbyte-python-cdk/blob/main/docs/RELEASES.md. If running this workflow from main or from a dev branch, please enter the desired version number here, for instance 1.2.3dev0 or 1.2.3rc1." required: false publish_to_pypi: description: "Publish to PyPI. If true, the workflow will publish to PyPI."