diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml deleted file mode 100644 index 3dcb86dea..000000000 --- a/.github/workflows/publish_sdm_connector.yml +++ /dev/null @@ -1,178 +0,0 @@ -# This flow publishes the Source-Declarative-Manifest (SDM) -# connector to DockerHub as a Docker image. -# TODO: Delete this workflow file once the unified publish flow is implemented and proven stable. - -name: Publish SDM Connector - -on: - workflow_dispatch: - inputs: - version: - description: The version to publish, ie 1.0.0 or 1.0.0-dev1. - If omitted, and if run from a release branch, the version will be - inferred from the git tag. - If omitted, and if run from a non-release branch, then only a SHA-based - Docker tag will be created. - required: false - dry_run: - description: If true, the workflow will not push to DockerHub. - type: boolean - required: false - default: false - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Detect Release Tag Version - if: startsWith(github.ref, 'refs/tags/v') - run: | - DETECTED_VERSION=${{ github.ref_name }} - echo "Version ref set to '${DETECTED_VERSION}'" - # Remove the 'v' prefix if it exists - DETECTED_VERSION="${DETECTED_VERSION#v}" - echo "Setting version to '$DETECTED_VERSION'" - echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV - - - name: Validate and set VERSION from tag ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }}) - id: set_version - if: github.event_name == 'workflow_dispatch' - run: | - INPUT_VERSION=${{ github.event.inputs.version }} - echo "Version input set to '${INPUT_VERSION}'" - # Exit with success if both detected and input versions are empty - if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then - echo "No version detected or input. Will publish to SHA tag instead." - echo 'VERSION=' >> $GITHUB_ENV - exit 0 - fi - # Remove the 'v' prefix if it exists - INPUT_VERSION="${INPUT_VERSION#v}" - # Fail if detected version is non-empty and different from the input version - if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then - echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'." - exit 1 - fi - # Set the version to the input version if non-empty, otherwise the detected version - VERSION="${INPUT_VERSION:-$DETECTED_VERSION}" - # Fail if the version is still empty - if [ -z "$VERSION" ]; then - echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." - exit 1 - fi - echo "Setting version to '$VERSION'" - echo "VERSION=${VERSION}" >> $GITHUB_ENV - echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT - # Check if version is a prerelease version (will not tag 'latest') - if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "IS_PRERELEASE=false" >> $GITHUB_ENV - echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT - else - echo "IS_PRERELEASE=true" >> $GITHUB_ENV - echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT - fi - - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: hynek/build-and-inspect-python-package@v2 - name: Build package with version ref '${{ env.VERSION || '0.0.0dev0' }}' - env: - # Pass in the evaluated version from the previous step - # More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables - POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION || '0.0.0dev0'}} - - - uses: actions/upload-artifact@v4 - with: - name: Packages-${{ github.run_id }} - path: | - /tmp/baipp/dist/*.whl - /tmp/baipp/dist/*.tar.gz - outputs: - VERSION: ${{ steps.set_version.outputs.VERSION }} - IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }} - - publish_sdm: - name: Publish SDM to DockerHub - if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - needs: [build] - environment: - name: DockerHub - url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags - env: - VERSION: ${{ needs.build.outputs.VERSION }} - IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # We need to download the build artifact again because the previous job was on a different runner - - name: Download Build Artifact - uses: actions/download-artifact@v4 - with: - name: Packages-${{ github.run_id }} - path: dist - - - name: Set up QEMU for multi-platform builds - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - - name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )" - if: env.VERSION != '' - run: | - tag="airbyte/source-declarative-manifest:${{ env.VERSION }}" - if [ -z "$tag" ]; then - echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." - exit 1 - fi - echo "Checking if tag '$tag' exists on DockerHub..." - if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then - echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite." - exit 1 - fi - echo "No existing tag '$tag' found. Proceeding with publish." - - - name: Build and push (sha tag) - # Only run if the version is not set - if: env.VERSION == '' && github.event.inputs.dry_run == 'false' - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: | - airbyte/source-declarative-manifest:${{ github.sha }} - - - name: "Build and push (version tag: ${{ env.VERSION || 'none'}})" - # Only run if the version is set - if: env.VERSION != '' && github.event.inputs.dry_run == 'false' - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: | - airbyte/source-declarative-manifest:${{ env.VERSION }} - - - name: Build and push ('latest' tag) - # Only run if version is set and IS_PRERELEASE is false - if: env.VERSION != '' && env.IS_PRERELEASE == 'false' && github.event.inputs.dry_run == 'false' - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: | - airbyte/source-declarative-manifest:latest diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 2bfd7cb2f..8baf6765e 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -14,7 +14,7 @@ on: workflow_dispatch: inputs: version: - description: "Version. The version to publish, ie 1.0.0 or 1.0.0-dev1. In most cases, you can leave this blank. If run from a release tag (recommended), the version number will be inferred from the git tag." + description: "Note that this workflow is intended for prereleases. For public-facing stable releases, please use the GitHub Releases workflow instead: https://github.com/airbytehq/airbyte-python-cdk/blob/main/docs/RELEASES.md. If running this workflow from main or from a dev branch, please enter the desired version number here, for instance 1.2.3dev0 or 1.2.3rc1." required: false publish_to_pypi: description: "Publish to PyPI. If true, the workflow will publish to PyPI." @@ -30,7 +30,7 @@ on: description: "Update Connector Builder. If true, the workflow will create a PR to bump the CDK version used by Connector Builder." type: boolean required: true - default: true + default: false jobs: build: diff --git a/docs/RELEASES.md b/docs/RELEASES.md index c51ebd6bf..5a92e090b 100644 --- a/docs/RELEASES.md +++ b/docs/RELEASES.md @@ -1,6 +1,6 @@ # Airbyte Python CDK - Release Management Guide -## Publishing stable releases of the CDK +## Publishing stable releases of the CDK and SDM A few seconds after any PR is merged to `main` , a release draft will be created or updated on the releases page here: https://github.com/airbytehq/airbyte-python-cdk/releases. Here are the steps to publish a CDK release: @@ -13,34 +13,16 @@ A few seconds after any PR is merged to `main` , a release draft will be created - *Only maintainers can see release drafts. Non-maintainers will only see published releases.* - If you create a tag on accident that you need to remove, contact a maintainer to delete the tag and the release. -- You can monitor the PyPi release process here in the GitHub Actions view: https://github.com/airbytehq/airbyte-python-cdk/actions/workflows/pypi_publish.yml +- You can monitor the PyPI release process here in the GitHub Actions view: https://github.com/airbytehq/airbyte-python-cdk/actions/workflows/pypi_publish.yml - **_[▶️ Loom Walkthrough](https://www.loom.com/share/ceddbbfc625141e382fd41c4f609dc51?sid=78e13ef7-16c8-478a-af47-4978b3ff3fad)_** -## Publishing Pre-Release Versions of the CDK +## Publishing Pre-Release Versions of the CDK and/or SDM (Internal) -Publishing a pre-release version is similar to publishing a stable version. However, instead of using the auto-generated release draft, you’ll create a new release draft. +This process is slightly different from the above, since we don't necessarily want public release notes to be published for internal testing releases. The same underlying workflow will be run, but we'll kick it off directly: -1. Navigate to the releases page: https://github.com/airbytehq/airbyte-python-cdk/releases -2. Click “Draft a new release”. -3. In the tag selector, type the version number of the prerelease you’d like to create and copy-past the same into the Release name box. - - The release should be like `vX.Y.Zsuffix` where `suffix` is something like `dev0`, `dev1` , `alpha0`, `beta1`, etc. - -## Publishing new versions of SDM (source-declarative-manifest) - -Prereqs: - -1. The SDM publish process assumes you have already published the CDK. If you have not already done so, you’ll want to first publish the CDK using the steps above. While this prereq is not technically *required*, it is highly recommended. - -Publish steps: - -1. Navigate to the GitHub action page here: https://github.com/airbytehq/airbyte-python-cdk/actions/workflows/publish_sdm_connector.yml -2. Click “Run workflow” to start the process of invoking a new manual workflow. -3. Click the drop-down for “Run workflow from” and then select the “tags” tab to browse already-created tags. Select the tag of the published CDK version you want to use for the SDM publish process. Notes: - 1. Optionally you can type part of the version number to filter down the list. - 2. You can ignore the version prompt box (aka leave blank) when publishing from a release tag. The version will be detected from the git tag. - 3. You can optionally click the box for “Dry run” if you want to observe the process before running the real thing. The dry run option will perform all steps *except* for the DockerHub publish step. -4. Without changing any other options, you can click “Run workflow” to run the workflow. -5. Watch the GitHub Action run. If successful, you should see it publish to DockerHub and a URL will appear on the “Summary” view once it has completed. - -- **_[▶️ Loom Walkthrough](https://www.loom.com/share/bc8ddffba9384fcfacaf535608360ee1)_** +1. Navigate to the "Packaging and Publishing" workflow in GitHub Actions. +2. Type the version number - including a valid pre-release suffix. Examples: `1.2.3dev0`, `1.2.3rc1`, `1.2.3b0`, etc. +3. Select `main` or your dev branch from the "Use workflow from" dropdown. +4. Select your options and click "Run workflow". +5. Monitor the workflow to ensure the process has succeeded.