PYTHON-2434 Automatically combine release wheels + sdist into one archive (#803)

Author: ShaneHarvey

.evergreen/config.yml

@@ -870,6 +870,16 @@ functions:
           # Remove all Docker images
           docker rmi -f $(docker images -a -q) &> /dev/null || true
 
+  "build release":
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: "src"
+        script: |
+          set -o xtrace
+          ${PREPARE_SHELL}
+          .evergreen/release.sh
+
   "upload release":
     - command: archive.targz_pack
       params:
@@ -882,12 +892,63 @@ functions:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
         local_file: release-files.tgz
-        remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/release/${task_id}-${execution}-release-files.tar.gz
+        remote_file: ${UPLOAD_BUCKET}/release/${revision}/${task_id}-${execution}-release-files.tar.gz
         bucket: mciuploads
         permissions: public-read
         content_type: ${content_type|application/gzip}
         display_name: Release files
 
+  "download and merge releases":
+    - command: shell.exec
+      params:
+        silent: true
+        script: |
+          export AWS_ACCESS_KEY_ID=${aws_key}
+          export AWS_SECRET_ACCESS_KEY=${aws_secret}
+
+          # Download all the task coverage files.
+          aws s3 cp --recursive s3://mciuploads/${UPLOAD_BUCKET}/release/${revision}/ release/
+    - command: shell.exec
+      params:
+        shell: "bash"
+        script: |
+          set -o xtrace
+          ${PREPARE_SHELL}
+          # Combine releases into one directory.
+          ls -la release/
+          mkdir releases
+          # Copy old manylinux release first since we want the newer manylinux
+          # wheels to override them.
+          mkdir old_manylinux
+          if mv release/*old_manylinux* old_manylinux; then
+            for REL in old_manylinux/*; do
+              tar zxvf $REL -C releases/
+            done
+          fi
+          for REL in release/*; do
+            tar zxvf $REL -C releases/
+          done
+          # Build source distribution.
+          cd src/
+          /opt/python/3.6/bin/python3 setup.py sdist
+          cp dist/* ../releases
+    - command: archive.targz_pack
+      params:
+        target: "release-files-all.tgz"
+        source_dir: "releases/"
+        include:
+          - "*"
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: release-files-all.tgz
+        remote_file: ${UPLOAD_BUCKET}/release-all/${revision}/${task_id}-${execution}-release-files-all.tar.gz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: ${content_type|application/gzip}
+        display_name: Release files all
+
 pre:
   - func: "fetch source"
   - func: "prepare resources"
@@ -972,22 +1033,31 @@ tasks:
                genhtml --version || true
                valgrind --version || true
 
-    - name: "release"
-      tags: ["release"]
+    - name: "release-mac"
+      tags: ["release_tag"]
+      run_on: macos-1014
+      commands:
+        - func: "build release"
+        - func: "upload release"
+
+    - name: "release-windows"
+      tags: ["release_tag"]
+      run_on: windows-64-vsMulti-small
+      commands:
+        - func: "build release"
+        - func: "upload release"
+
+    - name: "release-manylinux"
+      tags: ["release_tag"]
+      run_on: ubuntu2004-large
       exec_timeout_secs: 216000  # 60 minutes (manylinux task is slow).
       commands:
-        - command: shell.exec
-          type: test
-          params:
-            working_dir: "src"
-            script: |
-              set -o xtrace
-              ${PREPARE_SHELL}
-              .evergreen/release.sh
+        - func: "build release"
         - func: "upload release"
 
     - name: "release-old-manylinux"
-      tags: ["release"]
+      tags: ["release_tag"]
+      run_on: ubuntu2004-large
       exec_timeout_secs: 216000  # 60 minutes (manylinux task is slow).
       commands:
         - command: shell.exec
@@ -1000,6 +1070,16 @@ tasks:
               .evergreen/build-manylinux.sh BUILD_WITH_TAG
         - func: "upload release"
 
+    - name: "release-combine"
+      tags: ["release_tag"]
+      run_on: ubuntu2004-small
+      depends_on:
+        - name: "*"
+          variant: ".release_tag"
+          patch_optional: true
+      commands:
+        - func: "download and merge releases"
+
 # Standard test tasks {{{
 
     - name: "mockupdb"
@@ -2530,19 +2610,12 @@ buildvariants:
   tasks:
     - name: "load-balancer-test"
 
-- matrix_name: "Release"
-  matrix_spec:
-    platform: [ubuntu-20.04, windows-64-vsMulti-small, macos-1014]
-  display_name: "Release ${platform}"
+- name: Release
+  display_name: Release
   batchtime: 20160 # 14 days
+  tags: ["release_tag"]
   tasks:
-    - name: "release"
-  rules:
-    - if:
-        platform: ubuntu-20.04
-      then:
-        add_tasks:
-          - name: "release-old-manylinux"
+  - ".release_tag"
 
       # Platform notes
       # i386 builds of OpenSSL or Cyrus SASL are not available

RELEASE.rst

@@ -55,48 +55,37 @@ Doing a Release
 8. Push commit / tag, eg ``git push && git push --tags``.
 
 9. Pushing a tag will trigger a release process in Evergreen which builds
-   wheels and eggs for manylinux, macOS, and Windows. Wait for these jobs to
-   complete and then download the "Release files" archive from each task. See:
+   wheels for manylinux, macOS, and Windows. Wait for the "release-combine"
+   task to complete and then download the "Release files all" archive. See:
    https://evergreen.mongodb.com/waterfall/mongo-python-driver?bv_filter=release
 
-   Unpack each downloaded archive so that we can upload the included files. For
-   the next steps let's assume we unpacked these files into the following paths::
+   The contents should look like this::
 
-     $ ls path/to/manylinux
+     $ ls path/to/archive
+     pymongo-<version>-cp310-cp310-macosx_10_9_universal2.whl
+     ...
      pymongo-<version>-cp38-cp38-manylinux2014_x86_64.whl
      ...
-     $ ls path/to/windows/
      pymongo-<version>-cp38-cp38-win_amd64.whl
      ...
-
-10. Build the source distribution::
-
-     $ git clone git@github.com:mongodb/mongo-python-driver.git
-     $ cd mongo-python-driver
-     $ git checkout "<release version number>"
-     $ python3 setup.py sdist
-
-    This will create the following distribution::
-
-     $ ls dist
      pymongo-<version>.tar.gz
 
-11. Upload all the release packages to PyPI with twine::
+10. Upload all the release packages to PyPI with twine::
 
-     $ python3 -m twine upload dist/*.tar.gz path/to/manylinux/* path/to/mac/* path/to/windows/*
+     $ python3 -m twine upload path/to/archive/*
 
-12. Make sure the new version appears on https://pymongo.readthedocs.io/. If the
+11. Make sure the new version appears on https://pymongo.readthedocs.io/. If the
     new version does not show up automatically, trigger a rebuild of "latest":
     https://readthedocs.org/projects/pymongo/builds/
 
-13. Bump the version number to <next version>.dev0 in setup.py/__init__.py,
+12. Bump the version number to <next version>.dev0 in setup.py/__init__.py,
     commit, push.
 
-14. Publish the release version in Jira.
+13. Publish the release version in Jira.
 
-15. Announce the release on:
+14. Announce the release on:
     https://developer.mongodb.com/community/forums/c/community/release-notes/
 
-16. File a ticket for DOCSP highlighting changes in server version and Python
+15. File a ticket for DOCSP highlighting changes in server version and Python
     version compatibility or the lack thereof, for example:
     https://jira.mongodb.org/browse/DOCSP-13536