fix: build wheel Github action inputs (#3858)

Eventual-Inc · Feb 25, 2025 · fc97a92 · fc97a92
1 parent 93ca38b
commit fc97a92
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 41 deletions.
diff --git a/.github/ci-scripts/generate_simple_py_index.py b/.github/ci-scripts/generate_simple_py_index.py
@@ -31,24 +31,24 @@ def write_file_ensure_dir(filename, s):
         f.write(s)
 
 
-def generate_root_index(pkg_names):
-    links = [f'<a href="{urllib.parse.quote(name)}/">{name}</a>' for name in pkg_names]
+def generate_root_index(prefix, pkg_names):
+    links = "\n".join([f'<a href="/{prefix}/{urllib.parse.quote(name)}/">{name}</a>' for name in pkg_names])
 
     return f"""<!DOCTYPE html>
 <html>
   <body>
-    {"\n".join(links)}
+    {links}
   </body>
 </html>"""
 
 
-def generate_pkg_index(wheel_names):
-    links = [f'<a href="../{urllib.parse.quote(name)}">{name}</a>' for name in wheel_names]
+def generate_pkg_index(prefix, wheel_names):
+    links = "\n".join([f'<a href="/{prefix}/{urllib.parse.quote(name)}">{name}</a>' for name in wheel_names])
 
     return f"""<!DOCTYPE html>
 <html>
   <body>
-    {"\n".join(links)}
+    {links}
   </body>
 </html>"""
 
@@ -61,7 +61,7 @@ def main():
     s3_prefix = "s3://"
     s3_url = sys.argv[1]
     assert s3_url.startswith(s3_prefix)
-    (bucket, _, prefix) = s3_url.removeprefix(s3_prefix).partition("/")
+    (bucket, _, prefix) = s3_url.removeprefix(s3_prefix).removesuffix("/").partition("/")
 
     s3 = boto3.client("s3")
     paginator = s3.get_paginator("list_objects_v2")
@@ -71,18 +71,18 @@ def main():
     for page in pages:
         for obj in page["Contents"]:
             if obj["Key"].endswith(".whl"):
-                wheel_name = obj["Key"]
+                wheel_name = obj["Key"].removeprefix(prefix + "/")
 
                 pkg_name, _, _, _ = parse_wheel_filename(wheel_name)
-                if pkg_name not in pkg_name:
+                if pkg_name not in pkg_map:
                     pkg_map[pkg_name] = []
                 pkg_map[pkg_name].append(wheel_name)
 
-    root_index = generate_root_index(pkg_map.keys())
+    root_index = generate_root_index(prefix, pkg_map.keys())
     write_file_ensure_dir("dist/indices/index.html", root_index)
 
     for pkg_name, wheel_names in pkg_map.items():
-        pkg_index = generate_pkg_index(wheel_names)
+        pkg_index = generate_pkg_index(prefix, wheel_names)
         write_file_ensure_dir(f"dist/indices/{pkg_name}/index.html", pkg_index)
 
 

diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml
@@ -34,15 +34,19 @@ jobs:
     steps:
     - uses: actions/checkout@v4
       with:
-        fetch-depth: 1
+        fetch-depth: 0
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ env.PYTHON_VERSION }}
     - run: pip install uv
     - run: uv pip install twine yq setuptools_scm
 
     - name: Patch package version
-      run: tomlq -i -t ".package.version = \"$(python -m setuptools_scm)\"" Cargo.toml
+      run: |
+        VERSION="$(python -m setuptools_scm | sed 's/\.dev/-dev/'g)" # replace ".dev" with "-dev" to comply with Cargo SemVer rules
+        echo "Setting package version to: $VERSION"
+        tomlq -i -t ".package.version = \"$VERSION\"" Cargo.toml
+        tomlq -i -t ".workspace.package.version = \"$VERSION\"" Cargo.toml
 
     - name: Patch name to daft-lts if LTS
       if: ${{ inputs.lts }}
@@ -51,7 +55,7 @@ jobs:
     - name: Configure RUSTFLAGS for x86
       if: ${{ (inputs.arch == 'x86_64') }}
       run: |
-        if [[ ${{ inputs.lts }} ]]; then
+        if [[ "${{ inputs.lts }}" == "true" ]]; then
           echo "RUSTFLAGS=-C target-feature=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b" >> $GITHUB_ENV && \
           echo "CFLAGS=-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16" >> $GITHUB_ENV
         else

diff --git a/.github/workflows/nightly-publish-s3.yml b/.github/workflows/nightly-publish-s3.yml
@@ -13,36 +13,44 @@ on:
   workflow_dispatch:
 
 env:
-  OUTPUT_BUCKET: s3://github-actions-artifacts-bucket/builds/nightly/
+  S3_BUCKET: s3://github-actions-artifacts-bucket/builds/nightly/
   AWS_REGION: us-west-2
   CLOUDFRONT_DIST_ID: E3H8WN738AJ1D4
 
 jobs:
   build:
+    name: 'Build Daft wheel for ${{ matrix.os }}-${{ matrix.arch }}-lts=${{ matrix.lts }}'
     uses: ./.github/workflows/build-wheel.yml
     with:
       os: ${{ matrix.os }}
-      arch: ${{ matrix.compile_arch }}
+      arch: ${{ matrix.arch }}
       lts: ${{ matrix.lts }}
       build_type: release
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu, macos, windows]
-        compile_arch: [x86_64, aarch64]
-        lts: [0, 1]
+        arch: [x86_64, aarch64]
+        lts: [false, true]
 
         exclude:
         - os: windows
-          compile_arch: aarch64
-        - lts: 1
-          compile_arch: aarch64
+          arch: aarch64
+        - lts: true
+          arch: aarch64
 
   publish:
     name: Publish wheels to S3
     runs-on: ubuntu-latest
     needs: build
+    permissions:
+      id-token: write
+      contents: read
 
     steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
     - uses: actions/download-artifact@v4
       with:
         pattern: wheels-*
@@ -55,26 +63,22 @@ jobs:
         role-to-assume: ${{ secrets.ACTIONS_AWS_ROLE_ARN }}
         role-session-name: DaftPythonPackageGitHubWorkflow
     - name: Upload wheels to S3
-      uses: reggionick/s3-deploy@v4
-      with:
-        folder: dist
-        bucket: ${{ env.OUTPUT_BUCKET }}
-        bucket-region: ${{ env.AWS_REGION }}
-        dist-id: ${{ env.CLOUDFRONT_DIST_ID }}
+      run: aws s3 cp --no-progress --recursive dist/ ${{ env.S3_BUCKET }}
 
     - name: Install boto3 and packaging
       run: pip3 install boto3 packaging
     - name: Generate Python simple repository API files
-      run: python3 .github/ci-scripts/generate_simple_py_index.py
+      run: python3 .github/ci-scripts/generate_simple_py_index.py ${{ env.S3_BUCKET }}
     - name: Upload index files to S3
-      uses: reggionick/s3-deploy@v4
-      with:
-        folder: dist/indices
-        bucket: ${{ env.OUTPUT_BUCKET }}
-        bucket-region: ${{ env.AWS_REGION }}
-        dist-id: ${{ env.CLOUDFRONT_DIST_ID }}
+      run: aws s3 cp --no-progress --recursive dist/indices/ ${{ env.S3_BUCKET }}
+
+    - name: Invalidate Cloudfront cache
+      run: aws cloudfront create-invalidation --distribution-id ${{ env.CLOUDFRONT_DIST_ID }} --paths '/builds/nightly*'
+
     - name: Print install instructions
-      run: 'echo "To install the nightly build, run: \npip install getdaft --pre --extra-index-url https://d1p3klp2t5517h.cloudfront.net/builds/nightly"'
+      run: |
+        echo "To install the nightly build, run:"
+        echo "pip install getdaft --pre --extra-index-url https://d1p3klp2t5517h.cloudfront.net/builds/nightly"
 
   on-failure:
     name: Send Slack notification on failure

diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml
@@ -15,23 +15,25 @@ defaults:
 
 jobs:
   build:
+    name: 'Build Daft wheel for ${{ matrix.os }}-${{ matrix.arch }}-lts=${{ matrix.lts }}'
     uses: ./.github/workflows/build-wheel.yml
     with:
       os: ${{ matrix.os }}
-      arch: ${{ matrix.compile_arch }}
+      arch: ${{ matrix.arch }}
       lts: ${{ matrix.lts }}
       build_type: release
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu, macos, windows]
-        compile_arch: [x86_64, aarch64]
-        lts: [0, 1]
+        arch: [x86_64, aarch64]
+        lts: [false, true]
 
         exclude:
         - os: windows
-          compile_arch: aarch64
-        - lts: 1
-          compile_arch: aarch64
+          arch: aarch64
+        - lts: true
+          arch: aarch64
 
   publish:
     name: Publish wheels to PyPI