diff --git a/.github/ci-scripts/generate_simple_py_index.py b/.github/ci-scripts/generate_simple_py_index.py index 61b4aa2887..4acd0fdd41 100644 --- a/.github/ci-scripts/generate_simple_py_index.py +++ b/.github/ci-scripts/generate_simple_py_index.py @@ -31,24 +31,24 @@ def write_file_ensure_dir(filename, s): f.write(s) -def generate_root_index(pkg_names): - links = [f'{name}' for name in pkg_names] +def generate_root_index(prefix, pkg_names): + links = "\n".join([f'{name}' for name in pkg_names]) return f""" - {"\n".join(links)} + {links} """ -def generate_pkg_index(wheel_names): - links = [f'{name}' for name in wheel_names] +def generate_pkg_index(prefix, wheel_names): + links = "\n".join([f'{name}' for name in wheel_names]) return f""" - {"\n".join(links)} + {links} """ @@ -61,7 +61,7 @@ def main(): s3_prefix = "s3://" s3_url = sys.argv[1] assert s3_url.startswith(s3_prefix) - (bucket, _, prefix) = s3_url.removeprefix(s3_prefix).partition("/") + (bucket, _, prefix) = s3_url.removeprefix(s3_prefix).removesuffix("/").partition("/") s3 = boto3.client("s3") paginator = s3.get_paginator("list_objects_v2") @@ -71,18 +71,18 @@ def main(): for page in pages: for obj in page["Contents"]: if obj["Key"].endswith(".whl"): - wheel_name = obj["Key"] + wheel_name = obj["Key"].removeprefix(prefix + "/") pkg_name, _, _, _ = parse_wheel_filename(wheel_name) - if pkg_name not in pkg_name: + if pkg_name not in pkg_map: pkg_map[pkg_name] = [] pkg_map[pkg_name].append(wheel_name) - root_index = generate_root_index(pkg_map.keys()) + root_index = generate_root_index(prefix, pkg_map.keys()) write_file_ensure_dir("dist/indices/index.html", root_index) for pkg_name, wheel_names in pkg_map.items(): - pkg_index = generate_pkg_index(wheel_names) + pkg_index = generate_pkg_index(prefix, wheel_names) write_file_ensure_dir(f"dist/indices/{pkg_name}/index.html", pkg_index) diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 072230647a..b8fa2e84f1 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -34,7 +34,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 1 + fetch-depth: 0 - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -42,7 +42,11 @@ jobs: - run: uv pip install twine yq setuptools_scm - name: Patch package version - run: tomlq -i -t ".package.version = \"$(python -m setuptools_scm)\"" Cargo.toml + run: | + VERSION="$(python -m setuptools_scm | sed 's/\.dev/-dev/'g)" # replace ".dev" with "-dev" to comply with Cargo SemVer rules + echo "Setting package version to: $VERSION" + tomlq -i -t ".package.version = \"$VERSION\"" Cargo.toml + tomlq -i -t ".workspace.package.version = \"$VERSION\"" Cargo.toml - name: Patch name to daft-lts if LTS if: ${{ inputs.lts }} @@ -51,7 +55,7 @@ jobs: - name: Configure RUSTFLAGS for x86 if: ${{ (inputs.arch == 'x86_64') }} run: | - if [[ ${{ inputs.lts }} ]]; then + if [[ "${{ inputs.lts }}" == "true" ]]; then echo "RUSTFLAGS=-C target-feature=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b" >> $GITHUB_ENV && \ echo "CFLAGS=-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16" >> $GITHUB_ENV else diff --git a/.github/workflows/nightly-publish-s3.yml b/.github/workflows/nightly-publish-s3.yml index 304ce6258a..4457dab048 100644 --- a/.github/workflows/nightly-publish-s3.yml +++ b/.github/workflows/nightly-publish-s3.yml @@ -13,36 +13,44 @@ on: workflow_dispatch: env: - OUTPUT_BUCKET: s3://github-actions-artifacts-bucket/builds/nightly/ + S3_BUCKET: s3://github-actions-artifacts-bucket/builds/nightly/ AWS_REGION: us-west-2 CLOUDFRONT_DIST_ID: E3H8WN738AJ1D4 jobs: build: + name: 'Build Daft wheel for ${{ matrix.os }}-${{ matrix.arch }}-lts=${{ matrix.lts }}' uses: ./.github/workflows/build-wheel.yml with: os: ${{ matrix.os }} - arch: ${{ matrix.compile_arch }} + arch: ${{ matrix.arch }} lts: ${{ matrix.lts }} build_type: release strategy: + fail-fast: false matrix: os: [ubuntu, macos, windows] - compile_arch: [x86_64, aarch64] - lts: [0, 1] + arch: [x86_64, aarch64] + lts: [false, true] exclude: - os: windows - compile_arch: aarch64 - - lts: 1 - compile_arch: aarch64 + arch: aarch64 + - lts: true + arch: aarch64 publish: name: Publish wheels to S3 runs-on: ubuntu-latest needs: build + permissions: + id-token: write + contents: read steps: + - uses: actions/checkout@v4 + with: + submodules: true - uses: actions/download-artifact@v4 with: pattern: wheels-* @@ -55,26 +63,22 @@ jobs: role-to-assume: ${{ secrets.ACTIONS_AWS_ROLE_ARN }} role-session-name: DaftPythonPackageGitHubWorkflow - name: Upload wheels to S3 - uses: reggionick/s3-deploy@v4 - with: - folder: dist - bucket: ${{ env.OUTPUT_BUCKET }} - bucket-region: ${{ env.AWS_REGION }} - dist-id: ${{ env.CLOUDFRONT_DIST_ID }} + run: aws s3 cp --no-progress --recursive dist/ ${{ env.S3_BUCKET }} - name: Install boto3 and packaging run: pip3 install boto3 packaging - name: Generate Python simple repository API files - run: python3 .github/ci-scripts/generate_simple_py_index.py + run: python3 .github/ci-scripts/generate_simple_py_index.py ${{ env.S3_BUCKET }} - name: Upload index files to S3 - uses: reggionick/s3-deploy@v4 - with: - folder: dist/indices - bucket: ${{ env.OUTPUT_BUCKET }} - bucket-region: ${{ env.AWS_REGION }} - dist-id: ${{ env.CLOUDFRONT_DIST_ID }} + run: aws s3 cp --no-progress --recursive dist/indices/ ${{ env.S3_BUCKET }} + + - name: Invalidate Cloudfront cache + run: aws cloudfront create-invalidation --distribution-id ${{ env.CLOUDFRONT_DIST_ID }} --paths '/builds/nightly*' + - name: Print install instructions - run: 'echo "To install the nightly build, run: \npip install getdaft --pre --extra-index-url https://d1p3klp2t5517h.cloudfront.net/builds/nightly"' + run: | + echo "To install the nightly build, run:" + echo "pip install getdaft --pre --extra-index-url https://d1p3klp2t5517h.cloudfront.net/builds/nightly" on-failure: name: Send Slack notification on failure diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 93cab338d6..dc9bcd9c9b 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -15,23 +15,25 @@ defaults: jobs: build: + name: 'Build Daft wheel for ${{ matrix.os }}-${{ matrix.arch }}-lts=${{ matrix.lts }}' uses: ./.github/workflows/build-wheel.yml with: os: ${{ matrix.os }} - arch: ${{ matrix.compile_arch }} + arch: ${{ matrix.arch }} lts: ${{ matrix.lts }} build_type: release strategy: + fail-fast: false matrix: os: [ubuntu, macos, windows] - compile_arch: [x86_64, aarch64] - lts: [0, 1] + arch: [x86_64, aarch64] + lts: [false, true] exclude: - os: windows - compile_arch: aarch64 - - lts: 1 - compile_arch: aarch64 + arch: aarch64 + - lts: true + arch: aarch64 publish: name: Publish wheels to PyPI