Reduce Docker image rebuilds (#2269)

* Move `acquire-build-image` into `.github` * Move the `tools-hash` script into `.github` * Upload to ECR from PRs as well * Move build tools into `tools/ci-build/` * Move CI scripts out of `ci-build` * Split CI for forks/non-forks * Remove `fetch-depth: 0` from PR workflows
smithy-lang · Feb 1, 2023 · bed7b97 · bed7b97
1 parent d9c0b2e
commit bed7b97
Show file tree

Hide file tree

Showing 159 changed files with 192 additions and 144 deletions.
diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml
@@ -1,11 +1,11 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# Use this action to execute the action scripts in tools/ci-build/scripts within the Docker build image.
+# Use this action to execute the action scripts in tools/ci-scripts.
 name: smithy-rs Docker Build
 description: Run Docker build command for smithy-rs
 inputs:
-  # The name of the script in tools/ci-build/scripts to run
+  # The name of the script in tools/ci-scripts to run
   action:
     description: What action to run in the Docker build
     required: true
@@ -43,7 +43,7 @@ runs:
       # from attempting to download an image from ECR since it will already exist,
       # which enables testing build image modifications as part of the pull request.
       if [[ -d smithy-rs-base-image ]]; then
-        IMAGE_TAG="$(./smithy-rs/tools/ci-build/tools-hash)"
+        IMAGE_TAG="$(./smithy-rs/.github/scripts/docker-image-hash)"
         docker load -i smithy-rs-base-image/smithy-rs-base-image
         docker tag "smithy-rs-base-image:${IMAGE_TAG}" "smithy-rs-base-image:local"
       fi
@@ -52,7 +52,7 @@ runs:
       # or from ECR. We disable building the image from scratch so that any mistakes in the CI
       # configuration won't cause each individual action to build its own image, which would
       # drastically increase the total CI time. Fail fast!
-      ALLOW_LOCAL_BUILD=false ./smithy-rs/tools/ci-build/acquire-build-image
+      ALLOW_LOCAL_BUILD=false ./smithy-rs/.github/scripts/acquire-build-image
     # This runs the commands from the matrix strategy
   - name: Run ${{ inputs.action }}
     shell: bash

diff --git a/tools/ci-build/acquire-build-image → .github/scripts/acquire-build-image b/tools/ci-build/acquire-build-image → .github/scripts/acquire-build-image
@@ -45,6 +45,7 @@ class Context:
         self.start_path = start_path
         self.script_path = script_path
         self.tools_path = tools_path
+        self.docker_image_path = tools_path + "/ci-build"
         self.user_id = user_id
         self.image_tag = image_tag
         self.allow_local_build = allow_local_build
@@ -56,7 +57,7 @@ class Context:
         script_path = os.path.dirname(os.path.realpath(__file__))
         tools_path = get_cmd_output("git rev-parse --show-toplevel", cwd=script_path)[1] + "/tools"
         user_id = get_cmd_output("id -u")[1]
-        image_tag = get_cmd_output("./ci-build/tools-hash", cwd=tools_path)[1]
+        image_tag = get_cmd_output("./docker-image-hash", cwd=script_path)[1]
         allow_local_build = os.getenv("ALLOW_LOCAL_BUILD") != "false"
         github_actions = os.getenv("GITHUB_ACTIONS") == "true"
         print(f"Start path: {start_path}")
@@ -118,10 +119,10 @@ class Shell:
         run(f"docker build -t \"smithy-rs-base-image:{image_tag}\" .", cwd=path)
 
     # Builds the local build image
-    def docker_build_build_image(self, user_id, script_path):
+    def docker_build_build_image(self, user_id, docker_image_path):
         run(
             f"docker build -t smithy-rs-build-image --file add-local-user.dockerfile --build-arg=USER_ID={user_id} .",
-            cwd=script_path
+            cwd=docker_image_path
         )
 
     # Saves the Docker image named `image_name` with `image_tag` to `output_path`
@@ -188,7 +189,7 @@ def acquire_build_image(context=Context.default(), shell=Shell()):
                 return 1
 
             announce("Building a new image locally.")
-            shell.docker_build_base_image(context.image_tag, context.tools_path)
+            shell.docker_build_base_image(context.image_tag, context.docker_image_path)
 
             if context.github_actions:
                 announce("Saving base image for use in later jobs...")
@@ -205,7 +206,7 @@ def acquire_build_image(context=Context.default(), shell=Shell()):
 
     announce("Creating local build image...")
     shell.docker_tag(LOCAL_BASE_IMAGE_NAME, context.image_tag, LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)
-    shell.docker_build_build_image(context.user_id, context.script_path)
+    shell.docker_build_build_image(context.user_id, context.docker_image_path)
     return 0
 
 
@@ -373,7 +374,7 @@ class SelfTest(unittest.TestCase):
 
         shell.docker_image_exists_locally.assert_called_once()
         shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)
-        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")
+        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path")
 
     # When:
     #  - the base image doesn't exist locally
@@ -393,7 +394,7 @@ class SelfTest(unittest.TestCase):
         shell.docker_build_base_image.assert_called_with("someimagetag", "/tmp/test/tools-path")
         shell.docker_save.assert_not_called()
         shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)
-        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")
+        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path")
 
     # When:
     #  - the base image doesn't exist locally
@@ -413,7 +414,7 @@ class SelfTest(unittest.TestCase):
         shell.docker_build_base_image.assert_called_with("someimagetag", "/tmp/test/tools-path")
         shell.docker_save.assert_not_called()
         shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)
-        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")
+        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path")
 
     # When:
     #  - the base image doesn't exist locally
@@ -437,7 +438,7 @@ class SelfTest(unittest.TestCase):
             "/tmp/test/start-path/smithy-rs-base-image"
         )
         shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)
-        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")
+        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path")
 
     # When:
     #  - the base image doesn't exist locally
@@ -477,7 +478,7 @@ class SelfTest(unittest.TestCase):
             call(REMOTE_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, "someimagetag"),
             call(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)
         ])
-        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")
+        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path")
 
 
 def main():

diff --git a/tools/ci-build/tools-hash → .github/scripts/docker-image-hash b/tools/ci-build/tools-hash → .github/scripts/docker-image-hash
diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml
@@ -31,8 +31,8 @@ jobs:
       uses: actions/checkout@v3
     - name: Build image
       run: |
-        IMAGE_TAG="$(./tools/ci-build/tools-hash)"
-        cd tools
+        IMAGE_TAG="$(./.github/scripts/docker-image-hash)"
+        cd tools/ci-build
         docker build \
           -t "${{ env.ecr_repository }}:${IMAGE_TAG}" \
           -t "${{ env.ecr_repository }}:main" \
@@ -45,7 +45,7 @@ jobs:
         aws-region: us-west-2
     - name: Upload image
       run: |
-        IMAGE_TAG="$(./tools/ci-build/tools-hash)"
+        IMAGE_TAG="$(./.github/scripts/docker-image-hash)"
         aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
         docker push "${{ env.ecr_repository }}:${IMAGE_TAG}"
         docker push "${{ env.ecr_repository }}:main"

diff --git a/.github/workflows/ci-pr-forks.yml b/.github/workflows/ci-pr-forks.yml
@@ -0,0 +1,45 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# This workflow runs CI for pull requests from forks, which can't make use of secrets.
+
+name: CI (from fork)
+on:
+  pull_request:
+
+# Allow one instance of this workflow per pull request, and cancel older runs when new changes are pushed
+concurrency:
+  group: ci-forks-yaml-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # This job detects if the PR made changes to build tools. If it did, then it builds a new
+  # build Docker image. Otherwise, it downloads a build image from Public ECR. In both cases,
+  # it uploads the image as a build artifact for other jobs to download and use.
+  acquire-base-image:
+    name: Acquire Base Image
+    if: ${{ github.event.pull_request.head.repo.full_name != 'awslabs/smithy-rs' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        path: smithy-rs
+    - name: Acquire base image
+      id: acquire
+      env:
+        DOCKER_BUILDKIT: 1
+      run: ./smithy-rs/.github/scripts/acquire-build-image
+    - name: Upload base image
+      uses: actions/upload-artifact@v3
+      with:
+        name: smithy-rs-base-image
+        path: smithy-rs-base-image
+        retention-days: 1
+
+  # Run shared CI after the Docker build image has either been rebuilt or found in ECR
+  ci:
+    needs: acquire-base-image
+    if: ${{ github.event.pull_request.head.repo.full_name != 'awslabs/smithy-rs' }}
+    uses: ./.github/workflows/ci.yml
+    with:
+      run_sdk_examples: true
diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml
@@ -1,7 +1,7 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# This workflow runs CI and the PR Bot on pull requests.
+# This workflow runs CI and the PR Bot on pull requests that are not from forked repositories.
 
 name: CI
 on:
@@ -12,43 +12,55 @@ concurrency:
   group: ci-yaml-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  ecr_repository: public.ecr.aws/w0m4q9l7/github-awslabs-smithy-rs-ci
+
 jobs:
   # This job detects if the PR made changes to build tools. If it did, then it builds a new
   # build Docker image. Otherwise, it downloads a build image from Public ECR. In both cases,
   # it uploads the image as a build artifact for other jobs to download and use.
   acquire-base-image:
     name: Acquire Base Image
+    if: ${{ github.event.pull_request.head.repo.full_name == 'awslabs/smithy-rs' }}
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
     steps:
     - uses: actions/checkout@v3
       with:
         path: smithy-rs
-        fetch-depth: 0
     - name: Acquire base image
       id: acquire
       env:
         DOCKER_BUILDKIT: 1
-      run: ./smithy-rs/tools/ci-build/acquire-build-image
-    - name: Upload base image
-      uses: actions/upload-artifact@v3
+      run: ./smithy-rs/.github/scripts/acquire-build-image
+    - name: Acquire credentials
+      uses: aws-actions/configure-aws-credentials@v1-node16
       with:
-        name: smithy-rs-base-image
-        path: smithy-rs-base-image
-        retention-days: 1
+        role-to-assume: ${{ secrets.SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN }}
+        role-session-name: GitHubActions
+        aws-region: us-west-2
+    - name: Upload image
+      run: |
+        IMAGE_TAG="$(./smithy-rs/.github/scripts/docker-image-hash)"
+        docker tag "smithy-rs-base-image:${IMAGE_TAG}" "${{ env.ecr_repository }}:${IMAGE_TAG}"
+        aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
+        docker push "${{ env.ecr_repository }}:${IMAGE_TAG}"
 
   # Run shared CI after the Docker build image has either been rebuilt or found in ECR
   ci:
     needs: acquire-base-image
+    if: ${{ github.event.pull_request.head.repo.full_name == 'awslabs/smithy-rs' }}
     uses: ./.github/workflows/ci.yml
     with:
       run_sdk_examples: true
 
   # The PR bot requires a Docker build image, so make it depend on the `acquire-base-image` job.
   pr_bot:
     name: PR Bot
+    if: ${{ github.event.pull_request.head.repo.full_name == 'awslabs/smithy-rs' }}
     needs: acquire-base-image
-    # Only run this job on pull requests (not directly on main)
-    if: ${{ github.head_ref }}
     uses: ./.github/workflows/pull-request-bot.yml
     with:
       issue_number: ${{ github.event.number }}

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -20,7 +20,7 @@ on:
           If unspecified, it will default to the git reference or SHA that triggered the execution of this workflow.
         required: false
         type: string
-        default: ""
+        default: ''
 
 env:
   rust_version: 1.62.1
@@ -37,7 +37,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # These correspond to scripts in tools/ci-build/scripts that will be run in the Docker build image
+        # These correspond to scripts in tools/ci-scripts that will be run in the Docker build image
         actions:
         - action: generate-aws-sdk
         - action: generate-aws-sdk-smoketest
@@ -72,7 +72,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # These correspond to scripts in tools/ci-build/scripts that will be run in the Docker build image
+        # These correspond to scripts in tools/ci-scripts that will be run in the Docker build image
         test:
         - action: check-aws-sdk-adhoc-tests
         - action: check-client-codegen-integration-tests
@@ -109,7 +109,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # These correspond to scripts in tools/ci-build/scripts that will be run in the Docker build image
+        # These correspond to scripts in tools/ci-scripts that will be run in the Docker build image
         test:
         - action: check-aws-config
         - action: check-aws-sdk-canary

diff --git a/.github/workflows/claim-crate-names.yml b/.github/workflows/claim-crate-names.yml
@@ -45,7 +45,7 @@ jobs:
         fetch-depth: 0
     - name: Acquire base image
       id: acquire
-      run: ./smithy-rs/tools/ci-build/acquire-build-image
+      run: ./smithy-rs/.github/scripts/acquire-build-image
     - name: Upload base image
       uses: actions/upload-artifact@v3
       with:
@@ -72,7 +72,7 @@ jobs:
         RELEASE_AUTOMATION_BOT_CRATESIO_TOKEN: ${{ secrets.RELEASE_AUTOMATION_BOT_CRATESIO_TOKEN }}
       run: |
         cargo login -- "${RELEASE_AUTOMATION_BOT_CRATESIO_TOKEN}"
-        cargo install --path tools/publisher
+        cargo install --path tools/ci-build/publisher
         # Verify the publisher tool installed successfully
         publisher --version
         publisher claim-crate-names -y
diff --git a/.github/workflows/manual-pull-request-bot.yml b/.github/workflows/manual-pull-request-bot.yml
@@ -53,7 +53,7 @@ jobs:
         fetch-depth: 0
     - name: Acquire base image
       id: acquire
-      run: ./smithy-rs/tools/ci-build/acquire-build-image
+      run: ./smithy-rs/.github/scripts/acquire-build-image
     - name: Upload base image
       uses: actions/upload-artifact@v3
       with:

diff --git a/.github/workflows/pull-request-bot.yml b/.github/workflows/pull-request-bot.yml
@@ -122,7 +122,7 @@ jobs:
 
         cargo doc --no-deps --all-features
         popd
-        ./tools/generate-doc-preview-index.sh ${{ inputs.base_revision }}
+        ./tools/ci-scripts/generate-doc-preview-index.sh ${{ inputs.base_revision }}
 
         echo 'bot-message=A [new doc preview](https://d2luzm2xt3nokh.cloudfront.net/docs/'${{ inputs.head_revision }}'/index.html) is ready to view.' >> "${GITHUB_OUTPUT}"
     - uses: aws-actions/configure-aws-credentials@v1-node16