s3 upload

AztecProtocol · Sep 30, 2024 · b095c0b · b095c0b
1 parent 3509011
commit b095c0b
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 43 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -178,10 +178,7 @@ jobs:
           # command to produce the images in case they don't exist
           builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
           run: |
-            set -eux
-            cd ./yarn-project/end-to-end/
-            export FORCE_COLOR=1
-            ../../scripts/earthly-ci -P --no-output +${{ matrix.test }}
+            cd ./yarn-project/end-to-end/ && ../../scripts/earthly-ci -P --no-output +${{ matrix.test }}
 
   # all the benchmarking end-to-end integration tests for aztec (not required to merge)
   bench-e2e:
@@ -213,11 +210,7 @@ jobs:
           builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
           tester_ttl: 40
           run: |
-            set -eux
-            cd ./yarn-project/end-to-end/
-            export FORCE_COLOR=1
-            export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
-            ../../scripts/earthly-ci --exec-stats -P --no-output +${{ matrix.test }}
+            cd ./yarn-project/end-to-end/ &&  ../../scripts/earthly-ci --exec-stats -P --no-output +${{ matrix.test }}
 
   acir-bench:
     runs-on: ubuntu-20.04
@@ -243,12 +236,7 @@ jobs:
           builder_images_to_copy: aztecprotocol/barretenberg-acir-benches:${{ env.GIT_COMMIT }}
           # command to produce the images in case they don't exist
           builder_command: cd noir && ../scripts/earthly-ci +export-bench-acir-bb
-          run: |
-            set -eux
-            cd ./noir/
-            export FORCE_COLOR=1
-            export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
-            ../scripts/earthly-ci -P --no-output +bench-publish-acir-bb
+          run: cd noir && ../scripts/earthly-ci -P --no-output ./noir/+bench-publish-acir-bb
 
   bench-summary:
     needs:
@@ -590,8 +578,6 @@ jobs:
         with:
           runner_type: builder-x86
           run: |
-            set -eux
-            echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
             scripts/earthly-ci --push ./barretenberg/cpp/+bench-binaries
       - name: Run Bench
         uses: ./.github/ensure-tester
@@ -684,8 +670,7 @@ jobs:
         working-directory: ./noir-projects/
         timeout-minutes: 40
         run: |
-          earthly-ci \
-          --artifact +gates-report/gates_report.json
+          earthly-ci --artifact +gates-report/gates_report.json
           mv gates_report.json ../protocol_circuits_report.json
 
       - name: Compare gates reports
@@ -720,8 +705,7 @@ jobs:
         working-directory: ./noir-projects/
         timeout-minutes: 40
         run: |
-          earthly-ci \
-          --artifact +public-functions-report/public_functions_report.json
+          earthly-ci --artifact +public-functions-report/public_functions_report.json
           mv public_functions_report.json ../public_functions_report.json
 
       - name: Compare public functions bytecode size reports

diff --git a/scripts/earthly-ci b/scripts/earthly-ci
@@ -33,17 +33,6 @@ function wipe_non_cache_docker_state {
 }
 
 EARTHLY_RUN_STATS_JSON="earthly-run-stats.json"
-# Run earthly with our necesary secrets initialized
-# AWS credentials can be blank, however we will not use the S3 cache at all if so.
-if [ "$(git rev-parse --abbrev-ref HEAD)" = master ] || [ "$(git rev-parse --abbrev-ref HEAD)" = *build-cache* ] ; then
-  # We upload to S3 on master. Meant for CI on master.
-  S3_BUILD_CACHE_UPLOAD=true
-else
-  # Don't upload to S3 on pull request branches (for now, at least)
-  S3_BUILD_CACHE_UPLOAD=false
-fi
-
-EARTHLY_ARGS=""
 
 function run_earthly() {
   # We abuse secrets with regular config that we don't want to alter the cache (see https://docs.earthly.dev/docs/guides/secrets)
@@ -52,7 +41,7 @@ function run_earthly() {
           --secret AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} \
           --secret AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} \
           --secret S3_BUILD_CACHE_MINIO_URL="" \
-          --secret S3_BUILD_CACHE_UPLOAD="$S3_BUILD_CACHE_UPLOAD" \
+          --secret S3_BUILD_CACHE_UPLOAD="true" \
           --secret S3_BUILD_CACHE_DOWNLOAD="true" \
           --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${AZTEC_BOT_GITHUB_TOKEN:-} \
           $EARTHLY_ARGS \

diff --git a/scripts/run_on_builder b/scripts/run_on_builder
@@ -6,11 +6,14 @@ cd "$(dirname "$0")/.."
 
 # Define environment variables
 ENV_VARS="
-  DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD
-  AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
-  AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
-  BUILD_INSTANCE_SSH_KEY=$BUILD_INSTANCE_SSH_KEY
-  GIT_COMMIT=${GIT_COMMIT:-}
+  export DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD
+  export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
+  export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+  export BUILD_INSTANCE_SSH_KEY=$BUILD_INSTANCE_SSH_KEY
+  export GIT_COMMIT=${GIT_COMMIT:-}
+  export EARTHLY_BUILD_ARGS=\"${EARTHLY_BUILD_ARGS:-}\"
+  export FORCE_COLOR=1
+  echo ${{ $DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
 "
 
 # Format the environment variables for the SSH command

diff --git a/scripts/run_on_tester b/scripts/run_on_tester
@@ -6,11 +6,14 @@ cd "$(dirname "$0")/.."
 
 # Define environment variables
 ENV_VARS="
-  DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD
-  AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
-  AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
-  BUILD_INSTANCE_SSH_KEY=$BUILD_INSTANCE_SSH_KEY
-  GIT_COMMIT=${GIT_COMMIT:-}
+  export DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD
+  export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
+  export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+  export BUILD_INSTANCE_SSH_KEY=$BUILD_INSTANCE_SSH_KEY
+  export GIT_COMMIT=${GIT_COMMIT:-}
+  export EARTHLY_BUILD_ARGS=\"${EARTHLY_BUILD_ARGS:-}\"
+  export FORCE_COLOR=1
+  echo ${{ $DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
 "
 
 # Format the environment variables for the SSH command