diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 63cf0e937d1..b5be9236e09 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -282,9 +282,9 @@ jobs: strategy: fail-fast: false matrix: - project: ${{ fromJson( needs.benchmark-projects-list.outputs.projects )}} + include: ${{ fromJson( needs.benchmark-projects-list.outputs.projects )}} - name: External repo compilation and execution reports - ${{ matrix.project.repo }}/${{ matrix.project.path }} + name: External repo compilation and execution reports - ${{ matrix.repo }}/${{ matrix.path }} steps: - uses: actions/checkout@v4 with: @@ -302,32 +302,77 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - repository: ${{ matrix.project.repo }} + repository: ${{ matrix.repo }} path: test-repo - ref: ${{ matrix.project.ref }} + ref: ${{ matrix.ref }} - name: Fetch noir dependencies - working-directory: ./test-repo/${{ matrix.project.path }} + working-directory: ./test-repo/${{ matrix.path }} run: | # We run `nargo check` to pre-fetch any dependencies so we don't measure the time to download these # when benchmarking. nargo check - name: Generate compilation report - working-directory: ./test-repo/${{ matrix.project.path }} + id: compilation_report + working-directory: ./test-repo/${{ matrix.path }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh touch parse_time.sh chmod +x parse_time.sh cp /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh - ./compilation_report.sh 1 ${{ matrix.project.num_runs }} + ./compilation_report.sh 1 ${{ matrix.num_runs }} + + PACKAGE_NAME=${{ matrix.path }} + PACKAGE_NAME=$(basename $PACKAGE_NAME) + REPORT_NAME=compilation_report_$PACKAGE_NAME.json + REPORT_PATH=$(pwd)/$REPORT_NAME + mv ./compilation_report.json $REPORT_PATH + + echo "report_name=$REPORT_NAME" >> $GITHUB_OUTPUT + echo "report_path=$REPORT_PATH" >> $GITHUB_OUTPUT env: - FLAGS: ${{ matrix.project.flags }} + FLAGS: ${{ matrix.flags }} + + - name: Upload compilation report + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.compilation_report.outputs.report_name }} + path: ${{ steps.compilation_report.outputs.report_path }} + retention-days: 3 + overwrite: true + - name: Generate execution report + id: execution_report + working-directory: ./test-repo/${{ matrix.path }} + if: ${{ !matrix.cannot_execute }} + run: | + mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh + mv /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh + ./execution_report.sh 1 ${{ matrix.num_runs }} + + PACKAGE_NAME=${{ matrix.path }} + PACKAGE_NAME=$(basename $PACKAGE_NAME) + REPORT_NAME=execution_report_$PACKAGE_NAME.json + REPORT_PATH=$(pwd)/$REPORT_NAME + mv ./execution_report.json $REPORT_PATH + + echo "report_name=$REPORT_NAME" >> $GITHUB_OUTPUT + echo "report_path=$REPORT_PATH" >> $GITHUB_OUTPUT + + - name: Upload execution report + if: ${{ !matrix.cannot_execute }} + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.execution_report.outputs.report_name }} + path: ${{ steps.execution_report.outputs.report_path }} + retention-days: 3 + overwrite: true + - name: Check compilation time limit run: | - TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/compilation_report.json) - TIME_LIMIT=${{ matrix.project.compilation-timeout }} + TIME=$(jq '.[0].value' ${{ steps.compilation_report.outputs.report_path }}) + TIME_LIMIT=${{ matrix.compilation-timeout }} if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "$TIME_LIMIT"; then # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause. echo "Failing due to compilation exceeding timeout..." @@ -336,19 +381,11 @@ jobs: exit 1 fi - - name: Generate execution report - working-directory: ./test-repo/${{ matrix.project.path }} - if: ${{ !matrix.project.cannot_execute }} - run: | - mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh - mv /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh - ./execution_report.sh 1 ${{ matrix.project.num_runs }} - - name: Check execution time limit - if: ${{ !matrix.project.cannot_execute }} + if: ${{ !matrix.cannot_execute }} run: | - TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/execution_report.json) - TIME_LIMIT=${{ matrix.project.execution-timeout }} + TIME=$(jq '.[0].value' ${{ steps.execution_report.outputs.report_path }}) + TIME_LIMIT=${{ matrix.execution-timeout }} if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "$TIME_LIMIT"; then # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause. echo "Failing due to execution exceeding timeout..." @@ -357,41 +394,6 @@ jobs: exit 1 fi - - name: Move compilation report - id: compilation_report - shell: bash - run: | - PACKAGE_NAME=${{ matrix.project.path }} - PACKAGE_NAME=$(basename $PACKAGE_NAME) - mv ./test-repo/${{ matrix.project.path }}/compilation_report.json ./compilation_report_$PACKAGE_NAME.json - echo "compilation_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT - - - name: Move execution report - id: execution_report - shell: bash - if: ${{ !matrix.project.cannot_execute }} - run: | - PACKAGE_NAME=${{ matrix.project.path }} - PACKAGE_NAME=$(basename $PACKAGE_NAME) - mv ./test-repo/${{ matrix.project.path }}/execution_report.json ./execution_report_$PACKAGE_NAME.json - echo "execution_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT - - - name: Upload compilation report - uses: actions/upload-artifact@v4 - with: - name: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }} - path: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }}.json - retention-days: 3 - overwrite: true - - - name: Upload execution report - uses: actions/upload-artifact@v4 - with: - name: execution_report_${{ steps.execution_report.outputs.execution_report_name }} - path: execution_report_${{ steps.execution_report.outputs.execution_report_name }}.json - retention-days: 3 - overwrite: true - external_repo_memory_report: needs: [build-nargo, benchmark-projects-list] runs-on: ubuntu-22.04 @@ -423,19 +425,61 @@ jobs: ref: ${{ matrix.ref }} - name: Generate compilation memory report + id: compilation_report working-directory: ./test-repo/${{ matrix.path }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh mv /home/runner/work/noir/noir/scripts/test_programs/parse_memory.sh ./parse_memory.sh ./memory_report.sh 1 + # Rename the memory report as the execution report is about to write to the same file - cp memory_report.json compilation_memory_report.json + PACKAGE_NAME=${{ matrix.path }} + PACKAGE_NAME=$(basename $PACKAGE_NAME) + REPORT_NAME=compilation_mem_report_$PACKAGE_NAME.json + REPORT_PATH=$(pwd)/$REPORT_NAME + mv ./memory_report.json $REPORT_PATH + + echo "report_name=$REPORT_NAME" >> $GITHUB_OUTPUT + echo "report_path=$REPORT_PATH" >> $GITHUB_OUTPUT env: FLAGS: ${{ matrix.flags }} + - name: Upload compilation memory report + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.compilation_report.outputs.report_name }} + path: ${{ steps.compilation_report.outputs.report_path }} + retention-days: 3 + overwrite: true + + - name: Generate execution memory report + id: execution_report + working-directory: ./test-repo/${{ matrix.path }} + if: ${{ !matrix.cannot_execute }} + run: | + ./memory_report.sh 1 1 + + PACKAGE_NAME=${{ matrix.path }} + PACKAGE_NAME=$(basename $PACKAGE_NAME) + REPORT_NAME=execution_mem_report_$PACKAGE_NAME.json + REPORT_PATH=$(pwd)/$REPORT_NAME + mv ./memory_report.json $REPORT_PATH + + echo "report_name=$REPORT_NAME" >> $GITHUB_OUTPUT + echo "report_path=$REPORT_PATH" >> $GITHUB_OUTPUT + + - name: Upload execution memory report + if: ${{ !matrix.cannot_execute }} + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.execution_report.outputs.report_name }} + path: ${{ steps.execution_report.outputs.report_path }} + retention-days: 3 + overwrite: true + - name: Check compilation memory limit run: | - MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/compilation_memory_report.json) + MEMORY=$(jq '.[0].value' ${{ steps.compilation_report.outputs.report_path }}) MEMORY_LIMIT=${{ matrix.compilation-memory-limit }} if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$MEMORY" "$MEMORY_LIMIT"; then # Don't bump this limit without understanding why this has happened and confirming that you're not the cause. @@ -445,16 +489,10 @@ jobs: exit 1 fi - - name: Generate execution memory report - working-directory: ./test-repo/${{ matrix.path }} - if: ${{ !matrix.cannot_execute }} - run: | - ./memory_report.sh 1 1 - - name: Check execution memory limit if: ${{ !matrix.cannot_execute }} run: | - MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/memory_report.json) + MEMORY=$(jq '.[0].value' ${{ steps.execution_report.outputs.report_path }}) MEMORY_LIMIT=${{ matrix.execution-memory-limit }} if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$MEMORY" "$MEMORY_LIMIT"; then # Don't bump this limit without understanding why this has happened and confirming that you're not the cause. @@ -464,40 +502,6 @@ jobs: exit 1 fi - - name: Move compilation report - id: compilation_mem_report - shell: bash - run: | - PACKAGE_NAME=${{ matrix.path }} - PACKAGE_NAME=$(basename $PACKAGE_NAME) - mv ./test-repo/${{ matrix.path }}/compilation_memory_report.json ./memory_report_$PACKAGE_NAME.json - echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT - - - name: Upload compilation memory report - uses: actions/upload-artifact@v4 - with: - name: compilation_mem_report_${{ steps.compilation_mem_report.outputs.memory_report_name }} - path: memory_report_${{ steps.compilation_mem_report.outputs.memory_report_name }}.json - retention-days: 3 - overwrite: true - - - name: Move execution report - id: execution_mem_report - if: ${{ !matrix.cannot_execute }} - run: | - PACKAGE_NAME=${{ matrix.path }} - PACKAGE_NAME=$(basename $PACKAGE_NAME) - mv ./test-repo/${{ matrix.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json - echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT - - - name: Upload execution memory report - uses: actions/upload-artifact@v4 - with: - name: execution_mem_report_${{ steps.execution_mem_report.outputs.memory_report_name }} - path: memory_report_${{ steps.execution_mem_report.outputs.memory_report_name }}.json - retention-days: 3 - overwrite: true - upload_compilation_report: name: Upload compilation report needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report] diff --git a/.github/workflows/test-js-packages.yml b/.github/workflows/test-js-packages.yml index cfc97d10146..9f6e4c246df 100644 --- a/.github/workflows/test-js-packages.yml +++ b/.github/workflows/test-js-packages.yml @@ -541,17 +541,6 @@ jobs: fi env: NARGO_IGNORE_TEST_FAILURES_FROM_FOREIGN_CALLS: true - - - name: Check test time limit - run: | - TIME=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json) - if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.timeout }}"; then - # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause. - echo "Failing due to test suite exceeding timeout..." - echo "Timeout: ${{ matrix.timeout }}" - echo "Test suite took: $TIME". - exit 1 - fi - name: Compare test results working-directory: ./noir-repo @@ -566,6 +555,17 @@ jobs: retention-days: 3 overwrite: true + - name: Check test time limit + run: | + TIME=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json) + if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.timeout }}"; then + # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause. + echo "Failing due to test suite exceeding timeout..." + echo "Timeout: ${{ matrix.timeout }}" + echo "Test suite took: $TIME". + exit 1 + fi + compile-noir-contracts: needs: [build-nargo] runs-on: ubuntu-22.04