From 35754c4385231a365e0d077ffa6747322c2c3c3b Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Fri, 20 Dec 2024 23:08:27 +0000 Subject: [PATCH 1/8] add memory reports for execution' --- .github/workflows/reports.yml | 115 ++++++++++++++++++++++++++++----- test_programs/memory_report.sh | 11 +++- 2 files changed, 106 insertions(+), 20 deletions(-) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index f7fe78f6565..7c10452e804 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -217,10 +217,24 @@ jobs: ./memory_report.sh mv memory_report.json ../memory_report.json - - name: Upload memory report + - name: Upload compilation memory report uses: actions/upload-artifact@v4 with: - name: in_progress_memory_report + name: in_progress_compilation_mem_report + path: memory_report.json + retention-days: 3 + overwrite: true + + - name: Generate execution memory report + working-directory: ./test_programs + run: | + ./memory_report.sh 0 1 + mv memory_report.json ../memory_report.json + + - name: Upload execution memory report + uses: actions/upload-artifact@v4 + with: + name: in_progress_execution_mem_report path: memory_report.json retention-days: 3 overwrite: true @@ -484,15 +498,38 @@ jobs: path: test-repo ref: ${{ matrix.project.ref }} - - name: Generate compilation report + - name: Generate compilation memory report working-directory: ./test-repo/${{ matrix.project.path }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh - chmod +x ./memory_report.sh ./memory_report.sh 1 + # Rename the memory report as the execution report is about to write to the same file + cp memory_report.json compilation_memory_report.json + + - name: Generate execution memory report + working-directory: ./test-repo/${{ matrix.project.path }} + run: | + ./memory_report.sh 1 1 - name: Move compilation report - id: report + id: compilation_mem_report + shell: bash + run: | + PACKAGE_NAME=${{ matrix.project.path }} + PACKAGE_NAME=$(basename $PACKAGE_NAME) + mv ./test-repo/${{ matrix.project.path }}/compilation_memory_report.json ./memory_report_$PACKAGE_NAME.json + echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT + + - name: Upload compilation memory report + uses: actions/upload-artifact@v4 + with: + name: compilation_mem_report_${{ steps.compilation_mem_report.outputs.memory_report_name }} + path: memory_report_${{ steps.compilation_mem_report.outputs.memory_report_name }}.json + retention-days: 3 + overwrite: true + + - name: Move execution report + id: execution_mem_report shell: bash run: | PACKAGE_NAME=${{ matrix.project.path }} @@ -500,16 +537,16 @@ jobs: mv ./test-repo/${{ matrix.project.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT - - name: Upload memory report + - name: Upload execution memory report uses: actions/upload-artifact@v4 with: - name: memory_report_${{ steps.report.outputs.memory_report_name }} - path: memory_report_${{ steps.report.outputs.memory_report_name }}.json + name: execution_mem_report_${{ steps.execution_mem_report.outputs.memory_report_name }} + path: memory_report_${{ steps.execution_mem_report.outputs.memory_report_name }}.json retention-days: 3 overwrite: true - upload_memory_report: - name: Upload memory report + upload_compilation_memory_report: + name: Upload compilation memory report needs: [generate_memory_report, external_repo_memory_report] # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails if: always() @@ -523,12 +560,12 @@ jobs: - name: Download initial memory report uses: actions/download-artifact@v4 with: - name: in_progress_memory_report + name: in_progress_compilation_mem_report - name: Download matrix memory reports uses: actions/download-artifact@v4 with: - pattern: memory_report_* + pattern: compilation_mem_report_* path: ./reports - name: Merge memory reports using jq @@ -536,21 +573,65 @@ jobs: mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh ./merge-bench-reports.sh memory_report - - name: Parse memory report - id: memory_report + - name: Parse compilation memory report + id: compilation_mem_report uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4 with: report: memory_report.json header: | - # Memory Report + # Compilation Memory Report memory_report: true - name: Add memory report to sticky comment if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' uses: marocchino/sticky-pull-request-comment@v2 with: - header: memory - message: ${{ steps.memory_report.outputs.markdown }} + header: compilation_memory + message: ${{ steps.compilation_mem_report.outputs.markdown }} + + upload_execution_memory_report: + name: Upload execution memory report + needs: [generate_memory_report, external_repo_memory_report] + # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails + if: always() + runs-on: ubuntu-latest + permissions: + pull-requests: write + + steps: + - uses: actions/checkout@v4 + + - name: Download initial memory report + uses: actions/download-artifact@v4 + with: + name: in_progress_execution_mem_report + + - name: Download matrix memory reports + uses: actions/download-artifact@v4 + with: + pattern: execution_mem_report_* + path: ./reports + + - name: Merge memory reports using jq + run: | + mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh + ./merge-bench-reports.sh memory_report + + - name: Parse execution memory report + id: execution_mem_report + uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4 + with: + report: memory_report.json + header: | + # Execution Memory Report + memory_report: true + + - name: Add execution memory report to sticky comment + if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: execution_memory + message: ${{ steps.execution_mem_report.outputs.markdown }} upload_execution_report: name: Upload execution report diff --git a/test_programs/memory_report.sh b/test_programs/memory_report.sh index 4d03726d374..e501464c198 100755 --- a/test_programs/memory_report.sh +++ b/test_programs/memory_report.sh @@ -12,7 +12,7 @@ current_dir=$(pwd) base_path="$current_dir/execution_success" # If there is an argument that means we want to generate a report for only the current directory -if [ "$#" -ne 0 ]; then +if [ "$1" == "1" ]; then base_path="$current_dir" tests_to_profile=(".") fi @@ -31,11 +31,16 @@ for test_name in ${tests_to_profile[@]}; do echo " ," >> $current_dir"/memory_report.json" fi - if [ "$#" -ne 0 ]; then + if [ "$1" == "1" ]; then test_name=$(basename $current_dir) fi - heaptrack --output $current_dir/$test_name"_heap" $NARGO compile --force + COMMAND="compile --force --silence-warnings" + if [ "$2" == "1" ]; then + COMMAND="execute --silence-warnings" + fi + + heaptrack --output $current_dir/$test_name"_heap" $NARGO $COMMAND if test -f $current_dir/$test_name"_heap.gz"; then heaptrack --analyze $current_dir/$test_name"_heap.gz" > $current_dir/$test_name"_heap_analysis.txt" From 88c69fb1460492ab3a02e5d4382bbb2b08745335 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Fri, 20 Dec 2024 23:41:59 +0000 Subject: [PATCH 2/8] update memory report we compare against and git commit --- .github/workflows/reports.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 7c10452e804..227fe435725 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -439,7 +439,7 @@ jobs: - name: Parse compilation report id: compilation_report - uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4 + uses: noir-lang/noir-bench-report@9c847f328acdfb1a0c106c898bbe7f77db4ac161 with: report: compilation_report.json header: | @@ -575,7 +575,7 @@ jobs: - name: Parse compilation memory report id: compilation_mem_report - uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4 + uses: noir-lang/noir-bench-report@9c847f328acdfb1a0c106c898bbe7f77db4ac161 with: report: memory_report.json header: | @@ -616,12 +616,14 @@ jobs: run: | mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh ./merge-bench-reports.sh memory_report + # Rename the memory report as to not clash with the compilation memory report file name + cp memory_report.json execution_memory_report.json - name: Parse execution memory report id: execution_mem_report - uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4 + uses: noir-lang/noir-bench-report@9c847f328acdfb1a0c106c898bbe7f77db4ac161 with: - report: memory_report.json + report: execution_memory_report.json header: | # Execution Memory Report memory_report: true @@ -663,7 +665,7 @@ jobs: - name: Parse execution report id: execution_report - uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4 + uses: noir-lang/noir-bench-report@9c847f328acdfb1a0c106c898bbe7f77db4ac161 with: report: execution_report.json header: | From 2ae9445cf29cea6d35b2d3f4bda23a41331fe478 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Fri, 20 Dec 2024 23:50:42 +0000 Subject: [PATCH 3/8] no # in header --- .github/workflows/reports.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 227fe435725..0dda98305fd 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -579,7 +579,7 @@ jobs: with: report: memory_report.json header: | - # Compilation Memory Report + Compilation Memory Report memory_report: true - name: Add memory report to sticky comment @@ -625,7 +625,7 @@ jobs: with: report: execution_memory_report.json header: | - # Execution Memory Report + Execution Memory Report memory_report: true - name: Add execution memory report to sticky comment @@ -669,7 +669,7 @@ jobs: with: report: execution_report.json header: | - # Execution Report + Execution Report execution_report: true - name: Add memory report to sticky comment From 31d8b5d9afcb19b7906fcb61a67d9e2b20f9107b Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Sat, 21 Dec 2024 00:00:48 +0000 Subject: [PATCH 4/8] bump commit --- .github/workflows/reports.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 0dda98305fd..d93b4ad99bf 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -439,7 +439,7 @@ jobs: - name: Parse compilation report id: compilation_report - uses: noir-lang/noir-bench-report@9c847f328acdfb1a0c106c898bbe7f77db4ac161 + uses: noir-lang/noir-bench-report@52a725064f39388338ba3cbac0f0696430493e8a with: report: compilation_report.json header: | @@ -575,7 +575,7 @@ jobs: - name: Parse compilation memory report id: compilation_mem_report - uses: noir-lang/noir-bench-report@9c847f328acdfb1a0c106c898bbe7f77db4ac161 + uses: noir-lang/noir-bench-report@52a725064f39388338ba3cbac0f0696430493e8a with: report: memory_report.json header: | @@ -621,7 +621,7 @@ jobs: - name: Parse execution memory report id: execution_mem_report - uses: noir-lang/noir-bench-report@9c847f328acdfb1a0c106c898bbe7f77db4ac161 + uses: noir-lang/noir-bench-report@52a725064f39388338ba3cbac0f0696430493e8a with: report: execution_memory_report.json header: | @@ -665,7 +665,7 @@ jobs: - name: Parse execution report id: execution_report - uses: noir-lang/noir-bench-report@9c847f328acdfb1a0c106c898bbe7f77db4ac161 + uses: noir-lang/noir-bench-report@52a725064f39388338ba3cbac0f0696430493e8a with: report: execution_report.json header: | From 57536494fd4fcd320b8fa8fc9bf8b1ab748e7a73 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Sat, 21 Dec 2024 00:45:07 +0000 Subject: [PATCH 5/8] update commit to bring back accidental reverted code --- .github/workflows/reports.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index d93b4ad99bf..6e64c7aa1ca 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -439,7 +439,7 @@ jobs: - name: Parse compilation report id: compilation_report - uses: noir-lang/noir-bench-report@52a725064f39388338ba3cbac0f0696430493e8a + uses: noir-lang/noir-bench-report@e0a9b8e6bad698c2b1d55819fef45324a9a2f12c with: report: compilation_report.json header: | @@ -575,7 +575,7 @@ jobs: - name: Parse compilation memory report id: compilation_mem_report - uses: noir-lang/noir-bench-report@52a725064f39388338ba3cbac0f0696430493e8a + uses: noir-lang/noir-bench-report@e0a9b8e6bad698c2b1d55819fef45324a9a2f12c with: report: memory_report.json header: | @@ -621,7 +621,7 @@ jobs: - name: Parse execution memory report id: execution_mem_report - uses: noir-lang/noir-bench-report@52a725064f39388338ba3cbac0f0696430493e8a + uses: noir-lang/noir-bench-report@e0a9b8e6bad698c2b1d55819fef45324a9a2f12c with: report: execution_memory_report.json header: | @@ -665,7 +665,7 @@ jobs: - name: Parse execution report id: execution_report - uses: noir-lang/noir-bench-report@52a725064f39388338ba3cbac0f0696430493e8a + uses: noir-lang/noir-bench-report@e0a9b8e6bad698c2b1d55819fef45324a9a2f12c with: report: execution_report.json header: | From eb0b42957d7b72a2178678b085f924e11be833fa Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Sat, 21 Dec 2024 00:45:32 +0000 Subject: [PATCH 6/8] no hashtag --- .github/workflows/reports.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 6e64c7aa1ca..2536b3a2572 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -443,7 +443,7 @@ jobs: with: report: compilation_report.json header: | - # Compilation Report + Compilation Report memory_report: false - name: Add memory report to sticky comment From 03485effc95086cbb1533dc2ee0edc97d0177ec6 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 2 Jan 2025 14:46:43 -0500 Subject: [PATCH 7/8] Update .github/workflows/reports.yml Co-authored-by: Tom French <15848336+TomAFrench@users.noreply.github.com> --- .github/workflows/reports.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 6459aa37875..9112ff47a3d 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -442,7 +442,7 @@ jobs: - name: Parse compilation report id: compilation_report - uses: noir-lang/noir-bench-report@e0a9b8e6bad698c2b1d55819fef45324a9a2f12c + uses: noir-lang/noir-bench-report@6ba151d7795042c4ff51864fbeb13c0a6a79246c with: report: compilation_report.json header: | From 336bde84441b9bae4409f09f139dc840d81f3045 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 2 Jan 2025 19:47:11 +0000 Subject: [PATCH 8/8] update remaining commits --- .github/workflows/reports.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 9112ff47a3d..4b2e5b5e878 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -580,7 +580,7 @@ jobs: - name: Parse compilation memory report id: compilation_mem_report - uses: noir-lang/noir-bench-report@e0a9b8e6bad698c2b1d55819fef45324a9a2f12c + uses: noir-lang/noir-bench-report@6ba151d7795042c4ff51864fbeb13c0a6a79246c with: report: memory_report.json header: | @@ -626,7 +626,7 @@ jobs: - name: Parse execution memory report id: execution_mem_report - uses: noir-lang/noir-bench-report@e0a9b8e6bad698c2b1d55819fef45324a9a2f12c + uses: noir-lang/noir-bench-report@6ba151d7795042c4ff51864fbeb13c0a6a79246c with: report: execution_memory_report.json header: | @@ -670,7 +670,7 @@ jobs: - name: Parse execution report id: execution_report - uses: noir-lang/noir-bench-report@e0a9b8e6bad698c2b1d55819fef45324a9a2f12c + uses: noir-lang/noir-bench-report@6ba151d7795042c4ff51864fbeb13c0a6a79246c with: report: execution_report.json header: |