test-js-packages.yml

name: Javascript Tests

on:
  pull_request:
  merge_group:
  push:
    branches:
      - master

# This will cancel previous runs when a branch or PR is updated
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }}
  cancel-in-progress: true

jobs:
  critical-library-list:
    name: Load critical library list
    runs-on: ubuntu-22.04
    outputs:
      libraries: ${{ steps.get_critical_libraries.outputs.libraries }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Build list of libraries
        id: get_critical_libraries
        run: |
          LIBRARIES=$(yq ./EXTERNAL_NOIR_LIBRARIES.yml -o json | jq -c '.libraries | map({ repo, path: (.path // ""), timeout:(.timeout // 2), nargo_args })')
          echo "libraries=$LIBRARIES"
          echo "libraries=$LIBRARIES" >> $GITHUB_OUTPUT
        env:
          GH_TOKEN: ${{ github.token }}

  yarn-lock:
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      # Errors if installation would result in modifications to yarn.lock
      - name: Install
        run: yarn --immutable
        shell: bash

  build-nargo:
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Checkout Noir repo
        uses: actions/checkout@v4

      - name: Setup toolchain
        uses: dtolnay/rust-toolchain@1.75.0

      - uses: Swatinem/rust-cache@v2
        with:
          key: x86_64-unknown-linux-gnu
          cache-on-failure: true
          save-if: ${{ github.event_name != 'merge_group' }}

      - name: Build Nargo
        run: cargo build --package nargo_cli --release

      - name: Package artifacts
        run: |
          mkdir dist
          cp ./target/release/nargo ./dist/nargo
          7z a -ttar -so -an ./dist/* | 7z a -si ./nargo-x86_64-unknown-linux-gnu.tar.gz

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: nargo
          path: ./dist/*
          retention-days: 3

  build-noirc-abi:
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Checkout Noir repo
        uses: actions/checkout@v4

      - name: Setup toolchain
        uses: dtolnay/rust-toolchain@1.75.0

      - uses: Swatinem/rust-cache@v2
        with:
          key: noirc-abi
          cache-on-failure: true
          save-if: ${{ github.event_name != 'merge_group' }}

      - name: Install Yarn dependencies
        uses: ./.github/actions/setup

      - name: Build noirc_abi
        run: ./.github/scripts/noirc-abi-build.sh

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: noirc_abi_wasm
          path: |
            ./tooling/noirc_abi_wasm/nodejs
            ./tooling/noirc_abi_wasm/web
          retention-days: 10

  build-noir-wasm:
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Checkout sources
        uses: actions/checkout@v4

      - name: Setup toolchain
        uses: dtolnay/rust-toolchain@1.75.0

      - uses: Swatinem/rust-cache@v2
        with:
          key: noir-wasm
          cache-on-failure: true
          save-if: ${{ github.event_name != 'merge_group' }}

      - name: Install Yarn dependencies
        uses: ./.github/actions/setup

      - name: Build noir_js_types
        run: yarn workspace @noir-lang/types build

      - name: Build noir_wasm
        run: ./.github/scripts/noir-wasm-build.sh

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: noir_wasm
          path: |
            ./compiler/wasm/dist
            ./compiler/wasm/build
          retention-days: 3

  build-acvm-js:
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Checkout sources
        uses: actions/checkout@v4

      - name: Setup toolchain
        uses: dtolnay/rust-toolchain@1.75.0

      - uses: Swatinem/rust-cache@v2
        with:
          key: acvm-js
          cache-on-failure: true
          save-if: ${{ github.event_name != 'merge_group' }}

      - name: Install Yarn dependencies
        uses: ./.github/actions/setup

      - name: Build acvm_js
        run: ./.github/scripts/acvm_js-build.sh

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: acvm-js
          path: |
            ./acvm-repo/acvm_js/nodejs
            ./acvm-repo/acvm_js/web
          retention-days: 3

  test-acvm_js-node:
    needs: [build-acvm-js]
    name: ACVM JS (Node.js)
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Checkout sources
        uses: actions/checkout@v4

      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: acvm-js
          path: ./acvm-repo/acvm_js

      - name: Set up test environment
        uses: ./.github/actions/setup

      - name: Run node tests
        run: yarn workspace @noir-lang/acvm_js test

  test-acvm_js-browser:
    needs: [build-acvm-js]
    name: ACVM JS (Browser)
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Checkout sources
        uses: actions/checkout@v4

      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: acvm-js
          path: ./acvm-repo/acvm_js

      - name: Set up test environment
        uses: ./.github/actions/setup

      - name: Install Playwright
        uses: ./.github/actions/install-playwright

      - name: Run browser tests
        run: yarn workspace @noir-lang/acvm_js test:browser

  test-noirc-abi:
    needs: [build-noirc-abi]
    name: noirc_abi
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Checkout sources
        uses: actions/checkout@v4

      - name: Download wasm package artifact
        uses: actions/download-artifact@v4
        with:
          name: noirc_abi_wasm
          path: ./tooling/noirc_abi_wasm

      - name: Install Yarn dependencies
        uses: ./.github/actions/setup

      - name: Run node tests
        run: yarn workspace @noir-lang/noirc_abi test

      - name: Install Playwright
        uses: ./.github/actions/install-playwright

      - name: Run browser tests
        run: yarn workspace @noir-lang/noirc_abi test:browser

  test-noir-js:
    needs: [build-nargo, build-acvm-js, build-noirc-abi]
    name: Noir JS
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download nargo binary
        uses: ./.github/actions/download-nargo

      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: acvm-js
          path: ./acvm-repo/acvm_js

      - name: Download wasm package artifact
        uses: actions/download-artifact@v4
        with:
          name: noirc_abi_wasm
          path: ./tooling/noirc_abi_wasm

      - name: Install Yarn dependencies
        uses: ./.github/actions/setup

      - name: Build noir_js_types
        run: yarn workspace @noir-lang/types build

      - name: Run noir_js tests
        run: |
          yarn workspace @noir-lang/noir_js build
          yarn workspace @noir-lang/noir_js test

  test-noir-wasm:
    needs: [build-noir-wasm, build-nargo]
    name: noir_wasm
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Checkout sources
        uses: actions/checkout@v4

      - name: Download wasm package artifact
        uses: actions/download-artifact@v4
        with:
          name: noir_wasm
          path: ./compiler/wasm

      - name: Install Yarn dependencies
        uses: ./.github/actions/setup

      - name: Download nargo binary
        uses: ./.github/actions/download-nargo

      - name: Build fixtures
        run: yarn workspace @noir-lang/noir_wasm test:build_fixtures

      - name: Install Playwright
        uses: ./.github/actions/install-playwright

      - name: Run node tests
        run: yarn workspace @noir-lang/noir_wasm test:node

      - name: Run browser tests
        run: yarn workspace @noir-lang/noir_wasm test:browser

  test-noir-codegen:
    needs: [build-acvm-js, build-noirc-abi, build-nargo]
    name: noir_codegen
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download nargo binary
        uses: ./.github/actions/download-nargo

      - name: Download acvm_js package artifact
        uses: actions/download-artifact@v4
        with:
          name: acvm-js
          path: ./acvm-repo/acvm_js

      - name: Download noirc_abi package artifact
        uses: actions/download-artifact@v4
        with:
          name: noirc_abi_wasm
          path: ./tooling/noirc_abi_wasm

      - name: Install Yarn dependencies
        uses: ./.github/actions/setup

      - name: Build noir_js_types
        run: yarn workspace @noir-lang/types build

      - name: Build noir_js
        run: yarn workspace @noir-lang/noir_js build

      - name: Run noir_codegen tests
        run: yarn workspace @noir-lang/noir_codegen test

  test-integration-node:
    name: Integration Tests (Node)
    runs-on: ubuntu-22.04
    needs: [build-acvm-js, build-noir-wasm, build-nargo, build-noirc-abi]
    timeout-minutes: 30

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install `bb`
        run: |
          ./scripts/install_bb.sh
          echo "$HOME/.bb/" >> $GITHUB_PATH

      - name: Download nargo binary
        uses: ./.github/actions/download-nargo

      - name: Download acvm_js package artifact
        uses: actions/download-artifact@v4
        with:
          name: acvm-js
          path: ./acvm-repo/acvm_js

      - name: Download noir_wasm package artifact
        uses: actions/download-artifact@v4
        with:
          name: noir_wasm
          path: ./compiler/wasm

      - name: Download noirc_abi package artifact
        uses: actions/download-artifact@v4
        with:
          name: noirc_abi_wasm
          path: ./tooling/noirc_abi_wasm

      - name: Install Yarn dependencies
        uses: ./.github/actions/setup

      - name: Setup `integration-tests`
        run: |
          # Note the lack of spaces between package names.
          PACKAGES_TO_BUILD="@noir-lang/types,@noir-lang/noir_js"
          yarn workspaces foreach -vtp --from "{$PACKAGES_TO_BUILD}" run build

      - name: Run `integration-tests`
        working-directory: ./compiler/integration-tests
        run: |
          yarn test:node

  test-integration-browser:
    name: Integration Tests (Browser)
    runs-on: ubuntu-22.04
    needs: [build-acvm-js, build-noir-wasm, build-noirc-abi]
    timeout-minutes: 30

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download acvm_js package artifact
        uses: actions/download-artifact@v4
        with:
          name: acvm-js
          path: ./acvm-repo/acvm_js

      - name: Download noir_wasm package artifact
        uses: actions/download-artifact@v4
        with:
          name: noir_wasm
          path: ./compiler/wasm

      - name: Download noirc_abi package artifact
        uses: actions/download-artifact@v4
        with:
          name: noirc_abi_wasm
          path: ./tooling/noirc_abi_wasm

      - name: Install Yarn dependencies
        uses: ./.github/actions/setup

      - name: Install Playwright
        uses: ./.github/actions/install-playwright

      - name: Setup `integration-tests`
        run: |
          # Note the lack of spaces between package names.
          PACKAGES_TO_BUILD="@noir-lang/types,@noir-lang/noir_js"
          yarn workspaces foreach -vtp --from "{$PACKAGES_TO_BUILD}" run build

      - name: Run `integration-tests`
        working-directory: ./compiler/integration-tests
        run: |
          yarn test:browser

  test-examples:
    name: Example scripts
    runs-on: ubuntu-22.04
    needs: [build-nargo]
    timeout-minutes: 30

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Foundry
        uses: foundry-rs/foundry-toolchain@v1.2.0
        with:
          version: nightly-8660e5b941fe7f4d67e246cfd3dafea330fb53b1

      - name: Install `bb`
        run: |
          ./scripts/install_bb.sh
          echo "$HOME/.bb/" >> $GITHUB_PATH

      - name: Download nargo binary
        uses: ./.github/actions/download-nargo

      - name: Run `prove_and_verify`
        working-directory: ./examples/prove_and_verify
        run: ./test.sh

      - name: Run `codegen_verifier`
        working-directory: ./examples/codegen_verifier
        run: ./test.sh

  external-repo-checks:
    needs: [build-nargo, critical-library-list]
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson( needs.critical-library-list.outputs.libraries )}}
         
    name: Check external repo - ${{ matrix.repo }}/${{ matrix.path }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: noir-repo

      - name: Checkout
        uses: actions/checkout@v4
        with:
          repository: ${{ matrix.repo }}
          path: test-repo
          ref: ${{ matrix.ref }}

      - name: Download nargo binary
        uses: ./noir-repo/.github/actions/download-nargo

      - name: Remove requirements on compiler version
        working-directory: ./test-repo
        run: |
          # Github actions seems to not expand "**" in globs by default.
          shopt -s globstar
          sed -i '/^compiler_version/d' ./**/Nargo.toml

      - name: Run nargo test
        id: test_report
        working-directory: ./test-repo/${{ matrix.path }}
        run: |
          output_file=${{ github.workspace }}/noir-repo/.github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.actual.jsonl
          BEFORE=$SECONDS
          nargo test --silence-warnings --skip-brillig-constraints-check --format json ${{ matrix.nargo_args }} | tee $output_file
          TIME=$(($SECONDS-$BEFORE))

          NAME=${{ matrix.repo }}/${{ matrix.path }}
          # Replace any slashes with underscores
          NAME=${NAME//\//_}
          TEST_REPORT_NAME=test_report_$NAME
          echo "test_report_name=$TEST_REPORT_NAME" >> $GITHUB_OUTPUT

          jq --null-input "[{ name: \"$NAME\", value: (\"$TIME\" | tonumber), unit: \"s\" }]" > $TEST_REPORT_NAME.json

          if [ ! -s $output_file ]; then
            # The file is empty so we delete it to signal that `nargo test` failed before it could run any tests
            rm -f $output_file
          fi
        env:
          NARGO_IGNORE_TEST_FAILURES_FROM_FOREIGN_CALLS: true
      
      - name: Check test time limit
        run: |
          TIME=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json)
          if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.timeout }}"; then
            # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
            echo "Failing due to test suite exceeding timeout..."
            echo "Timeout: ${{ matrix.timeout }}"
            echo "Test suite took: $TIME".
            exit 1
          fi

      - name: Compare test results
        working-directory: ./noir-repo
        run: .github/scripts/check_test_results.sh .github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.failures.jsonl .github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.actual.jsonl

      - name: Upload test report
        if: ${{ matrix.timeout > 10 }} # We want to avoid recording benchmarking for a ton of tiny libraries, these should be covered with aggressive timeouts
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.test_report.outputs.test_report_name }}
          path: ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json
          retention-days: 3
          overwrite: true

  compile-noir-contracts:
    needs: [build-nargo]
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    name: Compile `noir-contracts` zero inliner aggressiveness
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: noir-repo
      
      - name: Checkout
        uses: actions/checkout@v4
        with:
          repository: AztecProtocol/aztec-packages
          path: test-repo

      - name: Download nargo binary
        uses: ./noir-repo/.github/actions/download-nargo

      - name: Remove requirements on compiler version
        working-directory: ./test-repo
        run: |
          # Github actions seems to not expand "**" in globs by default.
          shopt -s globstar
          sed -i '/^compiler_version/d' ./**/Nargo.toml

      - name: Run nargo compile
        working-directory: ./test-repo/noir-projects/noir-contracts
        run: nargo compile --inliner-aggressiveness 0 

  upload_critical_library_report:
    name: Upload critical library report 
    needs: [external-repo-checks]
    # We want this job to run even if one variation of the matrix in `external-repo-checks` fails
    if: always() 
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write
      # deployments permission to deploy GitHub pages website
      deployments: write
      # contents permission to update benchmark contents in gh-pages branch
      contents: write

    steps:
      - uses: actions/checkout@v4

      - name: Download matrix test reports
        uses: actions/download-artifact@v4
        with:
          pattern: test_report_*
          path: ./reports
          merge-multiple: true

      - name: Merge test reports using jq
        run: |
          # Github actions seems to not expand "**" in globs by default.
          shopt -s globstar
          jq --slurp '. | flatten' ./reports/* | tee test_bench.json

      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
        with:
          name: "Test Suite Duration"
          tool: "customSmallerIsBetter"
          output-file-path: ./test_bench.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          # We want this to only run on master to avoid garbage data from PRs being added.
          auto-push: ${{ github.ref == 'refs/heads/master' }}
          alert-threshold: "120%"
          comment-on-alert: true
          comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
          fail-on-alert: false
          alert-comment-cc-users: "@TomAFrench"
          max-items-in-chart: 50


  # This is a job which depends on all test jobs and reports the overall status.
  # This allows us to add/remove test jobs without having to update the required workflows.
  tests-end:
    name: End
    runs-on: ubuntu-22.04
    # We want this job to always run (even if the dependant jobs fail) as we want this job to fail rather than skipping.
    if: ${{ always() }}
    needs:
      - yarn-lock
      - test-acvm_js-node
      - test-acvm_js-browser
      - test-noirc-abi
      - test-noir-js
      - test-noir-wasm
      - test-noir-codegen
      - test-integration-node
      - test-integration-browser
      - test-examples
      - compile-noir-contracts

    steps:
        - name: Report overall success
          run: |
            if [[ $FAIL == true ]]; then
                exit 1
            else
                exit 0
            fi
          env:
            # We treat any skipped or failing jobs as a failure for the workflow as a whole.
            FAIL: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }}