From 1fc7f28bf02ab42128181cc9cd751edd318a09c2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 5 Sep 2022 11:23:23 +0100 Subject: [PATCH 1/6] Reduce number of CI jobs run on PRs --- .ci/scripts/calculate_trial_jobs.py | 74 +++++++++++++++++++++++++++++ .github/workflows/tests.yml | 58 ++++++++-------------- 2 files changed, 95 insertions(+), 37 deletions(-) create mode 100755 .ci/scripts/calculate_trial_jobs.py diff --git a/.ci/scripts/calculate_trial_jobs.py b/.ci/scripts/calculate_trial_jobs.py new file mode 100755 index 000000000000..a8b532cbaceb --- /dev/null +++ b/.ci/scripts/calculate_trial_jobs.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Calculate the trial jobs to run based on if we're in a PR or not. + +import json +import os + +IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/") + +sqlite_tests = [ + { + "python-version": "3.7", + "database": "sqlite", + "extras": "all", + } +] + +if not IS_PR: + sqlite_tests.extend( + { + "python-version": version, + "database": "sqlite", + "extras": "all", + } + for version in ("3.8", "3.9", "3.10") + ) + + +postgres_tests = [ + { + "python-version": "3.7", + "database": "postgres", + "postgres-version": "10", + "extras": "all", + } +] + +if not IS_PR: + postgres_tests.append( + { + "python-version": "3.10", + "database": "postgres", + "postgres-version": "14", + "extras": "all", + } + ) + +no_extra_tests = [ + { + "python-version": "3.7", + "database": "sqlite", + "extras": "", + } +] + +print("::group::Calculated jobs") +print(json.dumps(sqlite_tests + postgres_tests + no_extra_tests, indent=4)) +print("::endgroup::") + +test_matrix = json.dumps(sqlite_tests + postgres_tests + no_extra_tests) +print(f"::set-output name=test_matrix::{test_matrix}") diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 720cb13907fd..a913eaa0376f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -73,63 +73,47 @@ jobs: steps: - run: "true" - trial: + trial-calculate-jobs: if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail needs: linting-done runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - id: get-matrix + run: .ci/scripts/calculate_trial_jobs.py + outputs: + test_matrix: ${{ steps.get-matrix.outputs.test_matrix }} + + trial: + if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail + needs: trial-calculate-jobs + runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] - database: ["sqlite"] - extras: ["all"] - is_pr: - - ${{ startsWith(github.ref, 'refs/pull/') }} - - # If we're a PR then we only test min and max python. - exclude: - - is_pr: true - python-version: 3.8 - - is_pr: true - python-version: 3.9 - - include: - # Newest Python without optional deps - - python-version: "3.10" - extras: "" - - # Oldest Python with PostgreSQL - - python-version: "3.7" - database: "postgres" - postgres-version: "10" - extras: "all" - - # Newest Python with newest PostgreSQL - - python-version: "3.10" - database: "postgres" - postgres-version: "14" - extras: "all" + job: ${{ fromJson(needs.trial-calculate-jobs.outputs.test_matrix) }} steps: - uses: actions/checkout@v2 - run: sudo apt-get -qq install xmlsec1 - - name: Set up PostgreSQL ${{ matrix.postgres-version }} - if: ${{ matrix.postgres-version }} + - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} + if: ${{ matrix.job.postgres-version }} run: | docker run -d -p 5432:5432 \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ - postgres:${{ matrix.postgres-version }} + postgres:${{ matrix.job.postgres-version }} - uses: matrix-org/setup-python-poetry@v1 with: - python-version: ${{ matrix.python-version }} - extras: ${{ matrix.extras }} + python-version: ${{ matrix.job.python-version }} + extras: ${{ matrix.job.extras }} - name: Await PostgreSQL - if: ${{ matrix.postgres-version }} + if: ${{ matrix.job.postgres-version }} timeout-minutes: 2 run: until pg_isready -h localhost; do sleep 1; done - run: poetry run trial --jobs=2 tests env: - SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }} + SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }} SYNAPSE_POSTGRES_HOST: localhost SYNAPSE_POSTGRES_USER: postgres SYNAPSE_POSTGRES_PASSWORD: postgres From 67cdd5807165ff69ff437b19286baac43f61a6b0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 5 Sep 2022 11:24:07 +0100 Subject: [PATCH 2/6] Newsfile --- changelog.d/13713.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/13713.misc diff --git a/changelog.d/13713.misc b/changelog.d/13713.misc new file mode 100644 index 000000000000..104409954279 --- /dev/null +++ b/changelog.d/13713.misc @@ -0,0 +1 @@ +Reduce number of CI checks we run for PRs. From 2a9c5624a3a6543750a5cad12b28223b42cce2b0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 5 Sep 2022 11:35:44 +0100 Subject: [PATCH 3/6] Also limit sytest jobs --- ...culate_trial_jobs.py => calculate_jobs.py} | 72 ++++++++++++++++--- .github/workflows/tests.yml | 46 ++++-------- 2 files changed, 77 insertions(+), 41 deletions(-) rename .ci/scripts/{calculate_trial_jobs.py => calculate_jobs.py} (51%) diff --git a/.ci/scripts/calculate_trial_jobs.py b/.ci/scripts/calculate_jobs.py similarity index 51% rename from .ci/scripts/calculate_trial_jobs.py rename to .ci/scripts/calculate_jobs.py index a8b532cbaceb..c99d183f44ca 100755 --- a/.ci/scripts/calculate_trial_jobs.py +++ b/.ci/scripts/calculate_jobs.py @@ -20,7 +20,11 @@ IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/") -sqlite_tests = [ +# First calculate the various trial jobs. +# +# For each type of test we only run on Py3.7 on PRs + +trial_sqlite_tests = [ { "python-version": "3.7", "database": "sqlite", @@ -29,7 +33,7 @@ ] if not IS_PR: - sqlite_tests.extend( + trial_sqlite_tests.extend( { "python-version": version, "database": "sqlite", @@ -39,7 +43,7 @@ ) -postgres_tests = [ +trial_postgres_tests = [ { "python-version": "3.7", "database": "postgres", @@ -49,7 +53,7 @@ ] if not IS_PR: - postgres_tests.append( + trial_postgres_tests.append( { "python-version": "3.10", "database": "postgres", @@ -58,7 +62,7 @@ } ) -no_extra_tests = [ +trial_no_extra_tests = [ { "python-version": "3.7", "database": "sqlite", @@ -66,9 +70,59 @@ } ] -print("::group::Calculated jobs") -print(json.dumps(sqlite_tests + postgres_tests + no_extra_tests, indent=4)) +print("::group::Calculated trial jobs") +print( + json.dumps( + trial_sqlite_tests + trial_postgres_tests + trial_no_extra_tests, indent=4 + ) +) +print("::endgroup::") + +test_matrix = json.dumps( + trial_sqlite_tests + trial_postgres_tests + trial_no_extra_tests +) +print(f"::set-output name=trial_test_matrix::{test_matrix}") + + +# First calculate the various sytest jobs. +# +# For each type of test we only run on focal on PRs + + +sytest_tests = [ + { + "sytest-tag": "focal", + }, + { + "sytest-tag": "focal", + "postgres": "postgres", + }, + { + "sytest-tag": "focal", + "postgres": "mulit-postgres", + "workers": "workers", + }, +] + +if not IS_PR: + sytest_tests.extend( + [ + { + "sytest-tag": "testing", + "postgres": "postgres", + }, + { + "sytest-tag": "buster", + "postgres": "mulit-postgres", + "workers": "workers", + }, + ] + ) + + +print("::group::Calculated sytest jobs") +print(json.dumps(sytest_tests, indent=4)) print("::endgroup::") -test_matrix = json.dumps(sqlite_tests + postgres_tests + no_extra_tests) -print(f"::set-output name=test_matrix::{test_matrix}") +test_matrix = json.dumps(sytest_tests) +print(f"::set-output name=sytest_test_matrix::{test_matrix}") diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a913eaa0376f..d4a67895ddb4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -73,7 +73,7 @@ jobs: steps: - run: "true" - trial-calculate-jobs: + calculate-test-jobs: if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail needs: linting-done runs-on: ubuntu-latest @@ -83,15 +83,16 @@ jobs: - id: get-matrix run: .ci/scripts/calculate_trial_jobs.py outputs: - test_matrix: ${{ steps.get-matrix.outputs.test_matrix }} + trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }} + sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }} trial: if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail - needs: trial-calculate-jobs + needs: calculate-test-jobs runs-on: ubuntu-latest strategy: matrix: - job: ${{ fromJson(needs.trial-calculate-jobs.outputs.test_matrix) }} + job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - uses: actions/checkout@v2 @@ -192,45 +193,26 @@ jobs: sytest: if: ${{ !failure() && !cancelled() }} - needs: linting-done + needs: calculate-test-jobs runs-on: ubuntu-latest container: - image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }} + image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }} volumes: - ${{ github.workspace }}:/src env: SYTEST_BRANCH: ${{ github.head_ref }} - POSTGRES: ${{ matrix.postgres && 1}} - MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}} - WORKERS: ${{ matrix.workers && 1 }} - REDIS: ${{ matrix.redis && 1 }} - BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} + POSTGRES: ${{ matrix.job.postgres && 1}} + MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}} + WORKERS: ${{ matrix.job.workers && 1 }} + REDIS: 1 + BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }} TOP: ${{ github.workspace }} strategy: fail-fast: false matrix: include: - - sytest-tag: focal - - - sytest-tag: focal - postgres: postgres - - - sytest-tag: testing - postgres: postgres - - - sytest-tag: focal - postgres: multi-postgres - workers: workers - - - sytest-tag: buster - postgres: multi-postgres - workers: workers - - - sytest-tag: buster - postgres: postgres - workers: workers - redis: redis + job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - uses: actions/checkout@v2 @@ -246,7 +228,7 @@ jobs: uses: actions/upload-artifact@v2 if: ${{ always() }} with: - name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) + name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) path: | /logs/results.tap /logs/**/*.log* From f43a4331739be66b43facce3fad664b52c0f88f2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 5 Sep 2022 11:41:29 +0100 Subject: [PATCH 4/6] Fix typo --- .ci/scripts/calculate_jobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/scripts/calculate_jobs.py b/.ci/scripts/calculate_jobs.py index c99d183f44ca..b1f604eeb0d0 100755 --- a/.ci/scripts/calculate_jobs.py +++ b/.ci/scripts/calculate_jobs.py @@ -99,7 +99,7 @@ }, { "sytest-tag": "focal", - "postgres": "mulit-postgres", + "postgres": "multi-postgres", "workers": "workers", }, ] @@ -113,7 +113,7 @@ }, { "sytest-tag": "buster", - "postgres": "mulit-postgres", + "postgres": "multi-postgres", "workers": "workers", }, ] From 9f0fbd5fee97bc31552efd120675c5d31fe1039e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 5 Sep 2022 11:50:31 +0100 Subject: [PATCH 5/6] Fix up --- .github/workflows/tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d4a67895ddb4..b14a1a06c49c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -211,8 +211,7 @@ jobs: strategy: fail-fast: false matrix: - include: - job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} + job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - uses: actions/checkout@v2 From 1f8166973dc1c87f517c788415c5e1e3dd7c4c2a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 5 Sep 2022 12:06:52 +0100 Subject: [PATCH 6/6] Fixup --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b14a1a06c49c..3ce4ffb0369e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -81,7 +81,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - id: get-matrix - run: .ci/scripts/calculate_trial_jobs.py + run: .ci/scripts/calculate_jobs.py outputs: trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }} sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }}