From 365b2841f87c939988101995a2c9cf5a4cad7dd5 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 12 Sep 2023 17:36:37 -0700 Subject: [PATCH 1/3] workflows/pr-subscriber: Use our own custom concurrency implementation The builtin concurrency functionality for the workflows will cancel a pending job if there is another job from the same workflow running. For the pr-subscriber job, this means that if multiple labels are added at the same time, then some of the pr-subscriber jobs will be cancelled and the PR will not have all the necessary mentions. --- .github/workflows/pr-subscriber.yml | 54 +++++++++++++++++++++-------- llvm/utils/git/requirements.txt | 12 +++++-- llvm/utils/git/requirements.txt.in | 2 +- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/.github/workflows/pr-subscriber.yml b/.github/workflows/pr-subscriber.yml index e31b5c448ee15..c96de150cf036 100644 --- a/.github/workflows/pr-subscriber.yml +++ b/.github/workflows/pr-subscriber.yml @@ -7,16 +7,9 @@ on: - completed permissions: + actions: read contents: read -concurrency: - # Ideally, we would use the PR number in the concurrency group, but we don't - # have access to it here. We need to ensure only one job is running for - # each PR at a time, because there is a potential race condition when - # updating the issue comment. - group: "PR Subscriber" - cancel-in-progress: false - jobs: auto-subscribe: runs-on: ubuntu-latest @@ -25,6 +18,44 @@ jobs: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' steps: + - name: Setup Automation Script + run: | + curl -O -L https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/github-automation.py + curl -O -L https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/requirements.txt + chmod a+x github-automation.py + pip install -r requirements.txt + + - name: 'Wait for other actions' + # We can't use the concurrency tag for these jobs, because it will + # cancel pending jobs if another job is running. + shell: python + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + import github + import os + import sys + import time + + def needs_to_wait(repo): + workflow_name = os.environ.get("GITHUB_WORKFLOW") + run_number = os.environ.get("GITHUB_RUN_NUMBER") + for status in ["in_progress", "queued"]: + for workflow in repo.get_workflow_runs(status = status): + if workflow.name != workflow_name: + continue + if workflow.run_number < int(run_number): + print("Workflow {} still {} ".format(workflow.run_number, status)) + return True + return False + + repo_name = os.environ.get("GITHUB_REPOSITORY") + token = os.environ.get("GITHUB_TOKEN") + gh = github.Github(token) + repo = gh.get_repo(repo_name) + while needs_to_wait(repo): + time.sleep(30) + # From: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ # Updated version here: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#using-data-from-the-triggering-workflow - name: 'Download artifact' @@ -50,13 +81,6 @@ jobs: - run: unzip pr.zip - - name: Setup Automation Script - run: | - curl -O -L https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/github-automation.py - curl -O -L https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/requirements.txt - chmod a+x github-automation.py - pip install -r requirements.txt - - name: Update watchers # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable run: | diff --git a/llvm/utils/git/requirements.txt b/llvm/utils/git/requirements.txt index c83b2ae5df2a5..bed449e6bf9f0 100644 --- a/llvm/utils/git/requirements.txt +++ b/llvm/utils/git/requirements.txt @@ -9,9 +9,13 @@ certifi==2023.7.22 # -r requirements.txt.in # requests cffi==1.15.1 - # via pynacl + # via + # cryptography + # pynacl charset-normalizer==2.1.1 # via requests +cryptography==41.0.3 + # via pyjwt deprecated==1.2.13 # via pygithub gitdb==4.0.9 @@ -22,9 +26,9 @@ idna==3.4 # via requests pycparser==2.21 # via cffi -pygithub==1.55 +pygithub==1.59.1 # via -r requirements.txt.in -pyjwt==2.5.0 +pyjwt[crypto]==2.5.0 # via pygithub pynacl==1.5.0 # via pygithub @@ -32,6 +36,8 @@ requests==2.28.1 # via pygithub smmap==5.0.0 # via gitdb +types-cryptography==3.3.23.2 + # via pyjwt urllib3==1.26.12 # via requests wrapt==1.14.1 diff --git a/llvm/utils/git/requirements.txt.in b/llvm/utils/git/requirements.txt.in index ee45d2349ea1f..a8bda5c811405 100644 --- a/llvm/utils/git/requirements.txt.in +++ b/llvm/utils/git/requirements.txt.in @@ -4,5 +4,5 @@ # pip-compile -o requirements.txt requirements.txt.in certifi>=2023.7.22 # https://security.snyk.io/vuln/SNYK-PYTHON-CERTIFI-5805047 -PyGithub +PyGithub==1.59.1 # For WorkflowRun.name GitPython>=3.1.32 # https://security.snyk.io/vuln/SNYK-PYTHON-GITPYTHON-5840584 From 0ca05fb8cdfcd1d8b5320064e7baad6700a6873d Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Wed, 13 Sep 2023 12:30:04 -0700 Subject: [PATCH 2/3] Move python code into an external file --- .github/workflows/pr-subscriber-wait.py | 26 +++++++++++++++++++++++++ .github/workflows/pr-subscriber.yml | 25 ++---------------------- 2 files changed, 28 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/pr-subscriber-wait.py diff --git a/.github/workflows/pr-subscriber-wait.py b/.github/workflows/pr-subscriber-wait.py new file mode 100644 index 0000000000000..8218fc89622d5 --- /dev/null +++ b/.github/workflows/pr-subscriber-wait.py @@ -0,0 +1,26 @@ +import github +import os +import sys +import time + +def needs_to_wait(repo): + + workflow_name = os.environ.get("GITHUB_WORKFLOW") + run_number = os.environ.get("GITHUB_RUN_NUMBER") + print("Workflow Name:", workflow_name, "Run Number:", run_number) + for status in ["in_progress", "queued"]: + for workflow in repo.get_workflow_runs(status = status): + print("Looking at ", workflow.name, "#", workflow.run_number) + if workflow.name != workflow_name: + continue + if workflow.run_number < int(run_number): + print("Workflow {} still {} ".format(workflow.run_number, status)) + return True + return False + +repo_name = os.environ.get("GITHUB_REPOSITORY") +token = os.environ.get("GITHUB_TOKEN") +gh = github.Github(token) +repo = gh.get_repo(repo_name) +while needs_to_wait(repo): + time.sleep(30) diff --git a/.github/workflows/pr-subscriber.yml b/.github/workflows/pr-subscriber.yml index c96de150cf036..1fc3bfed3a66b 100644 --- a/.github/workflows/pr-subscriber.yml +++ b/.github/workflows/pr-subscriber.yml @@ -22,39 +22,18 @@ jobs: run: | curl -O -L https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/github-automation.py curl -O -L https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/requirements.txt + curl -O -L https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/.github/workflows/pr-subscriber-wait.py chmod a+x github-automation.py pip install -r requirements.txt - name: 'Wait for other actions' # We can't use the concurrency tag for these jobs, because it will # cancel pending jobs if another job is running. - shell: python env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - import github - import os - import sys - import time + python3 pr-subscriber-wait.py - def needs_to_wait(repo): - workflow_name = os.environ.get("GITHUB_WORKFLOW") - run_number = os.environ.get("GITHUB_RUN_NUMBER") - for status in ["in_progress", "queued"]: - for workflow in repo.get_workflow_runs(status = status): - if workflow.name != workflow_name: - continue - if workflow.run_number < int(run_number): - print("Workflow {} still {} ".format(workflow.run_number, status)) - return True - return False - - repo_name = os.environ.get("GITHUB_REPOSITORY") - token = os.environ.get("GITHUB_TOKEN") - gh = github.Github(token) - repo = gh.get_repo(repo_name) - while needs_to_wait(repo): - time.sleep(30) # From: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ # Updated version here: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#using-data-from-the-triggering-workflow From ec8bcec0ea66288d48fbf230094eabb6b0d20d9f Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Wed, 13 Sep 2023 12:43:25 -0700 Subject: [PATCH 3/3] Fix code --- .github/workflows/pr-subscriber-wait.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-subscriber-wait.py b/.github/workflows/pr-subscriber-wait.py index 8218fc89622d5..633f78c147707 100644 --- a/.github/workflows/pr-subscriber-wait.py +++ b/.github/workflows/pr-subscriber-wait.py @@ -3,13 +3,13 @@ import sys import time -def needs_to_wait(repo): +def needs_to_wait(repo): workflow_name = os.environ.get("GITHUB_WORKFLOW") run_number = os.environ.get("GITHUB_RUN_NUMBER") print("Workflow Name:", workflow_name, "Run Number:", run_number) for status in ["in_progress", "queued"]: - for workflow in repo.get_workflow_runs(status = status): + for workflow in repo.get_workflow_runs(status=status): print("Looking at ", workflow.name, "#", workflow.run_number) if workflow.name != workflow_name: continue @@ -18,6 +18,7 @@ def needs_to_wait(repo): return True return False + repo_name = os.environ.get("GITHUB_REPOSITORY") token = os.environ.get("GITHUB_TOKEN") gh = github.Github(token)