diff --git a/.github/workflows/pr-subscriber-wait.py b/.github/workflows/pr-subscriber-wait.py
new file mode 100644
index 0000000000000..633f78c147707
--- /dev/null
+++ b/.github/workflows/pr-subscriber-wait.py
@@ -0,0 +1,27 @@
+import github
+import os
+import sys
+import time
+
+
+def needs_to_wait(repo):
+    workflow_name = os.environ.get("GITHUB_WORKFLOW")
+    run_number = os.environ.get("GITHUB_RUN_NUMBER")
+    print("Workflow Name:", workflow_name, "Run Number:", run_number)
+    for status in ["in_progress", "queued"]:
+        for workflow in repo.get_workflow_runs(status=status):
+            print("Looking at ", workflow.name, "#", workflow.run_number)
+            if workflow.name != workflow_name:
+                continue
+            if workflow.run_number < int(run_number):
+                print("Workflow {} still {} ".format(workflow.run_number, status))
+                return True
+    return False
+
+
+repo_name = os.environ.get("GITHUB_REPOSITORY")
+token = os.environ.get("GITHUB_TOKEN")
+gh = github.Github(token)
+repo = gh.get_repo(repo_name)
+while needs_to_wait(repo):
+    time.sleep(30)
diff --git a/.github/workflows/pr-subscriber.yml b/.github/workflows/pr-subscriber.yml
index e31b5c448ee15..1fc3bfed3a66b 100644
--- a/.github/workflows/pr-subscriber.yml
+++ b/.github/workflows/pr-subscriber.yml
@@ -7,16 +7,9 @@ on:
       - completed
 
 permissions:
+  actions: read
   contents: read
 
-concurrency:
-  # Ideally, we would use the PR number in the concurrency group, but we don't
-  # have access to it here.  We need to ensure only one job is running for
-  # each PR at a time, because there is a potential race condition when
-  # updating the issue comment.
-  group: "PR Subscriber"
-  cancel-in-progress: false
-
 jobs:
   auto-subscribe:
     runs-on: ubuntu-latest
@@ -25,6 +18,23 @@ jobs:
       github.event.workflow_run.event == 'pull_request' &&
       github.event.workflow_run.conclusion == 'success'
     steps:
+      - name: Setup Automation Script
+        run: |
+          curl -O -L https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/github-automation.py
+          curl -O -L https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/requirements.txt
+          curl -O -L https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/.github/workflows/pr-subscriber-wait.py
+          chmod a+x github-automation.py
+          pip install -r requirements.txt
+
+      - name: 'Wait for other actions'
+        # We can't use the concurrency tag for these jobs, because it will
+        # cancel pending jobs if another job is running.
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 pr-subscriber-wait.py
+
+
       # From: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
       # Updated version here: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#using-data-from-the-triggering-workflow
       - name: 'Download artifact'
@@ -50,13 +60,6 @@ jobs:
 
       - run: unzip pr.zip
 
-      - name: Setup Automation Script
-        run: |
-          curl -O -L https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/github-automation.py
-          curl -O -L https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/requirements.txt
-          chmod a+x github-automation.py
-          pip install -r requirements.txt
-
       - name: Update watchers
         # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
         run: |
diff --git a/llvm/utils/git/requirements.txt b/llvm/utils/git/requirements.txt
index c83b2ae5df2a5..bed449e6bf9f0 100644
--- a/llvm/utils/git/requirements.txt
+++ b/llvm/utils/git/requirements.txt
@@ -9,9 +9,13 @@ certifi==2023.7.22
     #   -r requirements.txt.in
     #   requests
 cffi==1.15.1
-    # via pynacl
+    # via
+    #   cryptography
+    #   pynacl
 charset-normalizer==2.1.1
     # via requests
+cryptography==41.0.3
+    # via pyjwt
 deprecated==1.2.13
     # via pygithub
 gitdb==4.0.9
@@ -22,9 +26,9 @@ idna==3.4
     # via requests
 pycparser==2.21
     # via cffi
-pygithub==1.55
+pygithub==1.59.1
     # via -r requirements.txt.in
-pyjwt==2.5.0
+pyjwt[crypto]==2.5.0
     # via pygithub
 pynacl==1.5.0
     # via pygithub
@@ -32,6 +36,8 @@ requests==2.28.1
     # via pygithub
 smmap==5.0.0
     # via gitdb
+types-cryptography==3.3.23.2
+    # via pyjwt
 urllib3==1.26.12
     # via requests
 wrapt==1.14.1
diff --git a/llvm/utils/git/requirements.txt.in b/llvm/utils/git/requirements.txt.in
index ee45d2349ea1f..a8bda5c811405 100644
--- a/llvm/utils/git/requirements.txt.in
+++ b/llvm/utils/git/requirements.txt.in
@@ -4,5 +4,5 @@
 # pip-compile -o requirements.txt  requirements.txt.in
 
 certifi>=2023.7.22  # https://security.snyk.io/vuln/SNYK-PYTHON-CERTIFI-5805047
-PyGithub
+PyGithub==1.59.1 # For WorkflowRun.name
 GitPython>=3.1.32  # https://security.snyk.io/vuln/SNYK-PYTHON-GITPYTHON-5840584