From 0ddae07a1910b3953e3475dd2b1d74cb533bfb76 Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 13 Jun 2022 11:48:36 -0500 Subject: [PATCH 1/5] Adds zombie removal tool --- .../terminate-zombie-build-instances.yml | 6 ++ tools/bin/gh_action_zombie_killer | 72 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100755 tools/bin/gh_action_zombie_killer diff --git a/.github/workflows/terminate-zombie-build-instances.yml b/.github/workflows/terminate-zombie-build-instances.yml index 53e214727a54e..f9e1622d8a6ab 100644 --- a/.github/workflows/terminate-zombie-build-instances.yml +++ b/.github/workflows/terminate-zombie-build-instances.yml @@ -34,3 +34,9 @@ jobs: # See https://docs.aws.amazon.com/cli/latest/reference/ec2/terminate-instances.html for terminate command. echo $to_terminate | jq '.[] | .InstanceId' | xargs --no-run-if-empty --max-args=1 aws ec2 terminate-instances --instance-ids + + steps: + - shell: List and Terminate GH actions runners Older than One Day + env: + GITHUB_PAT: ${{ secrets.OCTAVIA_PAT }} + run: ./tools/bin/gh_action_zombie_killer diff --git a/tools/bin/gh_action_zombie_killer b/tools/bin/gh_action_zombie_killer new file mode 100755 index 0000000000000..9d45ba1269dff --- /dev/null +++ b/tools/bin/gh_action_zombie_killer @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +# ------------- Import some defaults for the shell + +# Source shell defaults +# $0 is the currently running program (this file) +this_file_directory=$(dirname $0) +relative_path_to_defaults=$this_file_directory/../shell_defaults + +# if a file exists there, source it. otherwise complain +if test -f $relative_path_to_defaults; then + # source and '.' are the same program + source $relative_path_to_defaults +else + echo -e "\033[31m\nFAILED TO SOURCE TEST RUNNING OPTIONS.\033[39m" + echo -e "\033[31mTried $relative_path_to_defaults\033[39m" + exit 1 +fi + +echo "To run locally use GITHUB_PAT=\$YOUR_PAT_HERE before running" +token=$GITHUB_PAT +org=airbytehq +# FUN POSIX fact, every string is an array! +repo_list="airbyte airbyte-cloud" + + +for repo in $repo_list; do + # Start the while loop + runner_for_page_count=1 + page_count=0 + all_runner_ids="" + # keep paging through until we find them all + while test $runner_for_page_count -gt 0; do + page_count=$(($page_count+1)) + set +o xtrace + runner_response=$(curl \ + --silent \ + --header "Accept: application/vnd.github.v3+json" \ + --header "Authorization: token $token" \ + --request GET https://api.github.com/repos/$org/$repo/actions/runners?page=$page_count&per_page=100) + runner_response_wc=$(echo $runner_response | wc -w) + # For auth errors + if test $runner_response_wc -lt 100; then + echo -e "$blue_text""\$runner_response is \n\n$runner_response\n\n""$default_text" + fi + + runner_ids_for_page=$(echo $runner_response | \ + jq '.runners[] | select(.status!="online") | .id') + + runner_for_page_count=$(echo $runner_ids_for_page | wc -w) + echo -e "$blue_text""jq returned $runner_for_page_count runners for page $page_count""$default_text" + all_runner_ids=$runner_ids_for_page\n$all_runner_ids + all_runner_ids_count=$(echo $all_runner_ids | wc -w) + echo -e "$blue_text""Total count is now $all_runner_ids_count""$default_text" + done + + echo -e "$blue_text""Total ids returned: $all_runner_ids_count""$default_text" + cursor=0 + for this_runner in $all_runner_ids; do + cursor=$(($cursor+1)) + echo -e "$blue_text""Removing $cursor / $all_runner_ids_count""$default_text" + curl \ + --silent \ + --request DELETE \ + --header "Accept: application/vnd.github.v3+json" \ + --header "Authorization: token $token" \ + https://api.github.com/orgs/$org/actions/runners/$this_runner && \ + echo -e "$blue_text""OK ID $this_runner""$default_text" || \ + echo -e "$red_text""FAIL! ID $this_runner""$default_text" + done + +done From e3c849894d5ae70fe03f720b8dbce763eef0eb68 Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 13 Jun 2022 13:04:25 -0500 Subject: [PATCH 2/5] Corrects endpoint adds comments --- tools/bin/gh_action_zombie_killer | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/bin/gh_action_zombie_killer b/tools/bin/gh_action_zombie_killer index 9d45ba1269dff..c748c3f4aaa65 100755 --- a/tools/bin/gh_action_zombie_killer +++ b/tools/bin/gh_action_zombie_killer @@ -25,7 +25,7 @@ repo_list="airbyte airbyte-cloud" for repo in $repo_list; do - # Start the while loop + # Start the while loop to check for all runners runner_for_page_count=1 page_count=0 all_runner_ids="" @@ -55,6 +55,7 @@ for repo in $repo_list; do done echo -e "$blue_text""Total ids returned: $all_runner_ids_count""$default_text" + # DELETE THEM ALL! cursor=0 for this_runner in $all_runner_ids; do cursor=$(($cursor+1)) @@ -64,7 +65,7 @@ for repo in $repo_list; do --request DELETE \ --header "Accept: application/vnd.github.v3+json" \ --header "Authorization: token $token" \ - https://api.github.com/orgs/$org/actions/runners/$this_runner && \ + https://api.github.com/repos/$org/$repo/actions/runners/$this_runner && \ echo -e "$blue_text""OK ID $this_runner""$default_text" || \ echo -e "$red_text""FAIL! ID $this_runner""$default_text" done From 375e66bc435c97809b3251b82084deb09ca89da7 Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 13 Jun 2022 13:15:33 -0500 Subject: [PATCH 3/5] Adds API links --- tools/bin/gh_action_zombie_killer | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/bin/gh_action_zombie_killer b/tools/bin/gh_action_zombie_killer index c748c3f4aaa65..6f04454a597ce 100755 --- a/tools/bin/gh_action_zombie_killer +++ b/tools/bin/gh_action_zombie_killer @@ -33,6 +33,8 @@ for repo in $repo_list; do while test $runner_for_page_count -gt 0; do page_count=$(($page_count+1)) set +o xtrace + # API for endpoint: + # https://docs.github.com/en/rest/actions/self-hosted-runners#list-self-hosted-runners-for-a-repository runner_response=$(curl \ --silent \ --header "Accept: application/vnd.github.v3+json" \ @@ -60,6 +62,8 @@ for repo in $repo_list; do for this_runner in $all_runner_ids; do cursor=$(($cursor+1)) echo -e "$blue_text""Removing $cursor / $all_runner_ids_count""$default_text" + # API for endpoint: + # https://docs.github.com/en/rest/actions/self-hosted-runners#delete-a-self-hosted-runner-from-a-repository curl \ --silent \ --request DELETE \ From cf3b03047207d2bb6f7db04c38df8246eb8e5b89 Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 13 Jun 2022 13:22:21 -0500 Subject: [PATCH 4/5] Changes search logic, fixes escape character --- tools/bin/gh_action_zombie_killer | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/bin/gh_action_zombie_killer b/tools/bin/gh_action_zombie_killer index 6f04454a597ce..815f65d09cba4 100755 --- a/tools/bin/gh_action_zombie_killer +++ b/tools/bin/gh_action_zombie_killer @@ -41,17 +41,17 @@ for repo in $repo_list; do --header "Authorization: token $token" \ --request GET https://api.github.com/repos/$org/$repo/actions/runners?page=$page_count&per_page=100) runner_response_wc=$(echo $runner_response | wc -w) - # For auth errors + # For auth errors because auth errors are short if test $runner_response_wc -lt 100; then echo -e "$blue_text""\$runner_response is \n\n$runner_response\n\n""$default_text" fi runner_ids_for_page=$(echo $runner_response | \ - jq '.runners[] | select(.status!="online") | .id') + jq '.runners[] | select(.status=="offline") | .id') runner_for_page_count=$(echo $runner_ids_for_page | wc -w) echo -e "$blue_text""jq returned $runner_for_page_count runners for page $page_count""$default_text" - all_runner_ids=$runner_ids_for_page\n$all_runner_ids + all_runner_ids="$runner_ids_for_page $all_runner_ids" all_runner_ids_count=$(echo $all_runner_ids | wc -w) echo -e "$blue_text""Total count is now $all_runner_ids_count""$default_text" done From 536e7dcfdc289d0c6e3e019e5b571eab9f2dcbd3 Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 13 Jun 2022 13:26:56 -0500 Subject: [PATCH 5/5] Corrects help text --- .github/workflows/terminate-zombie-build-instances.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/terminate-zombie-build-instances.yml b/.github/workflows/terminate-zombie-build-instances.yml index f9e1622d8a6ab..2fcdc4e5120f6 100644 --- a/.github/workflows/terminate-zombie-build-instances.yml +++ b/.github/workflows/terminate-zombie-build-instances.yml @@ -36,7 +36,7 @@ jobs: echo $to_terminate | jq '.[] | .InstanceId' | xargs --no-run-if-empty --max-args=1 aws ec2 terminate-instances --instance-ids steps: - - shell: List and Terminate GH actions runners Older than One Day + - shell: List and Terminate GH actions in status 'offline' env: GITHUB_PAT: ${{ secrets.OCTAVIA_PAT }} run: ./tools/bin/gh_action_zombie_killer