From 6845db040a049c53b2933b26882dc04eeaf061e2 Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 18 Apr 2022 10:12:18 -0500 Subject: [PATCH 01/11] WIP --- .github/workflows/documentation.yml | 20 ++++++++++---------- tools/bin/deploy_docusaurus | 26 ++++++++++++++++++++------ 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 7dec4e5e919c9..346761bb07f9c 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -2,7 +2,7 @@ name: compile-docusaurus-static-assets on: push: - branches: [master, toph_an_endless_persuit_of_docs] + branches: [master, toph_docs_pipeline_fixes_bugs_n_gitbook] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -26,13 +26,13 @@ jobs: node-version: '16.13.0' cache: 'yarn' cache-dependency-path: docusaurus - # disabled for green build. more testing needed - # # Build Docusaurus website - # - name: Check for docusaurus changes - # run: ./tools/bin/check_docusaurus_build_changes - # # Install and build Docusaurus website - # - name: Deploy docs to production (it's weird) - # run: ./tools/bin/deploy_docusaurus - # env: - # GITHUB_TOKEN: ${{ secrets.OCTAVIA_PAT }} + disabled for green build. more testing needed + # Build Docusaurus website + - name: Check for docusaurus changes + run: ./tools/bin/check_docusaurus_build_changes + # Install and build Docusaurus website + - name: Deploy docs to production (it's weird) + run: ./tools/bin/deploy_docusaurus + env: + GITHUB_TOKEN: ${{ secrets.OCTAVIA_PAT }} diff --git a/tools/bin/deploy_docusaurus b/tools/bin/deploy_docusaurus index 3d96bc791bcb2..3b54def109852 100755 --- a/tools/bin/deploy_docusaurus +++ b/tools/bin/deploy_docusaurus @@ -59,9 +59,15 @@ yarn install # generate static content yarn build -# context https://v1.docusaurus.io/docs/en/publishing#using-github-pages -# write a prod website to airbytehq/airbyte gh_pages branch -GIT_USER=octavia-squidington-iii yarn run deploy +# test -t checks for a tty (interactive shell) +if test -t; then + # context https://v1.docusaurus.io/docs/en/publishing#using-github-pages + # write a prod website to airbytehq/airbyte gh_pages branch + GIT_USER=octavia-squidington-iii GIT_TOKEN=$GITHUB_TOKEN yarn run deploy +else + yarn run deploy +fi + @@ -71,6 +77,9 @@ pwd # We should be here but we are playing with fire git fetch +# We force push gh-pages. Local copies confuse things +# This line looks overly complex to enure we don't error if that branch doesn't exist +git branch -D gh-pages 2> /dev/null || true # checkout the branch tracking it's remote git switch gh-pages @@ -94,11 +103,16 @@ git commit --message "Adds CNAME to deploy for $revision" # non functional. for debugging git branch -# note that this is NOT airbyte repo -git push --force https://$GITHUB_TOKEN@github.com/airbytehq/airbytehq.github.io.git +# Check istty for local/remote deploys (we expect cloud to be non-interactive) +if test -t; then + # note that this is NOT airbyte repo + git push --force https://$GITHUB_TOKEN@github.com/airbytehq/airbytehq.github.io.git +else + git push --force https://git@github.com/airbytehq/airbytehq.github.io.git +fi + # Want to push from your own computer? uncomment this line and comment out the push above -# git push --force https://git@github.com/airbytehq/airbytehq.github.io.git set +o xtrace echo -e "$blue_text""Script exiting 0 GREAT SUCCESS!!!?""$default_text" From 40bdc3770a099cab764a75d45d85d278a8c16eeb Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 18 Apr 2022 15:59:29 -0500 Subject: [PATCH 02/11] WIP2 --- tools/bin/deploy_docusaurus | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/tools/bin/deploy_docusaurus b/tools/bin/deploy_docusaurus index 3b54def109852..9e2ee7264d1dc 100755 --- a/tools/bin/deploy_docusaurus +++ b/tools/bin/deploy_docusaurus @@ -31,6 +31,7 @@ pwd # Yarn check (which is commonly used to check for program existance) +# -s/--silent doesn't exist in cloud's Ubuntu if ! which yarn > /dev/null; then echo -e "$red_text""yarn not found HALP!!\n\n""$default_text" exit 1 @@ -38,18 +39,26 @@ fi set -o xtrace -# touch is non-destructive and the cloud doesn't have this file -touch $HOME/.gitconfig - -# Set email and user name -if ! git config --list --global | grep --silent user.email; then +# local check for changes. Fail here early instead of 40 seconds from now +if ! test -z "$(git status --short)"; then set +o xtrace - echo -e "$blue_text""github email not found adding Octavia's""$default_text" - set -o xtrace - git config --global user.email="octavia-squidington-iii@users.noreply.github.com" - git config --global user.name="octavia-squidington-iii" + echo -e "$red_text""You have uncommitted changes!!!\n\n""$default_text" + echo -e "$red_text""Commit and try again""$default_text" + exit 1 fi +# touch is non-destructive and the cloud doesn't have this file +# touch $HOME/.gitconfig + +# # Set email and user name +# if ! git config --list --global | grep --silent user.email; then +# set +o xtrace +# echo -e "$blue_text""github email not found adding Octavia's""$default_text" +# set -o xtrace +# git config --global user.email="octavia-squidington-iii@users.noreply.github.com" +# git config --global user.name="octavia-squidington-iii" +# fi + cd docusaurus pwd @@ -59,8 +68,9 @@ yarn install # generate static content yarn build -# test -t checks for a tty (interactive shell) -if test -t; then +# Check tty for local/remote deploys (we expect cloud to be non-interactive) +# results like /dev/ttys000 || not a tty +if test "$(tty)" == "not a tty"; then # context https://v1.docusaurus.io/docs/en/publishing#using-github-pages # write a prod website to airbytehq/airbyte gh_pages branch GIT_USER=octavia-squidington-iii GIT_TOKEN=$GITHUB_TOKEN yarn run deploy @@ -81,7 +91,7 @@ git fetch # This line looks overly complex to enure we don't error if that branch doesn't exist git branch -D gh-pages 2> /dev/null || true # checkout the branch tracking it's remote -git switch gh-pages +git switch --track origin/gh-pages # For tracking in the commit message revision=$(git rev-parse --short HEAD) @@ -103,8 +113,7 @@ git commit --message "Adds CNAME to deploy for $revision" # non functional. for debugging git branch -# Check istty for local/remote deploys (we expect cloud to be non-interactive) -if test -t; then +if test "$(tty)" == "not a tty"; then # note that this is NOT airbyte repo git push --force https://$GITHUB_TOKEN@github.com/airbytehq/airbytehq.github.io.git else From 3d193b0c96445fe602082fb0111f9a29b13af69d Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 18 Apr 2022 16:03:23 -0500 Subject: [PATCH 03/11] WIP3 --- tools/bin/deploy_docusaurus | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/tools/bin/deploy_docusaurus b/tools/bin/deploy_docusaurus index 9e2ee7264d1dc..af00214b9b737 100755 --- a/tools/bin/deploy_docusaurus +++ b/tools/bin/deploy_docusaurus @@ -47,18 +47,6 @@ if ! test -z "$(git status --short)"; then exit 1 fi -# touch is non-destructive and the cloud doesn't have this file -# touch $HOME/.gitconfig - -# # Set email and user name -# if ! git config --list --global | grep --silent user.email; then -# set +o xtrace -# echo -e "$blue_text""github email not found adding Octavia's""$default_text" -# set -o xtrace -# git config --global user.email="octavia-squidington-iii@users.noreply.github.com" -# git config --global user.name="octavia-squidington-iii" -# fi - cd docusaurus pwd @@ -117,7 +105,7 @@ if test "$(tty)" == "not a tty"; then # note that this is NOT airbyte repo git push --force https://$GITHUB_TOKEN@github.com/airbytehq/airbytehq.github.io.git else - git push --force https://git@github.com/airbytehq/airbytehq.github.io.git + git push --force git@github.com:airbytehq/airbytehq.github.io.git fi From 590f59f20bf9d4bd693fb38bb35c9fa82f6f002a Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 18 Apr 2022 16:21:01 -0500 Subject: [PATCH 04/11] Remove comment/better comments --- .github/workflows/documentation.yml | 1 - tools/bin/deploy_docusaurus | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 346761bb07f9c..43e3a18d84e63 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -26,7 +26,6 @@ jobs: node-version: '16.13.0' cache: 'yarn' cache-dependency-path: docusaurus - disabled for green build. more testing needed # Build Docusaurus website - name: Check for docusaurus changes run: ./tools/bin/check_docusaurus_build_changes diff --git a/tools/bin/deploy_docusaurus b/tools/bin/deploy_docusaurus index af00214b9b737..c9792bc53a686 100755 --- a/tools/bin/deploy_docusaurus +++ b/tools/bin/deploy_docusaurus @@ -76,7 +76,7 @@ pwd # We should be here but we are playing with fire git fetch # We force push gh-pages. Local copies confuse things -# This line looks overly complex to enure we don't error if that branch doesn't exist +# This line uses || true to enure we don't error if that branch doesn't exist git branch -D gh-pages 2> /dev/null || true # checkout the branch tracking it's remote git switch --track origin/gh-pages @@ -108,6 +108,9 @@ else git push --force git@github.com:airbytehq/airbytehq.github.io.git fi +# Let's leave the tire fire of force pushes +git checkout - + # Want to push from your own computer? uncomment this line and comment out the push above From 87293ee52416d4ba3e4236fc6572c83bfa6c5a6e Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 18 Apr 2022 16:35:39 -0500 Subject: [PATCH 05/11] Remote deployment updates --- .github/workflows/documentation.yml | 2 +- tools/bin/check_docusaurus_build_changes | 4 ++-- tools/bin/deploy_docusaurus | 7 ++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 43e3a18d84e63..28469501bd4fe 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -27,7 +27,7 @@ jobs: cache: 'yarn' cache-dependency-path: docusaurus # Build Docusaurus website - - name: Check for docusaurus changes + - name: Check for docusaurus changes not committed run: ./tools/bin/check_docusaurus_build_changes # Install and build Docusaurus website - name: Deploy docs to production (it's weird) diff --git a/tools/bin/check_docusaurus_build_changes b/tools/bin/check_docusaurus_build_changes index 22f5b6f0d0b69..457c709e0d4b5 100755 --- a/tools/bin/check_docusaurus_build_changes +++ b/tools/bin/check_docusaurus_build_changes @@ -43,8 +43,8 @@ set -o errexit set +o xtrace if test $clean -eq 0; then - echo -e "$red_text""\n\n\nDocusaurs has no changes to commit!""$default_text" - echo -e "$red_text""Generated documentation should be as local testing""$default_text" + echo -e "$blue_text""\n\n\nDocusaurs has no changes to commit!""$default_text" + echo -e "$blue_text""Generated documentation should be as local testing""$default_text" else echo -e "$red_text""\n\n\ndocusaurs build resulted in changes from this commit.""$default_text" diff --git a/tools/bin/deploy_docusaurus b/tools/bin/deploy_docusaurus index c9792bc53a686..a937e7d3ccb65 100755 --- a/tools/bin/deploy_docusaurus +++ b/tools/bin/deploy_docusaurus @@ -102,8 +102,13 @@ git commit --message "Adds CNAME to deploy for $revision" git branch if test "$(tty)" == "not a tty"; then + set +o xtrace + echo -e "$blue_text""github email not found adding Octavia's""$default_text" + set -o xtrace + git config --global user.email="octavia-squidington-iii@users.noreply.github.com" + git config --global user.name="octavia-squidington-iii" # note that this is NOT airbyte repo - git push --force https://$GITHUB_TOKEN@github.com/airbytehq/airbytehq.github.io.git + git push --force https://octavia-squidington-iii:$GITHUB_TOKEN@github.com/airbytehq/airbytehq.github.io.git else git push --force git@github.com:airbytehq/airbytehq.github.io.git fi From 38717be6fa98c80cc55e3ab1dcbe9045ccdfc70c Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 18 Apr 2022 16:52:35 -0500 Subject: [PATCH 06/11] Adds GH user for yarn deploy --- .github/workflows/documentation.yml | 2 +- tools/bin/deploy_docusaurus | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 28469501bd4fe..84eebfa269d09 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -9,7 +9,7 @@ on: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - verify-docusaurus-static-files: + Deploy Docusaurus to docs.airbyte.io: runs-on: ubuntu-latest steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it diff --git a/tools/bin/deploy_docusaurus b/tools/bin/deploy_docusaurus index a937e7d3ccb65..f0fa11a857e0c 100755 --- a/tools/bin/deploy_docusaurus +++ b/tools/bin/deploy_docusaurus @@ -59,6 +59,11 @@ yarn build # Check tty for local/remote deploys (we expect cloud to be non-interactive) # results like /dev/ttys000 || not a tty if test "$(tty)" == "not a tty"; then + set +o xtrace + echo -e "$blue_text""github email not found adding Octavia's""$default_text" + set -o xtrace + git config user.email="octavia-squidington-iii@users.noreply.github.com" + git config user.name="octavia-squidington-iii" # context https://v1.docusaurus.io/docs/en/publishing#using-github-pages # write a prod website to airbytehq/airbyte gh_pages branch GIT_USER=octavia-squidington-iii GIT_TOKEN=$GITHUB_TOKEN yarn run deploy @@ -102,11 +107,6 @@ git commit --message "Adds CNAME to deploy for $revision" git branch if test "$(tty)" == "not a tty"; then - set +o xtrace - echo -e "$blue_text""github email not found adding Octavia's""$default_text" - set -o xtrace - git config --global user.email="octavia-squidington-iii@users.noreply.github.com" - git config --global user.name="octavia-squidington-iii" # note that this is NOT airbyte repo git push --force https://octavia-squidington-iii:$GITHUB_TOKEN@github.com/airbytehq/airbytehq.github.io.git else From f9d00766993f9af34ae2dc75909f43b2ff43e4f9 Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 18 Apr 2022 16:59:56 -0500 Subject: [PATCH 07/11] WIP 6 --- tools/bin/deploy_docusaurus | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/bin/deploy_docusaurus b/tools/bin/deploy_docusaurus index f0fa11a857e0c..96ac8031870a4 100755 --- a/tools/bin/deploy_docusaurus +++ b/tools/bin/deploy_docusaurus @@ -64,9 +64,10 @@ if test "$(tty)" == "not a tty"; then set -o xtrace git config user.email="octavia-squidington-iii@users.noreply.github.com" git config user.name="octavia-squidington-iii" + echo "machine github.com login octavia-squidington-iii password $GITHUB_TOKEN" > ~/.netrc # context https://v1.docusaurus.io/docs/en/publishing#using-github-pages # write a prod website to airbytehq/airbyte gh_pages branch - GIT_USER=octavia-squidington-iii GIT_TOKEN=$GITHUB_TOKEN yarn run deploy + GIT_USER="octavia-squidington-iii" yarn run publish-gh-pages else yarn run deploy fi From a86c0c03ca4588c5fe1b5cb3de3aa1b542421462 Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 18 Apr 2022 17:05:04 -0500 Subject: [PATCH 08/11] WIP7 --- .github/workflows/documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 84eebfa269d09..fb3d9e67aec79 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -9,7 +9,7 @@ on: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - Deploy Docusaurus to docs.airbyte.io: + Deploy-Docusaurus-to-docs.airbyte.io: runs-on: ubuntu-latest steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it From 23dab3300c9eeb58e03ac67dd6d1a54217d03c38 Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Mon, 18 Apr 2022 17:10:02 -0500 Subject: [PATCH 09/11] WIP8 --- .github/workflows/documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index fb3d9e67aec79..ef39b18bbdb7b 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -9,7 +9,7 @@ on: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - Deploy-Docusaurus-to-docs.airbyte.io: + deploy-docusaurus-to-docs-airbyte-io: runs-on: ubuntu-latest steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it From 17518e5f7513b519b128968541ff3d9c752f57fb Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Tue, 19 Apr 2022 13:17:53 -0500 Subject: [PATCH 10/11] WIP 10 --- tools/bin/deploy_docusaurus | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/deploy_docusaurus b/tools/bin/deploy_docusaurus index 96ac8031870a4..fb4e6b4a013e0 100755 --- a/tools/bin/deploy_docusaurus +++ b/tools/bin/deploy_docusaurus @@ -64,7 +64,7 @@ if test "$(tty)" == "not a tty"; then set -o xtrace git config user.email="octavia-squidington-iii@users.noreply.github.com" git config user.name="octavia-squidington-iii" - echo "machine github.com login octavia-squidington-iii password $GITHUB_TOKEN" > ~/.netrc + echo "machine github.com login octavia-squidington-iii password $GITHUB_TOKEN" > $HOME/.netrc # context https://v1.docusaurus.io/docs/en/publishing#using-github-pages # write a prod website to airbytehq/airbyte gh_pages branch GIT_USER="octavia-squidington-iii" yarn run publish-gh-pages From 75135a40f3ebf52f8c994effe47aa4210c594b8f Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Tue, 19 Apr 2022 13:23:13 -0500 Subject: [PATCH 11/11] Working easy from home not so much in the cloud --- .github/workflows/documentation.yml | 18 +++++++++--------- tools/bin/deploy_docusaurus | 1 + 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index ef39b18bbdb7b..793172b2911a4 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -2,7 +2,7 @@ name: compile-docusaurus-static-assets on: push: - branches: [master, toph_docs_pipeline_fixes_bugs_n_gitbook] + branches: [master] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -26,12 +26,12 @@ jobs: node-version: '16.13.0' cache: 'yarn' cache-dependency-path: docusaurus - # Build Docusaurus website - - name: Check for docusaurus changes not committed - run: ./tools/bin/check_docusaurus_build_changes - # Install and build Docusaurus website - - name: Deploy docs to production (it's weird) - run: ./tools/bin/deploy_docusaurus - env: - GITHUB_TOKEN: ${{ secrets.OCTAVIA_PAT }} + # # Build Docusaurus website + # - name: Check for docusaurus changes not committed + # run: ./tools/bin/check_docusaurus_build_changes + # # Install and build Docusaurus website + # - name: Deploy docs to production (it's weird) + # run: ./tools/bin/deploy_docusaurus + # env: + # GITHUB_TOKEN: ${{ secrets.OCTAVIA_PAT }} diff --git a/tools/bin/deploy_docusaurus b/tools/bin/deploy_docusaurus index fb4e6b4a013e0..2f74f66cac9ce 100755 --- a/tools/bin/deploy_docusaurus +++ b/tools/bin/deploy_docusaurus @@ -67,6 +67,7 @@ if test "$(tty)" == "not a tty"; then echo "machine github.com login octavia-squidington-iii password $GITHUB_TOKEN" > $HOME/.netrc # context https://v1.docusaurus.io/docs/en/publishing#using-github-pages # write a prod website to airbytehq/airbyte gh_pages branch + # NOT ACTUALLY WORKING GIT_USER="octavia-squidington-iii" yarn run publish-gh-pages else yarn run deploy