From 01ae06588d142f387d48bb62100af91a6b7020f8 Mon Sep 17 00:00:00 2001 From: Tom Gulliver Date: Thu, 12 Aug 2021 11:22:26 +0100 Subject: [PATCH 1/5] SP-771 - move some things ot actions --- .github/workflows/build.yml | 157 ++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..9796e30d --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,157 @@ +name: Build + +on: + pull_request: + branches: + - main + +defaults: + run: + shell: bash + +jobs: + build_and_test: + runs-on: "ubuntu-latest" + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: unfor19/install-aws-cli-action@v1 + + - name: Code Artifact login + run: | + ./login_code_artifact.sh -a 288342028542 -t pip + - name: Install requirements + working-directory: ./docs/ci_scripts + run: | + ./install_requirements.sh -d lambda_functions -r dev-requirements.txt + + - name: Run Flask8 + run: | + flake8 --ignore Q000 lambda_functions + - name: Test Coverage + run: | + python -m pytest lambda_functions/v1/tests --cov=lambda_functions/v1/functions/lpa/app/api --cov-fail-under=80 + - name: Run unit tests + run: | + python -m pytest -m "not (smoke_test or pact_test)" + + - name: Install lambda requirements + working-directory: ./docs/ci_scripts + run: | + ./install_requirements.sh \ + -d lambda_functions \ + -r requirements.txt \ + -p lambda_layers/python/lib/python3.7/site-packages + + - name: Create Artifact + run: | + chmod -R 755 ./lambda_functions + zip -r9 /tmp/opg-data-lpa.zip . + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: opg-data-lpa + path: /tmp/opg-data-lpa.zip + + terraform_checks: + runs-on: ubuntu-latest + needs: build_and_test + env: + TF_VAR_pagerduty_token: ${{ secrets.PAGERDUTY_TOKEN }} + strategy: + max-parallel: 1 + matrix: + include: + - environment: "development" + + - environment: "preproduction" + + - environment: "production" + steps: + - name: Retrieve Artifact + uses: actions/download-artifact@v2 + with: + name: opg-data-lpa + - name: unzip + run: | + unzip ./opg-data-lpa.zip -d . + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 0.14.11 + + - name: Setup environment + run: | + echo TF_WORKSPACE=${{ matrix.environment }} >> $GITHUB_ENV + - name: Terraform init + working-directory: ./terraform/environment + run: | + terraform init + + - name: Terraform formatting + working-directory: ./terraform/environment + run: | + terraform fmt -diff -check -recursive + - name: Validate Terraform + working-directory: ./terraform/environment + run: | + terraform validate + + - name: Terraform plan + working-directory: ./terraform/environment + run: | + terraform plan -input=false + + pact_verification: + parameters: + env: + # GIT_CONSUMER: << pipeline.parameters.consumer >> + # GIT_COMMIT_CONSUMER: << pipeline.parameters.consumerversion >> + PACT_PROVIDER: lpa + PACT_CONSUMER: sirius + PACT_BROKER_BASE_URL: https://pact-broker.api.opg.service.justice.gov.uk + PACT_BROKER_HTTP_AUTH_USER: admin + steps: + - uses: actions/checkout@v2 + - run: + name: Set BASH_ENV + command: ./set_env.sh >> $BASH_ENV + working_directory: ~/project/.circleci + - run: + name: spin up mock rest api + command: echo "Insert command to spin up env here" + background: true + - run: + name: check mock env started correctly + command: | + sleep 3 + echo "Insert commands to check mock env running correctly here" + - name: Pact Install + run: | + wget https://github.com/pact-foundation/pact-ruby-standalone/releases/download/v1.82.3/pact-1.82.3-linux-x86_64.tar.gz + tar xzf pact-1.82.3-linux-x86_64.tar.gz + - name: Code artifact login + working-directory: ./docs/ci_scripts + run: | + ./login_code_artifact.sh -a 288342028542 -t pip + + - name: install ci requirements + working_directory: ~/project/pact + run: | + pip3 install -r requirements.txt + + - name: verify pact + working-directory: ./pact + run: | + echo ${API_VERSION} + python check_pact_deployable.py \ + --provider_base_url="http://localhost:4343" \ + --pact_broker_url="${PACT_BROKER_BASE_URL}" \ + --broker_user_name="admin" \ + --broker_secret_name="pactbroker_admin" \ + --consumer_pacticipant="${PACT_CONSUMER}" \ + --provider_pacticipant="${PACT_PROVIDER}" \ + --api_version="${API_VERSION}" \ + --git_commit_consumer="${GIT_COMMIT_CONSUMER}" \ + --git_commit_provider="${GIT_COMMIT_PROVIDER}" || echo "Failed but because consumer pacts not set up yet" + From 34f4b6199a138591844a393e2f32a4f0330f03b8 Mon Sep 17 00:00:00 2001 From: Tom Gulliver Date: Thu, 12 Aug 2021 11:25:46 +0100 Subject: [PATCH 2/5] SP-771 - formatting --- .github/workflows/build.yml | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9796e30d..75d8b6ae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -102,8 +102,9 @@ jobs: run: | terraform plan -input=false - pact_verification: - parameters: + pact_verification: + runs-on: ubuntu-latest + needs: terraform_checks env: # GIT_CONSUMER: << pipeline.parameters.consumer >> # GIT_COMMIT_CONSUMER: << pipeline.parameters.consumerversion >> @@ -113,19 +114,6 @@ jobs: PACT_BROKER_HTTP_AUTH_USER: admin steps: - uses: actions/checkout@v2 - - run: - name: Set BASH_ENV - command: ./set_env.sh >> $BASH_ENV - working_directory: ~/project/.circleci - - run: - name: spin up mock rest api - command: echo "Insert command to spin up env here" - background: true - - run: - name: check mock env started correctly - command: | - sleep 3 - echo "Insert commands to check mock env running correctly here" - name: Pact Install run: | wget https://github.com/pact-foundation/pact-ruby-standalone/releases/download/v1.82.3/pact-1.82.3-linux-x86_64.tar.gz @@ -136,7 +124,7 @@ jobs: ./login_code_artifact.sh -a 288342028542 -t pip - name: install ci requirements - working_directory: ~/project/pact + working-directory: ./pact run: | pip3 install -r requirements.txt From d81b0443feac9a8b76370afffc5df6c16f0b8fb5 Mon Sep 17 00:00:00 2001 From: Tom Gulliver Date: Fri, 13 Aug 2021 16:01:32 +0100 Subject: [PATCH 3/5] SP-771 - add deploy and workspace destroy --- .github/workflows/build.yml | 129 ++++++++++++++++------- .github/workflows/deploy.yml | 62 +++++++++++ .github/workflows/destroy_workspaces.yml | 41 +++++++ 3 files changed, 192 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/destroy_workspaces.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 75d8b6ae..c462783c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,10 +16,6 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - uses: unfor19/install-aws-cli-action@v1 - - - name: Code Artifact login - run: | - ./login_code_artifact.sh -a 288342028542 -t pip - name: Install requirements working-directory: ./docs/ci_scripts run: | @@ -76,6 +72,15 @@ jobs: - name: unzip run: | unzip ./opg-data-lpa.zip -d . + + - name: Configure AWS Credentials For Terraform + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-1 + role-session-name: GitHubActionsTerraform + - uses: hashicorp/setup-terraform@v1 with: terraform_version: 0.14.11 @@ -102,44 +107,88 @@ jobs: run: | terraform plan -input=false - pact_verification: - runs-on: ubuntu-latest - needs: terraform_checks - env: - # GIT_CONSUMER: << pipeline.parameters.consumer >> - # GIT_COMMIT_CONSUMER: << pipeline.parameters.consumerversion >> - PACT_PROVIDER: lpa - PACT_CONSUMER: sirius - PACT_BROKER_BASE_URL: https://pact-broker.api.opg.service.justice.gov.uk - PACT_BROKER_HTTP_AUTH_USER: admin + # pact_verification: + # runs-on: ubuntu-latest + # needs: terraform_checks + # env: + # # GIT_CONSUMER: << pipeline.parameters.consumer >> + # # GIT_COMMIT_CONSUMER: << pipeline.parameters.consumerversion >> + # PACT_PROVIDER: lpa + # PACT_CONSUMER: sirius + # PACT_BROKER_BASE_URL: https://pact-broker.api.opg.service.justice.gov.uk + # PACT_BROKER_HTTP_AUTH_USER: admin + # steps: + # - uses: actions/checkout@v2 + # - name: Pact Install + # run: | + # wget https://github.com/pact-foundation/pact-ruby-standalone/releases/download/v1.82.3/pact-1.82.3-linux-x86_64.tar.gz + # tar xzf pact-1.82.3-linux-x86_64.tar.gz + # - name: Code artifact login + # working-directory: ./docs/ci_scripts + # run: | + # ./login_code_artifact.sh -a 288342028542 -t pip + + # - name: install ci requirements + # working-directory: ./pact + # run: | + # pip3 install -r requirements.txt + + # - name: verify pact + # working-directory: ./pact + # run: | + # echo ${API_VERSION} + # python check_pact_deployable.py \ + # --provider_base_url="http://localhost:4343" \ + # --pact_broker_url="${PACT_BROKER_BASE_URL}" \ + # --broker_user_name="admin" \ + # --broker_secret_name="pactbroker_admin" \ + # --consumer_pacticipant="${PACT_CONSUMER}" \ + # --provider_pacticipant="${PACT_PROVIDER}" \ + # --api_version="${API_VERSION}" \ + # --git_commit_consumer="${GIT_COMMIT_CONSUMER}" \ + # --git_commit_provider="${GIT_COMMIT_PROVIDER}" || echo "Failed but because consumer pacts not set up yet" + + ephemeral_environment: + name: Create Ephemeral Environment + runs-on: "ubuntu-latest" + needs: [build_and_test, terraform_checks] steps: - uses: actions/checkout@v2 - - name: Pact Install - run: | - wget https://github.com/pact-foundation/pact-ruby-standalone/releases/download/v1.82.3/pact-1.82.3-linux-x86_64.tar.gz - tar xzf pact-1.82.3-linux-x86_64.tar.gz - - name: Code artifact login - working-directory: ./docs/ci_scripts + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 1.0.0 + - name: Extract branch name + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_HEAD_REF#refs/heads/})" + id: extract_branch + - name: Install workspace manager + run: | + wget https://github.com/TomTucka/terraform-workspace-manager/releases/download/v0.3.0/terraform-workspace-manager_Linux_x86_64.tar.gz -O $HOME/terraform-workspace-manager.tar.gz + sudo tar -xvf $HOME/terraform-workspace-manager.tar.gz -C /usr/local/bin + sudo chmod +x /usr/local/bin/terraform-workspace-manager + - name: Configure AWS Credentials For Terraform + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-1 + role-session-name: GitHubActionsTerraform + - name: Download Artifact + uses: actions/download-artifact@v2 + with: + name: opg-data-lpa + - name: Unzip Artifact + run: unzip ./opg-data-lpa-codes.zip -d . + - name: Setup Workspace + run: echo TF_WORKSPACE= ${{ steps.extract_branch.outputs.branch }} + - name: Terraform init + working-directory: ./terraform/environment run: | - ./login_code_artifact.sh -a 288342028542 -t pip - - - name: install ci requirements - working-directory: ./pact - run: | - pip3 install -r requirements.txt - - - name: verify pact - working-directory: ./pact + terraform init + - name: Terraform apply + working-directory: ./terraform/environment run: | - echo ${API_VERSION} - python check_pact_deployable.py \ - --provider_base_url="http://localhost:4343" \ - --pact_broker_url="${PACT_BROKER_BASE_URL}" \ - --broker_user_name="admin" \ - --broker_secret_name="pactbroker_admin" \ - --consumer_pacticipant="${PACT_CONSUMER}" \ - --provider_pacticipant="${PACT_PROVIDER}" \ - --api_version="${API_VERSION}" \ - --git_commit_consumer="${GIT_COMMIT_CONSUMER}" \ - --git_commit_provider="${GIT_COMMIT_PROVIDER}" || echo "Failed but because consumer pacts not set up yet" + terraform apply --auto-approve + - name: Protect Workspace + run: terraform-workspace-manager -register-workspace=$TF_WORKSPACE -time-to-protect=1 -aws-account-id=288342028542 -aws-iam-role=integrations-ci diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..2a01183b --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,62 @@ +name: Deploy + +on: + push: + branches: + - main + +defaults: + run: + shell: bash + +jobs: + terraform_apply: + runs-on: "ubuntu-latest" + env: + TF_VAR_pagerduty_token: ${{ secrets.PAGERDUTY_TOKEN }} + strategy: + max-parallel: 1 + matrix: + include: + - tf_workspace: "development" + + - tf_workspace: "preproduction" + + - tf_workspace: "production" + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - name: Install lambda requirements + working-directory: ./docs/ci_scripts + run: | + ./install_requirements.sh \ + -d lambda_functions \ + -r requirements.txt \ + -p lambda_layers/python/lib/python3.7/site-packages + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 0.14.11 + - name: Configure AWS Credentials For Terraform + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-1 + role-session-name: GitHubActionsTerraform + - name: Setup Workspace + run: echo TF_WORKSPACE=${{ matrix.tf_workspace }} >> $GITHUB_ENV + + - name: Terraform init + working-directory: ./infrastructure/terraform + run: | + terraform init + - name: Plan ${{ matrix.tf_workspace }} + run: | + terraform init -input=false + terraform plan -lock-timeout=300s -input=false -parallelism=30 + + - name: Apply ${{ matrix.tf_workspace }} + if: github.ref == 'refs/heads/main' + run: | + terraform apply -lock-timeout=300s -input=false -auto-approve -parallelism=30 + diff --git a/.github/workflows/destroy_workspaces.yml b/.github/workflows/destroy_workspaces.yml new file mode 100644 index 00000000..46b654e3 --- /dev/null +++ b/.github/workflows/destroy_workspaces.yml @@ -0,0 +1,41 @@ +name: Destroy Ephemeral Environments + +on: + schedule: + - cron: '30 * * * 1-6' + + +defaults: + working-directory: ./terraform/Environment + run: + shell: bash + +jobs: + destroy_workspace: + name: Destroy + runs-on: "ubuntu-latest" + needs: terraform_checks + steps: + - uses: actions/checkout@v2 + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 1.0.0 + - name: Install workspace manager + run: | + wget https://github.com/TomTucka/terraform-workspace-manager/releases/download/v0.3.0/terraform-workspace-manager_Linux_x86_64.tar.gz -O $HOME/terraform-workspace-manager.tar.gz + sudo tar -xvf $HOME/terraform-workspace-manager.tar.gz -C /usr/local/bin + sudo chmod +x /usr/local/bin/terraform-workspace-manager + - name: Configure AWS Credentials For Terraform + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-1 + role-session-name: GitHubActionsTerraform + - name: Terraform init + run: | + terraform init + - name: Destroy Protected Workspaces + run: | + unset TF_WORKSPACE + ./scripts/workspace_cleanup.sh $(terraform-workspace-manager -protected-workspaces=true -aws-account-id=288342028542 -aws-iam-role=integrations-ci) From 8a4db9b55a447169381d30ac99a2d913de22d5d9 Mon Sep 17 00:00:00 2001 From: Tom Gulliver Date: Wed, 18 Aug 2021 10:05:01 +0100 Subject: [PATCH 4/5] Trigger Build --- .github/workflows/build.yml | 40 ++++++++++++++++++++---------------- .github/workflows/deploy.yml | 10 ++++----- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c462783c..d22a37b3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,9 +17,11 @@ jobs: - uses: actions/setup-python@v2 - uses: unfor19/install-aws-cli-action@v1 - name: Install requirements - working-directory: ./docs/ci_scripts run: | - ./install_requirements.sh -d lambda_functions -r dev-requirements.txt + for i in $(ls -d lambda_functions/*/ | awk -F'/' '{print $2}') + do + pip3 install -r ./lambda_functions/"${i}"/requirements/dev-requirements.txt + done - name: Run Flask8 run: | @@ -32,16 +34,17 @@ jobs: python -m pytest -m "not (smoke_test or pact_test)" - name: Install lambda requirements - working-directory: ./docs/ci_scripts run: | - ./install_requirements.sh \ - -d lambda_functions \ - -r requirements.txt \ - -p lambda_layers/python/lib/python3.7/site-packages + for i in $(ls -d lambda_functions/*/ | awk -F'/' '{print $2}' | grep '^v[1-9]\+') + do + export LAYER_PATH=lambda_functions/"${i}"/lambda_layers/python/lib/python3.7/site-packages + pip3 install -r lambda_functions/"${i}"/requirements/requirements.txt --target ./$LAYER_PATH/ + done - name: Create Artifact run: | chmod -R 755 ./lambda_functions + cd ./lambda_functions zip -r9 /tmp/opg-data-lpa.zip . - name: Upload artifact @@ -65,14 +68,8 @@ jobs: - environment: "production" steps: - - name: Retrieve Artifact - uses: actions/download-artifact@v2 - with: - name: opg-data-lpa - - name: unzip - run: | - unzip ./opg-data-lpa.zip -d . - + - uses: actions/checkout@v2 + - name: Configure AWS Credentials For Terraform uses: aws-actions/configure-aws-credentials@v1 with: @@ -84,7 +81,14 @@ jobs: - uses: hashicorp/setup-terraform@v1 with: terraform_version: 0.14.11 - + - name: Retrieve Artifact + uses: actions/download-artifact@v2 + with: + name: opg-data-lpa + - name: unzip + run: | + unzip ./opg-data-lpa.zip -d . + - name: Setup environment run: | echo TF_WORKSPACE=${{ matrix.environment }} >> $GITHUB_ENV @@ -173,12 +177,12 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: eu-west-1 role-session-name: GitHubActionsTerraform - - name: Download Artifact + - name: Retrieve Artifact uses: actions/download-artifact@v2 with: name: opg-data-lpa - name: Unzip Artifact - run: unzip ./opg-data-lpa-codes.zip -d . + run: unzip ./opg-data-lpa.zip -d . - name: Setup Workspace run: echo TF_WORKSPACE= ${{ steps.extract_branch.outputs.branch }} - name: Terraform init diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2a01183b..ad60b2c7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -27,12 +27,12 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - name: Install lambda requirements - working-directory: ./docs/ci_scripts run: | - ./install_requirements.sh \ - -d lambda_functions \ - -r requirements.txt \ - -p lambda_layers/python/lib/python3.7/site-packages + for i in $(ls -d lambda_functions/*/ | awk -F'/' '{print $2}' | grep '^v[1-9]\+') + do + export LAYER_PATH=lambda_functions/"${i}"/lambda_layers/python/lib/python3.7/site-packages + pip3 install -r lambda_functions/"${i}"/requirements/requirements.txt --target ./$LAYER_PATH/ + done - uses: hashicorp/setup-terraform@v1 with: terraform_version: 0.14.11 From fe74d55648f1cdd49895a3b4187e5c8198e2fe8c Mon Sep 17 00:00:00 2001 From: Tom Gulliver Date: Wed, 18 Aug 2021 10:52:59 +0100 Subject: [PATCH 5/5] SP-771 - remove circle config --- .circleci/config.yml | 458 ------------------------ .circleci/set_env.sh | 21 -- docs/ci_scripts/cancel_builds.py | 3 - docs/ci_scripts/install_requirements.sh | 29 -- docs/ci_scripts/login_code_artifact.sh | 29 -- docs/ci_scripts/requirements.txt | 2 - docs/ci_scripts/terraform_plan.sh | 24 -- 7 files changed, 566 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100755 .circleci/set_env.sh delete mode 100644 docs/ci_scripts/cancel_builds.py delete mode 100755 docs/ci_scripts/install_requirements.sh delete mode 100755 docs/ci_scripts/login_code_artifact.sh delete mode 100644 docs/ci_scripts/requirements.txt delete mode 100755 docs/ci_scripts/terraform_plan.sh diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index abb80ce8..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,458 +0,0 @@ ---- -version: 2.1 - -parameters: - run_pull_request: - type: boolean - default: true - run_commit_to_main: - type: boolean - default: true - run_verify_pact: - type: boolean - default: false - consumerversion: - type: string - default: "" - consumer: - type: string - default: "" - -workflows: - pull_request: - when: << pipeline.parameters.run_pull_request >> - jobs: - - build_and_test: - name: build and artifact - filters: {branches:{ignore:[main]}} - - - terraform_preflight: - name: terraform preflight checks - requires: [build and artifact] - filters: {branches:{ignore:[main]}} - - - terraform_action: - name: apply branch environment - requires: [terraform preflight checks] - tf_command: apply - filters: {branches:{ignore:[main]}} - - - workspace_protection: - name: protect current workspace - requires: [terraform preflight checks] - filters: {branches:{ignore:[main]}} - - - workflow_complete: - name: workflow complete - requires: [apply branch environment, protect current workspace] - filters: {branches:{ignore:[main]}} - - commit_to_main: - when: << pipeline.parameters.run_commit_to_main >> - jobs: - - build_and_test: - name: build and artifact - filters: {branches:{only:[main]}} - - - terraform_action: - name: development apply - requires: [build and artifact] - tf_command: apply --auto-approve - tf_workspace: development - filters: {branches:{only:[main]}} - - - terraform_action: - name: integration apply - requires: [development apply] - tf_command: apply --auto-approve - tf_workspace: integration - filters: {branches:{only:[main]}} - - - terraform_action: - name: preprod apply - requires: [integration apply] - tf_command: apply --auto-approve - tf_workspace: preproduction - filters: {branches:{only:[main]}} - - - approve: - name: approve release to production - type: approval - requires: [preprod apply] - filters: {branches:{only:[main]}} - - - terraform_action: - name: production apply - requires: [approve release to production] - tf_command: apply --auto-approve - tf_workspace: production - filters: {branches:{only:[main]}} - pact_tag_production: true - - verify_pact: - when: << pipeline.parameters.run_verify_pact >> - jobs: - - pact_verification: - name: verify the latest pact - - nightly_workspace_deletion: - triggers: - - schedule: - cron: "00 00 * * *" - filters: {branches:{only:[main]}} - jobs: - - destroy_workspaces: - name: destroy non protected workspaces - filters: {branches:{only:[main]}} - -orbs: - slack: circleci/slack@3.4.2 - dockerhub_helper: - orbs: - docker: circleci/docker@1.4.0 - commands: - dockerhub_login: - steps: - - docker/install-docker-credential-helper - - docker/check: - docker-password: DOCKER_ACCESS_TOKEN # pragma: allowlist secret - docker-username: DOCKER_USER - lpa-data: - executors: - python_with_tfvars: - docker: - - image: circleci/python:3 - auth: - username: $DOCKER_USER - password: $DOCKER_ACCESS_TOKEN - resource_class: small - environment: - TF_CLI_ARGS_plan: -input=false -lock=false - TF_CLI_ARGS_apply: -input=false -auto-approve - TF_CLI_ARGS_destroy: -input=false -auto-approve - TF_CLI_ARGS_init: -input=false -upgrade=true -reconfigure - TF_VAR_default_role: integrations-ci - TF_VERSION: 0.14.11 - TF_SHA256SUM: 171ef5a4691b6f86eab524feaf9a52d5221c875478bd63dd7e55fef3939f7fd4 # pragma: allowlist secret - python: - docker: - - image: circleci/python:3.8.1 - auth: - username: $DOCKER_USER - password: $DOCKER_ACCESS_TOKEN - commands: - terraform_install: - steps: - - run: - name: Download Terraform - command: curl -sfSO https://releases.hashicorp.com/terraform/${TF_VERSION}/terraform_${TF_VERSION}_linux_amd64.zip - - run: - name: Add Terraform SHA256SUM - command: echo "${TF_SHA256SUM} terraform_${TF_VERSION}_linux_amd64.zip" > SHA256SUMS - - run: - name: Check Terraform SHA256SUM - command: sha256sum -c --status SHA256SUMS - - run: - name: Install Terraform - command: sudo unzip terraform_${TF_VERSION}_linux_amd64.zip -d /bin - aws_cli_install: - steps: - - run: - name: install aws cli - command: | - pip3 install awscli --upgrade --user - pact_install: - steps: - - run: - name: Set up ready for pact tests - command: | - wget https://github.com/pact-foundation/pact-ruby-standalone/releases/download/v1.82.3/pact-1.82.3-linux-x86_64.tar.gz - tar xzf pact-1.82.3-linux-x86_64.tar.gz - working_directory: ~/project - login_codeartifact: - steps: - - run: - name: install aws cli - command: | - pip3 install awscli --upgrade --user - - run: - name: login to code artifact for pip - command: ./login_code_artifact.sh -a 288342028542 -t pip - working_directory: ~/project/docs/ci_scripts - cancel_redundant_builds: - steps: - - run: - name: install cancel build requirements - command: pip3 install -r requirements.txt - working_directory: ~/project/docs/ci_scripts - - run: - name: cancel previous builds - command: | - python3 cancel_builds.py \ - --circle_project_username="${CIRCLE_PROJECT_USERNAME}" \ - --circle_project_reponame="${CIRCLE_PROJECT_REPONAME}" \ - --circle_branch="${CIRCLE_BRANCH}" \ - --circle_builds_token="${CIRCLE_BUILDS_TOKEN}" \ - --terms_to_waitfor="apply,plan,terraform,destroy" \ - --prod_job_terms="production ,integration ,pact" - working_directory: ~/project/docs/ci_scripts - install_workspace_manager: - steps: - - run: - name: install workspace manager - command: | - wget https://github.com/TomTucka/terraform-workspace-manager/releases/download/v0.3.0/terraform-workspace-manager_Linux_x86_64.tar.gz -O $HOME/terraform-workspace-manager.tar.gz - sudo tar -xvf $HOME/terraform-workspace-manager.tar.gz -C /usr/local/bin - sudo chmod +x /usr/local/bin/terraform-workspace-manager - -jobs: - build_and_test: - executor: lpa-data/python - resource_class: small - steps: - - checkout - - lpa-data/login_codeartifact - - lpa-data/cancel_redundant_builds - - run: - name: install all requirements locally - command: ./install_requirements.sh -d lambda_functions -r dev-requirements.txt - working_directory: ~/project/docs/ci_scripts - - run: - name: run flake8 - python PEP8 linter, complexity checker - command: | - flake8 --ignore Q000 lambda_functions - - run: - name: get test coverage (fails below 80%) - command: | - python -m pytest lambda_functions/v1/tests --cov=lambda_functions/v1/functions/lpa/app/api --cov-fail-under=80 - - run: - name: run unit tests (with pytest) - command: | - python -m pytest -m "not (smoke_test or pact_test)" - - run: - name: install requirements for all lambda layers - command: | - ./install_requirements.sh \ - -d lambda_functions \ - -r requirements.txt \ - -p lambda_layers/python/lib/python3.7/site-packages - working_directory: ~/project/docs/ci_scripts - - run: - name: create artifact for future stages - command: | - chmod -R 755 ./lambda_functions - zip -r9 /tmp/opg-data-lpa.zip . - - persist_to_workspace: - root: /tmp - paths: - - opg-data-lpa.zip - - terraform_preflight: - executor: lpa-data/python_with_tfvars - resource_class: small - parameters: - tf_workspace: - description: terraform workspace - type: string - default: "" - working_directory: ~/project - steps: - - attach_workspace: - at: ~/project - - lpa-data/terraform_install - - run: - name: unzip artifact - command: unzip ./opg-data-lpa.zip -d . - - run: - name: initialize terraform - environment - working_directory: ~/project/terraform/environment - command: terraform init - - run: - name: Set BASH_ENV - command: ~/project/.circleci/set_env.sh >> $BASH_ENV - - run: - name: terraform check formatting - environment - working_directory: ~/project/terraform/environment - command: terraform fmt -diff -check -recursive - - run: - name: validate terraform - environment - working_directory: ~/project/terraform/environment - command: terraform validate - - run: - name: terraform plan - branch - command: terraform plan -input=false - working_directory: ~/project/terraform/environment - - run: - name: terraform plan - development - command: ./terraform_plan.sh -w development - working_directory: ~/project/docs/ci_scripts - - run: - name: terraform plan - preproduction - command: ./terraform_plan.sh -w preproduction - working_directory: ~/project/docs/ci_scripts - - run: - name: terraform plan - production - command: ./terraform_plan.sh -w production - working_directory: ~/project/docs/ci_scripts - - terraform_action: - executor: lpa-data/python_with_tfvars - resource_class: small - parameters: - tf_workspace: - description: terraform workspace - type: string - default: "" - tf_command: - description: terraform command - default: plan - type: string - pact_tag_production: - description: to tag the production commit - type: boolean - default: false - environment: - WORKSPACE: << parameters.tf_workspace >> - working_directory: ~/project - steps: - - attach_workspace: - at: ~/project - - lpa-data/terraform_install - - run: - name: unzip artifact - command: unzip ./opg-data-lpa.zip -d . - - run: - name: Set Environment Variables - command: | - ~/project/.circleci/set_env.sh >> $BASH_ENV - eval $(cat $BASH_ENV) - - run: - name: initialize terraform - environment - working_directory: ~/project/terraform/environment - command: terraform init - - run: - name: terraform << parameters.tf_command >> - environment - working_directory: ~/project/terraform/environment - command: terraform << parameters.tf_command >> - - when: - condition: << parameters.pact_tag_production >> - steps: - - lpa-data/login_codeartifact - - run: - name: install ci requirements - command: pip3 install -r requirements.txt - working_directory: ~/project/pact - - run: - name: tag pact commit with v_production - command: | - echo ${API_VERSION} - python tag_pact.py \ - --pact_broker_url"${PACT_BROKER_BASE_URL}" \ - --broker_user_name="admin" \ - --broker_secret_name="pactbroker_admin" \ - --pacticipant="${PACT_PROVIDER}" \ - --tag="${API_VERSION}_production" \ - --git_commit="${GIT_COMMIT_PROVIDER}" || echo "Failed but because consumer pacts not set up yet" - working_directory: ~/project/pact - pact_verification: - docker: - # Primary container image where all the steps run. - - image: circleci/python:3 - auth: - username: $DOCKER_USER - password: $DOCKER_ACCESS_TOKEN - resource_class: small - working_directory: ~/project - parameters: - environment: - GIT_CONSUMER: << pipeline.parameters.consumer >> - GIT_COMMIT_CONSUMER: << pipeline.parameters.consumerversion >> - steps: - - checkout - - run: - name: Set BASH_ENV - command: ./set_env.sh >> $BASH_ENV - working_directory: ~/project/.circleci - - run: - name: spin up mock rest api - command: echo "Insert command to spin up env here" - background: true - - run: - name: check mock env started correctly - command: | - sleep 3 - echo "Insert commands to check mock env running correctly here" - - lpa-data/pact_install - - lpa-data/login_codeartifact - - run: - name: install ci requirements - command: pip3 install -r requirements.txt - working_directory: ~/project/pact - - run: - name: verify pact - command: | - echo ${API_VERSION} - python check_pact_deployable.py \ - --provider_base_url="http://localhost:4343" \ - --pact_broker_url="${PACT_BROKER_BASE_URL}" \ - --broker_user_name="admin" \ - --broker_secret_name="pactbroker_admin" \ - --consumer_pacticipant="${PACT_CONSUMER}" \ - --provider_pacticipant="${PACT_PROVIDER}" \ - --api_version="${API_VERSION}" \ - --git_commit_consumer="${GIT_COMMIT_CONSUMER}" \ - --git_commit_provider="${GIT_COMMIT_PROVIDER}" || echo "Failed but because consumer pacts not set up yet" - working_directory: ~/project/pact - workspace_protection: - executor: lpa-data/python_with_tfvars - resource_class: small - working_directory: ~/project/environment - parameters: - protect_time: - description: time to protect workspace - type: string - default: "24" - environment: - PROTECT_TIME: << parameters.protect_time >> - steps: - - checkout: - path: ~/project - - lpa-data/terraform_install - - lpa-data/install_workspace_manager - - run: - name: Set environment - command: ~/project/.circleci/set_env.sh >> $BASH_ENV - - run: - name: Add workspace to protected list - command: terraform-workspace-manager -register-workspace="${TF_WORKSPACE}" -time-to-protect="${PROTECT_TIME}" -aws-account-id=288342028542 -aws-iam-role=integrations-ci - workflow_complete: - executor: lpa-data/python_with_tfvars - resource_class: small - working_directory: ~/project - steps: - - slack/status: - channel: opg-integrations - failure_message: Failure of LPA Data Workflow for Branch - ${CIRCLE_BRANCH} - success_message: Success of LPA Data Workflow for Branch - ${CIRCLE_BRANCH}. Ready to Merge! - destroy_workspaces: - executor: lpa-data/python_with_tfvars - resource_class: small - working_directory: ~/project/terraform/environment - steps: - - checkout: - path: ~/project - - lpa-data/terraform_install - - lpa-data/install_workspace_manager - - attach_workspace: {at: ~/project} - - run: - name: initialise terraform - command: terraform init - - run: - name: destroy unprotected workspaces - command: | - unset TF_WORKSPACE - ./scripts/workspace_cleanup.sh $(terraform-workspace-manager -protected-workspaces=true -aws-account-id=288342028542 -aws-iam-role=integrations-ci) - - slack/status: - channel: opg-integrations - failure_message: nightly destroy of lpa data workspaces has failed. - success_message: nightly destroy of lpa data workspaces has succeeded. diff --git a/.circleci/set_env.sh b/.circleci/set_env.sh deleted file mode 100755 index 39e99180..00000000 --- a/.circleci/set_env.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -set -e -PACT_BROKER_ADMIN="admin" -PACT_BASE_URL="https://pact-broker.api.opg.service.justice.gov.uk" -PROVIDER="lpa" -CONSUMER="sirius" - -WORKSPACE=${WORKSPACE:-$CIRCLE_BRANCH} -WORKSPACE=${WORKSPACE//[^[:alnum:]]/} -WORKSPACE=${WORKSPACE,,} -WORKSPACE=${WORKSPACE:0:14} -PROVIDER_VER="v1" -GIT_COMMIT=${CIRCLE_SHA1:0:7} - -echo "export GIT_COMMIT_PROVIDER=${GIT_COMMIT}" -echo "export TF_WORKSPACE=${WORKSPACE}" -echo "export API_VERSION=${PROVIDER_VER}" -echo "export PACT_PROVIDER=${PROVIDER}" -echo "export PACT_CONSUMER=${CONSUMER}" -echo "export PACT_BROKER_BASE_URL=${PACT_BASE_URL}" -echo "export PACT_BROKER_HTTP_AUTH_USER=${PACT_BROKER_ADMIN}" diff --git a/docs/ci_scripts/cancel_builds.py b/docs/ci_scripts/cancel_builds.py deleted file mode 100644 index afaa27cc..00000000 --- a/docs/ci_scripts/cancel_builds.py +++ /dev/null @@ -1,3 +0,0 @@ -from opg_integrations_shared.cancel_redundant_builds import main - -main() diff --git a/docs/ci_scripts/install_requirements.sh b/docs/ci_scripts/install_requirements.sh deleted file mode 100755 index 2f568c4a..00000000 --- a/docs/ci_scripts/install_requirements.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -while getopts d:r:p: option -do - case "${option}" - in - d) DIRECTORY=${OPTARG};; - r) REQ_FILE=${OPTARG};; - p) INSTALL_PATH=${OPTARG};; - *) echo "usage: $0 [-d] [-r]" >&2 - exit 1 ;; - esac -done - -if [ "${INSTALL_PATH}" != "" ] -then - FLAG_INSTALL_PATH=" --target ${INSTALL_PATH}" - for ver in $(ls -d ../../${DIRECTORY}/*/ | awk -F'/' '{print $4}' | grep '^v[1-9]\+') - do - LAYER_PATH="../../${DIRECTORY}/${ver}/${INSTALL_PATH}" - pip3 install -r "../../${DIRECTORY}/${ver}/requirements/${REQ_FILE}" --target "./${LAYER_PATH}/" - done -else - for ver in $(ls -d ../../${DIRECTORY}/*/ | awk -F'/' '{print $4}' | grep '^v[1-9]\+') - do - pip3 install -r "../../${DIRECTORY}/${ver}/requirements/${REQ_FILE}" - done -fi - diff --git a/docs/ci_scripts/login_code_artifact.sh b/docs/ci_scripts/login_code_artifact.sh deleted file mode 100755 index 622d9983..00000000 --- a/docs/ci_scripts/login_code_artifact.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -set -e - -while getopts t:a: option -do - case "${option}" - in - t) TOOL=${OPTARG};; - a) ACCOUNT=${OPTARG};; - *) echo "usage: $0 [-d] [-r]" >&2 - exit 1 ;; - esac -done - -export SECRET_STRING=$(aws sts assume-role \ ---role-arn "arn:aws:iam::${ACCOUNT}:role/integrations-ci" \ ---role-session-name AWSCLI-Session | \ -jq -r '.Credentials.SessionToken + " " + .Credentials.SecretAccessKey + " " + .Credentials.AccessKeyId') - -#local export so they only exist in this stage -export AWS_ACCESS_KEY_ID=$(echo "${SECRET_STRING}" | awk -F' ' '{print $3}') -export AWS_SECRET_ACCESS_KEY=$(echo "${SECRET_STRING}" | awk -F' ' '{print $2}') -export AWS_SESSION_TOKEN=$(echo "${SECRET_STRING}" | awk -F' ' '{print $1}') - -aws codeartifact login --tool "${TOOL}" \ ---repository opg-pip-shared-code-dev \ ---domain opg-moj \ ---domain-owner "${ACCOUNT}" \ ---region eu-west-1 diff --git a/docs/ci_scripts/requirements.txt b/docs/ci_scripts/requirements.txt deleted file mode 100644 index 08da3f61..00000000 --- a/docs/ci_scripts/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests -opg_integrations_shared==0.0.3 diff --git a/docs/ci_scripts/terraform_plan.sh b/docs/ci_scripts/terraform_plan.sh deleted file mode 100755 index fa65501e..00000000 --- a/docs/ci_scripts/terraform_plan.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -while getopts w: option -do - case "${option}" - in - w) TF_WS=${OPTARG};; - *) echo "usage: $0 [-d] [-r]" >&2 - exit 1 ;; - esac -done - -cd ~/project/terraform/environment -TF_DIR=".terraform" -if [ -d "${TF_DIR}" ]; then rm -Rf ${TF_DIR}; fi -export TF_WORKSPACE=${TF_WS} -echo "" -echo "=== Running Initialisation ===" -echo "" -terraform init -echo "" -echo "=== Running Plan on ${TF_WS} ===" -echo "" -terraform plan -input=false