Use workflows, more threads, fail fast, pg first

Author: jtcohen6

.circleci/config.yml

@@ -1,23 +1,15 @@
 
-version: 2
+version: 2.1
 
 jobs:
-  build:
+  
+  integration-postgres:
     docker:
       - image: circleci/python:3.6.3-stretch
       - image: circleci/postgres:9.6.5-alpine-ram
 
     steps:
       - checkout
-
-      - run:
-          run: setup_creds
-          command: |
-            echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json
-
-      - restore_cache:
-          key: deps2-{{ .Branch }}
-
       - run:
           name: "Run Tests - Postgres"
           environment:
@@ -27,22 +19,58 @@ jobs:
             POSTGRES_TEST_PORT: 5432
             POSTGRES_TEST_DBNAME: circle_test
           command: ./run_test.sh postgres
+      - store_artifacts:
+          path: ./logs
 
+  integration-redshift:
+    docker:
+      - image: circleci/python:3.6.3-stretch
+    steps:
+      - checkout
       - run:
           name: "Run Tests - Redshift"
           command: ./run_test.sh redshift
+      - store_artifacts:
+          path: ./logs
 
+  integration-snowflake:
+    docker:
+      - image: circleci/python:3.6.3-stretch
+    steps:
+      - checkout
       - run:
           name: "Run Tests - Snowflake"
           command: ./run_test.sh snowflake
-
+      - store_artifacts:
+          path: ./logs          
+  
+  integration-bigquery:
+    environment:
+      BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
+    docker:
+      - image: circleci/python:3.6.3-stretch
+    steps:
+      - checkout
+      - run:
+          name: "Set up credentials"
+          command: echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json
       - run:
           name: "Run Tests - BigQuery"
-          environment:
-              BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
           command: ./run_test.sh bigquery
+      - store_artifacts:
+          path: ./logs
 
-      - save_cache:
-          key: deps1-{{ .Branch }}
-          paths:
-            - "venv"
+workflows:
+  version: 2
+  test-all:
+    jobs:
+      - integration-postgres
+      - integration-redshift:
+          requires:
+            - integration-postgres
+      - integration-snowflake:
+          requires:
+            - integration-postgres
+      - integration-bigquery:
+          requires:
+            - integration-postgres

integration_tests/ci/sample.profiles.yml

@@ -17,7 +17,7 @@ integration_tests:
       port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}"
       dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}"
       schema: dbt_utils_integration_tests_postgres
-      threads: 1
+      threads: 5
 
     redshift:
       type: redshift
@@ -27,15 +27,15 @@ integration_tests:
       dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}"
       port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}"
       schema: dbt_utils_integration_tests_redshift
-      threads: 1
+      threads: 5
 
     bigquery:
       type: bigquery
       method: service-account
       keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}"
       project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}"
       schema: dbt_utils_integration_tests_bigquery
-      threads: 1
+      threads: 10
 
     snowflake:
       type: snowflake
@@ -46,4 +46,4 @@ integration_tests:
       database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}"
       warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}"
       schema: dbt_utils_integration_tests_snowflake
-      threads: 1
+      threads: 10

run_test.sh

@@ -24,5 +24,5 @@ if [[ ! -z $3 ]]; then _seeds="--select $3 --full-refresh"; fi
 
 dbt deps --target $1
 dbt seed --target $1 $_seeds
-dbt run --target $1 $_models
-dbt test --target $1 $_models
+dbt run -x --target $1 $_models
+dbt test -x --target $1 $_models