Merge pull request #316 from fishtown-analytics/update/circleci-workflows

Switch to CircleCI workflow

Author: jtcohen6

.circleci/config.yml

@@ -1,23 +1,15 @@
 
-version: 2
+version: 2.1
 
 jobs:
-  build:
+  
+  integration-postgres:
     docker:
       - image: circleci/python:3.6.3-stretch
       - image: circleci/postgres:9.6.5-alpine-ram
 
     steps:
       - checkout
-
-      - run:
-          run: setup_creds
-          command: |
-            echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json
-
-      - restore_cache:
-          key: deps2-{{ .Branch }}
-
       - run:
           name: "Run Tests - Postgres"
           environment:
@@ -27,22 +19,58 @@ jobs:
             POSTGRES_TEST_PORT: 5432
             POSTGRES_TEST_DBNAME: circle_test
           command: ./run_test.sh postgres
+      - store_artifacts:
+          path: ./logs
 
+  integration-redshift:
+    docker:
+      - image: circleci/python:3.6.3-stretch
+    steps:
+      - checkout
       - run:
           name: "Run Tests - Redshift"
           command: ./run_test.sh redshift
+      - store_artifacts:
+          path: ./logs
 
+  integration-snowflake:
+    docker:
+      - image: circleci/python:3.6.3-stretch
+    steps:
+      - checkout
       - run:
           name: "Run Tests - Snowflake"
           command: ./run_test.sh snowflake
-
+      - store_artifacts:
+          path: ./logs          
+  
+  integration-bigquery:
+    environment:
+      BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
+    docker:
+      - image: circleci/python:3.6.3-stretch
+    steps:
+      - checkout
+      - run:
+          name: "Set up credentials"
+          command: echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json
       - run:
           name: "Run Tests - BigQuery"
-          environment:
-              BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
           command: ./run_test.sh bigquery
+      - store_artifacts:
+          path: ./logs
 
-      - save_cache:
-          key: deps1-{{ .Branch }}
-          paths:
-            - "venv"
+workflows:
+  version: 2
+  test-all:
+    jobs:
+      - integration-postgres
+      - integration-redshift:
+          requires:
+            - integration-postgres
+      - integration-snowflake:
+          requires:
+            - integration-postgres
+      - integration-bigquery:
+          requires:
+            - integration-postgres

integration_tests/ci/sample.profiles.yml

@@ -17,7 +17,7 @@ integration_tests:
       port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}"
       dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}"
       schema: dbt_utils_integration_tests_postgres
-      threads: 1
+      threads: 5
 
     redshift:
       type: redshift
@@ -27,15 +27,15 @@ integration_tests:
       dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}"
       port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}"
       schema: dbt_utils_integration_tests_redshift
-      threads: 1
+      threads: 5
 
     bigquery:
       type: bigquery
       method: service-account
       keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}"
       project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}"
       schema: dbt_utils_integration_tests_bigquery
-      threads: 1
+      threads: 10
 
     snowflake:
       type: snowflake
@@ -46,4 +46,4 @@ integration_tests:
       database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}"
       warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}"
       schema: dbt_utils_integration_tests_snowflake
-      threads: 1
+      threads: 10

macros/materializations/insert_by_period_materialization.sql

@@ -142,7 +142,13 @@
           from {{tmp_relation.include(schema=False)}}
       );
     {%- endcall %}
-    {%- set rows_inserted = (load_result('main-' ~ i)['status'].split(" "))[2] | int -%}
+    {% set result = load_result('main-' ~ i) %}
+    {% if 'response' in result.keys() %} {# added in v0.19.0 #}
+        {% set rows_inserted = result['response']['rows_affected'] %}
+    {% else %} {# older versions #}
+        {% set rows_inserted = result['status'].split(" ")[2] | int %}
+    {% endif %}
+    
     {%- set sum_rows_inserted = loop_vars['sum_rows_inserted'] + rows_inserted -%}
     {%- if loop_vars.update({'sum_rows_inserted': sum_rows_inserted}) %} {% endif -%}
 
@@ -165,7 +171,7 @@
 
   {%- set status_string = "INSERT " ~ loop_vars['sum_rows_inserted'] -%}
 
-  {% call noop_statement(name='main', status=status_string) -%}
+  {% call noop_statement('main', status_string) -%}
     -- no-op
   {%- endcall %}
 

run_test.sh

@@ -24,5 +24,8 @@ if [[ ! -z $3 ]]; then _seeds="--select $3 --full-refresh"; fi
 
 dbt deps --target $1
 dbt seed --target $1 $_seeds
-dbt run --target $1 $_models
-dbt test --target $1 $_models
+if [ $1 == 'redshift' ]; then
+    dbt run -x -m test_insert_by_period --full-refresh --target redshift
+fi
+dbt run -x --target $1 $_models
+dbt test -x --target $1 $_models