Breakout Spark CI

Author: clausherther

.circleci/config.yml

@@ -2,7 +2,8 @@ version: 2.1
 
 jobs:
 
-  integration-tests:
+  integration-tests-core:
+
     docker:
       - image: cimg/python:3.9.9
       - image: cimg/postgres:14.0
@@ -13,19 +14,27 @@ jobs:
       DBT_PROFILES_DIR: ./integration_tests/ci
       DBT_PROJECT_DIR: ./integration_tests
       BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
-      DBT_VERSION: 1.6.0
+      DBT_VERSION: 1.6.*
 
     steps:
       - checkout
-      - run:
-          name: Install Python packages
+      - run: &pip-install-core
+          name: Install core Python packages & dbt-core
           command: |
             python3 -m venv venv
             . venv/bin/activate
             pip install -U pip setuptools wheel
-            pip install dbt-core==$DBT_VERSION dbt-postgres==$DBT_VERSION dbt-bigquery==$DBT_VERSION dbt-snowflake==$DBT_VERSION dbt-duckdb==$DBT_VERSION
+            pip install "dbt-core==$DBT_VERSION"
 
       - run:
+          name: Install dbt adapter packages
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install "dbt-postgres==$DBT_VERSION" "dbt-bigquery==$DBT_VERSION" "dbt-snowflake==$DBT_VERSION"
+            pip install "dbt-duckdb==$DBT_VERSION"
+
+      - run: &dbt-deps
           name: Install dbt dependencies
           command: |
             . venv/bin/activate
@@ -74,12 +83,65 @@ jobs:
       - store_artifacts:
           path: ./logs
 
+  integration-tests-spark-thrift:
+
+    docker:
+      - image: cimg/python:3.9.9
+      - image: godatadriven/spark:3.1.1
+        environment:
+          WAIT_FOR: localhost:5432
+        command: >
+          --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
+          --name Thrift JDBC/ODBC Server
+      - image: postgres:9.6.17-alpine
+        environment:
+          POSTGRES_USER: dbt
+          POSTGRES_PASSWORD: dbt
+          POSTGRES_DB: metastore
+
+    resource_class: small
+
+    environment:
+      DBT_PROFILES_DIR: ./integration_tests/ci
+      DBT_PROJECT_DIR: ./integration_tests
+      DBT_VERSION: 1.6.*
+
+    steps:
+      - checkout
+      - run:
+          name: Install Ubuntu packages
+          command: |
+            sudo apt-get update
+            sudo apt-get install libsasl2-dev libsasl2-2
+      - run: *pip-install-core
+      - run:
+          name: Install dbt adapter packages
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install dbt-spark "dbt-spark[PyHive]"
+      - run: *dbt-deps
+      - run:
+          name: Wait for Spark-Thrift
+          command: dockerize -wait tcp://localhost:10000 -timeout 15m -wait-retry-interval 5s
+      - run:
+          name: "Run Tests - Spark"
+          command: |
+            . venv/bin/activate
+            dbt build -t spark --project-dir $DBT_PROJECT_DIR
+
+      - store_artifacts:
+          path: ./logs
+
 workflows:
   version: 2
   test-all:
     jobs:
       - hold:
           type: approval
-      - integration-tests:
+      - integration-tests-core:
+          requires:
+            - hold
+      - integration-tests-spark-thrift:
           requires:
             - hold

integration_tests/ci/profiles.yml

@@ -37,4 +37,15 @@ integration_tests:
       type: duckdb
       path: ":memory:"
 
+    spark:
+      type: spark
+      method: thrift
+      host: 127.0.0.1
+      port: 10000
+      user: dbt
+      schema: analytics
+      connect_retries: 5
+      connect_timeout: 60
+      retry_all: true
+
   target: postgres

integration_tests/docker-compose.yml

@@ -3,7 +3,6 @@ services:
 
   dbt-spark3-thrift:
     image: godatadriven/spark:3.1.1
-    # image: apache/spark-py
     ports:
       - "10000:10000"
       - "4040:4040"