Skip to content

Commit 21d4830

Browse files
committed
Add Spark to CircleCI
1 parent c45e881 commit 21d4830

File tree

1 file changed

+55
-18
lines changed

1 file changed

+55
-18
lines changed

.circleci/config.yml

+55-18
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,10 @@ version: 2.1
22

33
jobs:
44

5-
integration-tests:
5+
integration-tests-core:
66
docker:
77
- image: cimg/python:3.9.9
88
- image: cimg/postgres:14.0
9-
- image: godatadriven/spark:3.1.1
10-
ports:
11-
- "10000:10000"
12-
- "4040:4040"
13-
depends_on:
14-
- dbt-hive-metastore
15-
command: >
16-
--class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
17-
--name Thrift JDBC/ODBC Server
18-
volumes:
19-
- ./.spark-warehouse/:/spark-warehouse/
20-
- ./docker/hive-site.xml:/usr/spark/conf/hive-site.xml
21-
- ./docker/spark-defaults.conf:/usr/spark/conf/spark-defaults.conf
22-
environment:
23-
- WAIT_FOR=dbt-hive-metastore:5432
249

2510
resource_class: small
2611

@@ -39,7 +24,8 @@ jobs:
3924
. venv/bin/activate
4025
pip install -U pip setuptools wheel
4126
pip install docker-compose
42-
pip install dbt-core==$DBT_VERSION dbt-postgres==$DBT_VERSION dbt-bigquery==$DBT_VERSION dbt-snowflake==$DBT_VERSION dbt-duckdb==$DBT_VERSION
27+
pip install dbt-core==$DBT_VERSION dbt-postgres==$DBT_VERSION dbt-bigquery==$DBT_VERSION dbt-snowflake==$DBT_VERSION
28+
pip install dbt-duckdb==$DBT_VERSION
4329
4430
- run:
4531
name: Install dbt dependencies
@@ -87,6 +73,54 @@ jobs:
8773
. venv/bin/activate
8874
dbt build -t duckdb --project-dir $DBT_PROJECT_DIR
8975
76+
- store_artifacts:
77+
path: ./logs
78+
79+
integration-tests-spark-thrift:
80+
docker:
81+
- image: cimg/python:3.9.9
82+
- image: godatadriven/spark:3.1.1
83+
environment:
84+
WAIT_FOR: localhost:5432
85+
command: >
86+
--class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
87+
--name Thrift JDBC/ODBC Server
88+
- image: postgres:9.6.17-alpine
89+
environment:
90+
POSTGRES_USER: dbt
91+
POSTGRES_PASSWORD: dbt
92+
POSTGRES_DB: metastore
93+
resource_class: small
94+
95+
environment:
96+
DBT_PROFILES_DIR: ./integration_tests/ci
97+
DBT_PROJECT_DIR: ./integration_tests
98+
DBT_VERSION: 1.6.0
99+
100+
steps:
101+
- checkout
102+
- run:
103+
name: Install Ubuntu dev packages
104+
command: |
105+
sudo apt-get update
106+
sudo apt-get install libsasl2-dev libsasl2-2
107+
- run:
108+
name: Install Python packages
109+
command: |
110+
python3 -m venv venv
111+
. venv/bin/activate
112+
pip install -U pip setuptools wheel
113+
pip install dbt-core==$DBT_VERSION
114+
pip install dbt-spark "dbt-spark[PyHive]"
115+
116+
- run:
117+
name: Install dbt dependencies
118+
command: |
119+
. venv/bin/activate
120+
dbt deps --project-dir $DBT_PROJECT_DIR
121+
- run:
122+
name: Wait for Spark-Thrift
123+
command: dockerize -wait tcp://localhost:10000 -timeout 15m -wait-retry-interval 5s
90124
- run:
91125
name: "Run Tests - Spark"
92126
command: |
@@ -102,6 +136,9 @@ workflows:
102136
jobs:
103137
- hold:
104138
type: approval
105-
- integration-tests:
139+
# - integration-tests-core:
140+
# requires:
141+
# - hold
142+
- integration-tests-spark-thrift:
106143
requires:
107144
- hold

0 commit comments

Comments
 (0)