Skip to content

Commit 477e211

Browse files
committed
Breakout Spark CI
1 parent b221a8a commit 477e211

File tree

3 files changed

+79
-7
lines changed

3 files changed

+79
-7
lines changed

.circleci/config.yml

+68-6
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ version: 2.1
22

33
jobs:
44

5-
integration-tests:
5+
integration-tests-core:
6+
67
docker:
78
- image: cimg/python:3.9.9
89
- image: cimg/postgres:14.0
@@ -13,19 +14,27 @@ jobs:
1314
DBT_PROFILES_DIR: ./integration_tests/ci
1415
DBT_PROJECT_DIR: ./integration_tests
1516
BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
16-
DBT_VERSION: 1.6.0
17+
DBT_VERSION: 1.6.*
1718

1819
steps:
1920
- checkout
20-
- run:
21-
name: Install Python packages
21+
- run: &pip-install-core
22+
name: Install core Python packages & dbt-core
2223
command: |
2324
python3 -m venv venv
2425
. venv/bin/activate
2526
pip install -U pip setuptools wheel
26-
pip install dbt-core==$DBT_VERSION dbt-postgres==$DBT_VERSION dbt-bigquery==$DBT_VERSION dbt-snowflake==$DBT_VERSION dbt-duckdb==$DBT_VERSION
27+
pip install "dbt-core==$DBT_VERSION"
2728
2829
- run:
30+
name: Install dbt adapter packages
31+
command: |
32+
python3 -m venv venv
33+
. venv/bin/activate
34+
pip install "dbt-postgres==$DBT_VERSION" "dbt-bigquery==$DBT_VERSION" "dbt-snowflake==$DBT_VERSION"
35+
pip install "dbt-duckdb==$DBT_VERSION"
36+
37+
- run: &dbt-deps
2938
name: Install dbt dependencies
3039
command: |
3140
. venv/bin/activate
@@ -74,12 +83,65 @@ jobs:
7483
- store_artifacts:
7584
path: ./logs
7685

86+
integration-tests-spark-thrift:
87+
88+
docker:
89+
- image: cimg/python:3.9.9
90+
- image: godatadriven/spark:3.1.1
91+
environment:
92+
WAIT_FOR: localhost:5432
93+
command: >
94+
--class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
95+
--name Thrift JDBC/ODBC Server
96+
- image: postgres:9.6.17-alpine
97+
environment:
98+
POSTGRES_USER: dbt
99+
POSTGRES_PASSWORD: dbt
100+
POSTGRES_DB: metastore
101+
102+
resource_class: small
103+
104+
environment:
105+
DBT_PROFILES_DIR: ./integration_tests/ci
106+
DBT_PROJECT_DIR: ./integration_tests
107+
DBT_VERSION: 1.6.*
108+
109+
steps:
110+
- checkout
111+
- run:
112+
name: Install Ubuntu packages
113+
command: |
114+
sudo apt-get update
115+
sudo apt-get install libsasl2-dev libsasl2-2
116+
- run: *pip-install-core
117+
- run:
118+
name: Install dbt adapter packages
119+
command: |
120+
python3 -m venv venv
121+
. venv/bin/activate
122+
pip install dbt-spark "dbt-spark[PyHive]"
123+
- run: *dbt-deps
124+
- run:
125+
name: Wait for Spark-Thrift
126+
command: dockerize -wait tcp://localhost:10000 -timeout 15m -wait-retry-interval 5s
127+
- run:
128+
name: "Run Tests - Spark"
129+
command: |
130+
. venv/bin/activate
131+
dbt build -t spark --project-dir $DBT_PROJECT_DIR
132+
133+
- store_artifacts:
134+
path: ./logs
135+
77136
workflows:
78137
version: 2
79138
test-all:
80139
jobs:
81140
- hold:
82141
type: approval
83-
- integration-tests:
142+
- integration-tests-core:
143+
requires:
144+
- hold
145+
- integration-tests-spark-thrift:
84146
requires:
85147
- hold

integration_tests/ci/profiles.yml

+11
Original file line numberDiff line numberDiff line change
@@ -37,4 +37,15 @@ integration_tests:
3737
type: duckdb
3838
path: ":memory:"
3939

40+
spark:
41+
type: spark
42+
method: thrift
43+
host: 127.0.0.1
44+
port: 10000
45+
user: dbt
46+
schema: analytics
47+
connect_retries: 5
48+
connect_timeout: 60
49+
retry_all: true
50+
4051
target: postgres

integration_tests/docker-compose.yml

-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ services:
33

44
dbt-spark3-thrift:
55
image: godatadriven/spark:3.1.1
6-
# image: apache/spark-py
76
ports:
87
- "10000:10000"
98
- "4040:4040"

0 commit comments

Comments
 (0)