@@ -2,25 +2,10 @@ version: 2.1
2
2
3
3
jobs :
4
4
5
- integration-tests :
5
+ integration-tests-core :
6
6
docker :
7
7
- image : cimg/python:3.9.9
8
8
- image : cimg/postgres:14.0
9
- - image : godatadriven/spark:3.1.1
10
- ports :
11
- - " 10000:10000"
12
- - " 4040:4040"
13
- depends_on :
14
- - dbt-hive-metastore
15
- command : >
16
- --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
17
- --name Thrift JDBC/ODBC Server
18
- volumes:
19
- - ./.spark-warehouse/:/spark-warehouse/
20
- - ./docker/hive-site.xml:/usr/spark/conf/hive-site.xml
21
- - ./docker/spark-defaults.conf:/usr/spark/conf/spark-defaults.conf
22
- environment:
23
- - WAIT_FOR=dbt-hive-metastore:5432
24
9
25
10
resource_class : small
26
11
39
24
. venv/bin/activate
40
25
pip install -U pip setuptools wheel
41
26
pip install docker-compose
42
- pip install dbt-core==$DBT_VERSION dbt-postgres==$DBT_VERSION dbt-bigquery==$DBT_VERSION dbt-snowflake==$DBT_VERSION dbt-duckdb==$DBT_VERSION
27
+ pip install dbt-core==$DBT_VERSION dbt-postgres==$DBT_VERSION dbt-bigquery==$DBT_VERSION dbt-snowflake==$DBT_VERSION
28
+ pip install dbt-duckdb==$DBT_VERSION
43
29
44
30
- run :
45
31
name : Install dbt dependencies
87
73
. venv/bin/activate
88
74
dbt build -t duckdb --project-dir $DBT_PROJECT_DIR
89
75
76
+ - store_artifacts :
77
+ path : ./logs
78
+
79
+ integration-tests-spark-thrift :
80
+ docker :
81
+ - image : cimg/python:3.9.9
82
+ - image : godatadriven/spark:3.1.1
83
+ environment :
84
+ WAIT_FOR : localhost:5432
85
+ command : >
86
+ --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
87
+ --name Thrift JDBC/ODBC Server
88
+ - image : postgres:9.6.17-alpine
89
+ environment :
90
+ POSTGRES_USER : dbt
91
+ POSTGRES_PASSWORD : dbt
92
+ POSTGRES_DB : metastore
93
+ resource_class : small
94
+
95
+ environment :
96
+ DBT_PROFILES_DIR : ./integration_tests/ci
97
+ DBT_PROJECT_DIR : ./integration_tests
98
+ DBT_VERSION : 1.6.0
99
+
100
+ steps :
101
+ - checkout
102
+ - run :
103
+ name : Install Ubuntu dev packages
104
+ command : |
105
+ sudo apt-get update
106
+ sudo apt-get install libsasl2-dev libsasl2-2
107
+ - run :
108
+ name : Install Python packages
109
+ command : |
110
+ python3 -m venv venv
111
+ . venv/bin/activate
112
+ pip install -U pip setuptools wheel
113
+ pip install dbt-core==$DBT_VERSION
114
+ pip install dbt-spark "dbt-spark[PyHive]"
115
+
116
+ - run :
117
+ name : Install dbt dependencies
118
+ command : |
119
+ . venv/bin/activate
120
+ dbt deps --project-dir $DBT_PROJECT_DIR
121
+ - run :
122
+ name : Wait for Spark-Thrift
123
+ command : dockerize -wait tcp://localhost:10000 -timeout 15m -wait-retry-interval 5s
90
124
- run :
91
125
name : " Run Tests - Spark"
92
126
command : |
@@ -102,6 +136,9 @@ workflows:
102
136
jobs :
103
137
- hold :
104
138
type : approval
105
- - integration-tests :
139
+ # - integration-tests-core:
140
+ # requires:
141
+ # - hold
142
+ - integration-tests-spark-thrift :
106
143
requires :
107
144
- hold
0 commit comments