Skip to content

Commit ae6e1b4

Browse files
dbeatty10jtcohen6
dbeatty10
and
jtcohen6
authored
Lift + shift for cross-db macros (#597)
* Initialize lift + shift, dateadd + datediff * Install branches of all plugins * Use dispatch for passthrough behavior * Only support dbt-core >= 1.2 * Switch to alternative dev branches * Lift and shift cross-database macros, fixtures, and tests into dbt-core and adapters * Test `current_timestamp()` adapter definition vs. original dbt-utils * Split out the `current_timestamp` work into its own separate effort (#599) * Reinstate original `current_timestamp` implementation * Reinstate original `type_*` implementations * Reinstate original `current_timestamp_in_utc` implementation * Kick out the `current_timestamp` + `type_*` tests * Preserve `dbt_utils` as one more step along the dispatch train * Remove branch identifiers Co-authored-by: Jeremy Cohen <jeremy@dbtlabs.com>
1 parent ac072a3 commit ae6e1b4

File tree

80 files changed

+222
-2526
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+222
-2526
lines changed

‎dev-requirements.txt

+6-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
11
pytest
22
pytest-dotenv
3-
dbt-tests-adapter
3+
git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
4+
git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter
5+
git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-postgres&subdirectory=plugins/postgres
6+
git+https://github.com/dbt-labs/dbt-redshift.git
7+
git+https://github.com/dbt-labs/dbt-snowflake.git
8+
git+https://github.com/dbt-labs/dbt-bigquery.git

‎macros/cross_db_utils/any_value.sql

+4-12
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,9 @@
1+
{# This is here for backwards compatibility only #}
2+
13
{% macro any_value(expression) -%}
24
{{ return(adapter.dispatch('any_value', 'dbt_utils') (expression)) }}
35
{% endmacro %}
46

5-
67
{% macro default__any_value(expression) -%}
7-
8-
any_value({{ expression }})
9-
10-
{%- endmacro %}
11-
12-
13-
{% macro postgres__any_value(expression) -%}
14-
{#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#}
15-
min({{ expression }})
16-
17-
{%- endmacro %}
8+
{{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}
9+
{% endmacro %}

‎macros/cross_db_utils/bool_or.sql

+4-19
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,9 @@
1+
{# This is here for backwards compatibility only #}
2+
13
{% macro bool_or(expression) -%}
24
{{ return(adapter.dispatch('bool_or', 'dbt_utils') (expression)) }}
35
{% endmacro %}
46

5-
67
{% macro default__bool_or(expression) -%}
7-
8-
bool_or({{ expression }})
9-
10-
{%- endmacro %}
11-
12-
13-
{% macro snowflake__bool_or(expression) -%}
14-
15-
boolor_agg({{ expression }})
16-
17-
{%- endmacro %}
18-
19-
20-
{% macro bigquery__bool_or(expression) -%}
21-
22-
logical_or({{ expression }})
23-
24-
{%- endmacro %}
8+
{{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}
9+
{% endmacro %}

‎macros/cross_db_utils/cast_bool_to_text.sql

+3-9
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,9 @@
1+
{# This is here for backwards compatibility only #}
2+
13
{% macro cast_bool_to_text(field) %}
24
{{ adapter.dispatch('cast_bool_to_text', 'dbt_utils') (field) }}
35
{% endmacro %}
46

5-
67
{% macro default__cast_bool_to_text(field) %}
7-
cast({{ field }} as {{ dbt_utils.type_string() }})
8-
{% endmacro %}
9-
10-
{% macro redshift__cast_bool_to_text(field) %}
11-
case
12-
when {{ field }} is true then 'true'
13-
when {{ field }} is false then 'false'
14-
end::text
8+
{{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}
159
{% endmacro %}

‎macros/cross_db_utils/concat.sql

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
1+
{# This is here for backwards compatibility only #}
22

33
{% macro concat(fields) -%}
44
{{ return(adapter.dispatch('concat', 'dbt_utils')(fields)) }}
55
{%- endmacro %}
66

77
{% macro default__concat(fields) -%}
8-
{{ fields|join(' || ') }}
8+
{{ return(adapter.dispatch('concat', 'dbt')(fields)) }}
99
{%- endmacro %}

‎macros/cross_db_utils/date_trunc.sql

+3-9
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,9 @@
1+
{# This is here for backwards compatibility only #}
2+
13
{% macro date_trunc(datepart, date) -%}
24
{{ return(adapter.dispatch('date_trunc', 'dbt_utils') (datepart, date)) }}
35
{%- endmacro %}
46

57
{% macro default__date_trunc(datepart, date) -%}
6-
date_trunc('{{datepart}}', {{date}})
7-
{%- endmacro %}
8-
9-
{% macro bigquery__date_trunc(datepart, date) -%}
10-
timestamp_trunc(
11-
cast({{date}} as timestamp),
12-
{{datepart}}
13-
)
14-
8+
{{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}
159
{%- endmacro %}

‎macros/cross_db_utils/dateadd.sql

+3-31
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,9 @@
1+
{# This is here for backwards compatibility only #}
2+
13
{% macro dateadd(datepart, interval, from_date_or_timestamp) %}
24
{{ return(adapter.dispatch('dateadd', 'dbt_utils')(datepart, interval, from_date_or_timestamp)) }}
35
{% endmacro %}
46

5-
67
{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}
7-
8-
dateadd(
9-
{{ datepart }},
10-
{{ interval }},
11-
{{ from_date_or_timestamp }}
12-
)
13-
14-
{% endmacro %}
15-
16-
17-
{% macro bigquery__dateadd(datepart, interval, from_date_or_timestamp) %}
18-
19-
datetime_add(
20-
cast( {{ from_date_or_timestamp }} as datetime),
21-
interval {{ interval }} {{ datepart }}
22-
)
23-
24-
{% endmacro %}
25-
26-
{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}
27-
28-
{{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))
29-
30-
{% endmacro %}
31-
32-
{# redshift should use default instead of postgres #}
33-
{% macro redshift__dateadd(datepart, interval, from_date_or_timestamp) %}
34-
35-
{{ return(dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp)) }}
36-
8+
{{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}
379
{% endmacro %}

‎macros/cross_db_utils/datediff.sql

+5-62
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,9 @@
1+
{# This is here for backwards compatibility only #}
2+
13
{% macro datediff(first_date, second_date, datepart) %}
24
{{ return(adapter.dispatch('datediff', 'dbt_utils')(first_date, second_date, datepart)) }}
35
{% endmacro %}
46

5-
6-
{% macro default__datediff(first_date, second_date, datepart) -%}
7-
8-
datediff(
9-
{{ datepart }},
10-
{{ first_date }},
11-
{{ second_date }}
12-
)
13-
14-
{%- endmacro %}
15-
16-
17-
{% macro bigquery__datediff(first_date, second_date, datepart) -%}
18-
19-
datetime_diff(
20-
cast({{second_date}} as datetime),
21-
cast({{first_date}} as datetime),
22-
{{datepart}}
23-
)
24-
25-
{%- endmacro %}
26-
27-
{% macro postgres__datediff(first_date, second_date, datepart) -%}
28-
29-
{% if datepart == 'year' %}
30-
(date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))
31-
{% elif datepart == 'quarter' %}
32-
({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))
33-
{% elif datepart == 'month' %}
34-
({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))
35-
{% elif datepart == 'day' %}
36-
(({{second_date}})::date - ({{first_date}})::date)
37-
{% elif datepart == 'week' %}
38-
({{ dbt_utils.datediff(first_date, second_date, 'day') }} / 7 + case
39-
when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then
40-
case when {{first_date}} <= {{second_date}} then 0 else -1 end
41-
else
42-
case when {{first_date}} <= {{second_date}} then 1 else 0 end
43-
end)
44-
{% elif datepart == 'hour' %}
45-
({{ dbt_utils.datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))
46-
{% elif datepart == 'minute' %}
47-
({{ dbt_utils.datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))
48-
{% elif datepart == 'second' %}
49-
({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))
50-
{% elif datepart == 'millisecond' %}
51-
({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))
52-
{% elif datepart == 'microsecond' %}
53-
({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))
54-
{% else %}
55-
{{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }}
56-
{% endif %}
57-
58-
{%- endmacro %}
59-
60-
61-
{# redshift should use default instead of postgres #}
62-
{% macro redshift__datediff(first_date, second_date, datepart) -%}
63-
64-
{{ return(dbt_utils.default__datediff(first_date, second_date, datepart)) }}
65-
66-
{%- endmacro %}
7+
{% macro default__datediff(first_date, second_date, datepart) %}
8+
{{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}
9+
{% endmacro %}

‎macros/cross_db_utils/escape_single_quotes.sql

+5-14
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,9 @@
1+
{# This is here for backwards compatibility only #}
2+
13
{% macro escape_single_quotes(expression) %}
24
{{ return(adapter.dispatch('escape_single_quotes', 'dbt_utils') (expression)) }}
35
{% endmacro %}
46

5-
{# /*Default to replacing a single apostrophe with two apostrophes: they're -> they''re*/ #}
6-
{% macro default__escape_single_quotes(expression) -%}
7-
{{ expression | replace("'","''") }}
8-
{%- endmacro %}
9-
10-
{# /*Snowflake uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #}
11-
{% macro snowflake__escape_single_quotes(expression) -%}
12-
{{ expression | replace("'", "\\'") }}
13-
{%- endmacro %}
14-
15-
{# /*BigQuery uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #}
16-
{% macro bigquery__escape_single_quotes(expression) -%}
17-
{{ expression | replace("'", "\\'") }}
18-
{%- endmacro %}
7+
{% macro default__escape_single_quotes(expression) %}
8+
{{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}
9+
{% endmacro %}

‎macros/cross_db_utils/except.sql

+3-10
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,9 @@
1+
{# This is here for backwards compatibility only #}
2+
13
{% macro except() %}
24
{{ return(adapter.dispatch('except', 'dbt_utils')()) }}
35
{% endmacro %}
46

5-
67
{% macro default__except() %}
7-
8-
except
9-
8+
{{ return(adapter.dispatch('except', 'dbt')()) }}
109
{% endmacro %}
11-
12-
{% macro bigquery__except() %}
13-
14-
except distinct
15-
16-
{% endmacro %}

‎macros/cross_db_utils/hash.sql

+3-7
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
1+
{# This is here for backwards compatibility only #}
2+
13
{% macro hash(field) -%}
24
{{ return(adapter.dispatch('hash', 'dbt_utils') (field)) }}
35
{%- endmacro %}
46

5-
67
{% macro default__hash(field) -%}
7-
md5(cast({{field}} as {{dbt_utils.type_string()}}))
8-
{%- endmacro %}
9-
10-
11-
{% macro bigquery__hash(field) -%}
12-
to_hex({{dbt_utils.default__hash(field)}})
8+
{{ return(adapter.dispatch('hash', 'dbt') (field)) }}
139
{%- endmacro %}

‎macros/cross_db_utils/intersect.sql

+3-10
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,9 @@
1+
{# This is here for backwards compatibility only #}
2+
13
{% macro intersect() %}
24
{{ return(adapter.dispatch('intersect', 'dbt_utils')()) }}
35
{% endmacro %}
46

5-
67
{% macro default__intersect() %}
7-
8-
intersect
9-
10-
{% endmacro %}
11-
12-
{% macro bigquery__intersect() %}
13-
14-
intersect distinct
15-
8+
{{ return(adapter.dispatch('intersect', 'dbt')()) }}
169
{% endmacro %}

‎macros/cross_db_utils/last_day.sql

+4-35
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
{# This is here for backwards compatibility only #}
2+
13
/*
24
This function has been tested with dateparts of month and quarters. Further
35
testing is required to validate that it will work on other dateparts.
@@ -7,39 +9,6 @@ testing is required to validate that it will work on other dateparts.
79
{{ return(adapter.dispatch('last_day', 'dbt_utils') (date, datepart)) }}
810
{% endmacro %}
911

10-
11-
{%- macro default_last_day(date, datepart) -%}
12-
cast(
13-
{{dbt_utils.dateadd('day', '-1',
14-
dbt_utils.dateadd(datepart, '1', dbt_utils.date_trunc(datepart, date))
15-
)}}
16-
as date)
17-
{%- endmacro -%}
18-
19-
20-
{% macro default__last_day(date, datepart) -%}
21-
{{dbt_utils.default_last_day(date, datepart)}}
22-
{%- endmacro %}
23-
24-
25-
{% macro postgres__last_day(date, datepart) -%}
26-
27-
{%- if datepart == 'quarter' -%}
28-
-- postgres dateadd does not support quarter interval.
29-
cast(
30-
{{dbt_utils.dateadd('day', '-1',
31-
dbt_utils.dateadd('month', '3', dbt_utils.date_trunc(datepart, date))
32-
)}}
33-
as date)
34-
{%- else -%}
35-
{{dbt_utils.default_last_day(date, datepart)}}
36-
{%- endif -%}
37-
38-
{%- endmacro %}
39-
40-
{# redshift should use default instead of postgres #}
41-
{% macro redshift__last_day(date, datepart) %}
42-
43-
{{ return(dbt_utils.default__last_day(date, datepart)) }}
44-
12+
{% macro default__last_day(date, datepart) %}
13+
{{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}
4514
{% endmacro %}

‎macros/cross_db_utils/length.sql

+5-17
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,9 @@
1+
{# This is here for backwards compatibility only #}
2+
13
{% macro length(expression) -%}
24
{{ return(adapter.dispatch('length', 'dbt_utils') (expression)) }}
35
{% endmacro %}
46

5-
6-
{% macro default__length(expression) %}
7-
8-
length(
9-
{{ expression }}
10-
)
11-
12-
{%- endmacro -%}
13-
14-
15-
{% macro redshift__length(expression) %}
16-
17-
len(
18-
{{ expression }}
19-
)
20-
21-
{%- endmacro -%}
7+
{% macro default__length(expression) -%}
8+
{{ return(adapter.dispatch('length', 'dbt') (expression)) }}
9+
{% endmacro %}

0 commit comments

Comments
 (0)