Use built-in adapter functionality for datatypes (#586)

* Use built-in adapter functionality for datatypes

* Experiment with adding functional tests

* Try to fix pip install

* Missing pytest plugin

* Refactor tests, passing on BQ

* Refactor again

* Revert bq target changes

* Migrate pieces into core + plugins

* Revert accidental changes to bash files

* Some code cleanup

* Restore needed import. Add speedup

* Add back imports from `main` branches

Co-authored-by: Doug Beatty <doug.beatty@dbtlabs.com>

Author: jtcohen6

dev-requirements.txt

@@ -6,3 +6,4 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-postgres&subdirectory=plugi
 git+https://github.com/dbt-labs/dbt-redshift.git
 git+https://github.com/dbt-labs/dbt-snowflake.git
 git+https://github.com/dbt-labs/dbt-bigquery.git
+pytest-xdist

macros/cross_db_utils/datatypes.sql

@@ -1,99 +1,67 @@
+{# These macros have been moved into dbt-core #}
+{# Here for backwards compatibility ONLY #}
+
 {# string  -------------------------------------------------     #}
 
 {%- macro type_string() -%}
-  {{ return(adapter.dispatch('type_string', 'dbt_utils')()) }}
+    {{ return(adapter.dispatch('type_string', 'dbt_utils')()) }}
 {%- endmacro -%}
 
 {% macro default__type_string() %}
-    string
+    {{ return(adapter.dispatch('type_string', 'dbt')()) }}
 {% endmacro %}
 
-{%- macro redshift__type_string() -%}
-    varchar
-{%- endmacro -%}
-
-{% macro postgres__type_string() %}
-    varchar
-{% endmacro %}
-
-{% macro snowflake__type_string() %}
-    varchar
-{% endmacro %}
-
-
 
 {# timestamp  -------------------------------------------------     #}
 
 {%- macro type_timestamp() -%}
-  {{ return(adapter.dispatch('type_timestamp', 'dbt_utils')()) }}
+    {{ return(adapter.dispatch('type_timestamp', 'dbt_utils')()) }}
 {%- endmacro -%}
 
 {% macro default__type_timestamp() %}
-    timestamp
-{% endmacro %}
-
-{% macro postgres__type_timestamp() %}
-    timestamp without time zone
-{% endmacro %}
-
-{% macro snowflake__type_timestamp() %}
-    timestamp_ntz
+    {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}
 {% endmacro %}
 
 
 {# float  -------------------------------------------------     #}
 
 {%- macro type_float() -%}
-  {{ return(adapter.dispatch('type_float', 'dbt_utils')()) }}
+    {{ return(adapter.dispatch('type_float', 'dbt_utils')()) }}
 {%- endmacro -%}
 
 {% macro default__type_float() %}
-    float
+    {{ return(adapter.dispatch('type_float', 'dbt')()) }}
 {% endmacro %}
 
-{% macro bigquery__type_float() %}
-    float64
-{% endmacro %}
 
 {# numeric  ------------------------------------------------     #}
 
 {%- macro type_numeric() -%}
-  {{ return(adapter.dispatch('type_numeric', 'dbt_utils')()) }}
+    {{ return(adapter.dispatch('type_numeric', 'dbt_utils')()) }}
 {%- endmacro -%}
 
 {% macro default__type_numeric() %}
-    numeric(28, 6)
-{% endmacro %}
-
-{% macro bigquery__type_numeric() %}
-    numeric
+    {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}
 {% endmacro %}
 
 
 {# bigint  -------------------------------------------------     #}
 
 {%- macro type_bigint() -%}
-  {{ return(adapter.dispatch('type_bigint', 'dbt_utils')()) }}
+    {{ return(adapter.dispatch('type_bigint', 'dbt_utils')()) }}
 {%- endmacro -%}
 
 {% macro default__type_bigint() %}
-    bigint
+    {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}
 {% endmacro %}
 
-{% macro bigquery__type_bigint() %}
-    int64
-{% endmacro %}
 
 {# int  -------------------------------------------------     #}
 
 {%- macro type_int() -%}
-  {{ return(adapter.dispatch('type_int', 'dbt_utils')()) }}
+    {{ return(adapter.dispatch('type_int', 'dbt_utils')()) }}
 {%- endmacro -%}
 
 {% macro default__type_int() %}
-    int
-{% endmacro %}
-
-{% macro bigquery__type_int() %}
-    int64
+    {{ return(adapter.dispatch('type_int', 'dbt')()) }}
 {% endmacro %}

run_functional_test.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-python3 -m pytest tests/functional --profile $1
+python3 -m pytest tests/functional -n4 --profile $1

tests/conftest.py

@@ -5,10 +5,10 @@
 
 
 def pytest_addoption(parser):
-    parser.addoption("--profile", action="store", default="apache_spark", type=str)
+    parser.addoption("--profile", action="store", default="postgres", type=str)
 
 
-# Using @pytest.mark.skip_adapter('apache_spark') uses the 'skip_by_adapter_type'
+# Using @pytest.mark.skip_profile('postgres') uses the 'skip_by_profile_type'
 # autouse fixture below
 def pytest_configure(config):
     config.addinivalue_line(

tests/functional/cross_db_utils/base_cross_db_macro.py

@@ -0,0 +1,46 @@
+import os
+import pytest
+from dbt.tests.util import run_dbt, check_relations_equal, get_relation_columns
+from dbt.tests.adapter.utils.data_types.base_data_type_macro import BaseDataTypeMacro
+
+class BaseDbtUtilsBackCompat(BaseDataTypeMacro):
+    # install this repo as a package
+    @pytest.fixture(scope="class")
+    def packages(self):
+        return {"packages": [{"local": os.getcwd()}]}
+
+    # call the macros from the 'dbt_utils' namespace
+    # instead of the unspecified / global namespace
+    def macro_namespace(self):
+        return "dbt_utils"
+
+    # actual test sequence needs to run 'deps' first
+    def test_check_types_assert_match(self, project):
+        run_dbt(['deps'])
+        super().test_check_types_assert_match(project)
+
+
+class BaseLegacyDataTypeMacro(BaseDbtUtilsBackCompat):       
+    def assert_columns_equal(self, project, expected_cols, actual_cols):
+        # we need to be a little more lenient when mapping between 'legacy' and 'new' types that are equivalent
+        # e.g. 'character varying' and 'text'
+        if expected_cols == actual_cols:
+            # cool, no need for jank
+            pass
+        else:
+            # this is pretty janky
+            # our goal here: reasonable confidence that the switch from the legacy version of the dbt_utils.type_{X} macro,
+            # and the new version, will not constitute a breaking change for end users
+            for (expected_col, actual_col) in zip(expected_cols, actual_cols):
+                expected = project.adapter.Column(*expected_col)
+                actual = project.adapter.Column(*actual_col)
+                print(f"Subtle type difference detected: {expected.data_type} vs. {actual.data_type}")
+                if any((
+                    expected.is_string() and actual.is_string(),
+                    expected.is_float() and actual.is_float(),
+                    expected.is_integer() and actual.is_integer(),
+                    expected.is_numeric() and actual.is_numeric(),
+                )):
+                    pytest.xfail()
+                else:
+                    pytest.fail()

tests/functional/cross_db_utils/fixture_cross_db_macro.py

@@ -0,0 +1,26 @@
+import pytest
+from tests.functional.data_type.base_data_type import BaseDbtUtilsBackCompat, BaseLegacyDataTypeMacro
+from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt
+
+
+class TestTypeBigInt(BaseDbtUtilsBackCompat, BaseTypeBigInt):
+    pass
+
+
+# previous dbt_utils code
+macros__legacy_sql = """
+{% macro default__type_bigint() %}
+    bigint
+{% endmacro %}
+
+{% macro bigquery__type_bigint() %}
+    int64
+{% endmacro %}
+"""
+
+class TestTypeBigIntLegacy(BaseLegacyDataTypeMacro, BaseTypeBigInt):
+    @pytest.fixture(scope="class")
+    def macros(self):
+        return {
+            "legacy.sql": macros__legacy_sql
+        }

tests/functional/data_type/base_data_type.py

@@ -0,0 +1,27 @@
+import pytest
+from tests.functional.data_type.base_data_type import BaseDbtUtilsBackCompat, BaseLegacyDataTypeMacro
+from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat
+
+
+class TestTypeFloat(BaseDbtUtilsBackCompat, BaseTypeFloat):
+    pass
+
+
+# previous dbt_utils code
+macros__legacy_sql = """
+{% macro default__type_float() %}
+    float
+{% endmacro %}
+
+{% macro bigquery__type_float() %}
+    float64
+{% endmacro %}
+"""
+
+
+class TestTypeFloatLegacy(BaseLegacyDataTypeMacro, BaseTypeFloat):
+    @pytest.fixture(scope="class")
+    def macros(self):
+        return {
+            "legacy.sql": macros__legacy_sql
+        }

tests/functional/data_type/test_type_bigint.py

@@ -0,0 +1,27 @@
+import pytest
+from tests.functional.data_type.base_data_type import BaseDbtUtilsBackCompat, BaseLegacyDataTypeMacro
+from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt
+
+
+class TestTypeInt(BaseDbtUtilsBackCompat, BaseTypeInt):
+    pass
+
+
+# previous dbt_utils code
+macros__legacy_sql = """
+{% macro default__type_int() %}
+    int
+{% endmacro %}
+
+{% macro bigquery__type_int() %}
+    int64
+{% endmacro %}
+"""
+
+
+class TestTypeFloatLegacy(BaseLegacyDataTypeMacro, BaseTypeInt):
+    @pytest.fixture(scope="class")
+    def macros(self):
+        return {
+            "legacy.sql": macros__legacy_sql
+        }

tests/functional/data_type/test_type_float.py

@@ -0,0 +1,49 @@
+import pytest
+from tests.functional.data_type.base_data_type import BaseDbtUtilsBackCompat, BaseLegacyDataTypeMacro
+from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric
+
+
+@pytest.mark.skip_profile('bigquery')
+class TestTypeNumeric(BaseDbtUtilsBackCompat, BaseTypeNumeric):
+    pass
+
+
+@pytest.mark.only_profile('bigquery')
+class TestBigQueryTypeNumeric(BaseDbtUtilsBackCompat, BaseTypeNumeric):
+    def numeric_fixture_type(self):
+        return "numeric"
+
+
+# previous dbt_utils code
+macros__legacy_sql = """
+{% macro default__type_numeric() %}
+    numeric(28, 6)
+{% endmacro %}
+{% macro bigquery__type_numeric() %}
+    numeric
+{% endmacro %}
+"""
+
+
+class BaseTypeNumericLegacy(BaseLegacyDataTypeMacro, BaseTypeNumeric):
+    @pytest.fixture(scope="class")
+    def macros(self):
+        return {
+            "legacy.sql": macros__legacy_sql
+        }
+
+
+@pytest.mark.skip_profile('bigquery')
+class TestTypeNumeric(BaseTypeNumeric):
+    pass
+
+
+@pytest.mark.skip_profile('bigquery')
+class TestTypeNumericLegacy(BaseTypeNumericLegacy):
+    pass
+
+
+@pytest.mark.only_profile('bigquery')
+class TestBigQueryTypeNumericLegacy(BaseTypeNumericLegacy):
+    def numeric_fixture_type(self):
+        return "numeric"

tests/functional/data_type/test_type_int.py

@@ -0,0 +1,35 @@
+import pytest
+from tests.functional.data_type.base_data_type import BaseDbtUtilsBackCompat, BaseLegacyDataTypeMacro
+from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString
+
+
+class TestTypeInt(BaseDbtUtilsBackCompat, BaseTypeString):
+    pass
+
+
+# previous dbt_utils code
+macros__legacy_sql = """
+{% macro default__type_string() %}
+    string
+{% endmacro %}
+
+{%- macro redshift__type_string() -%}
+    varchar
+{%- endmacro -%}
+
+{% macro postgres__type_string() %}
+    varchar
+{% endmacro %}
+
+{% macro snowflake__type_string() %}
+    varchar
+{% endmacro %}
+"""
+
+
+class TestTypeStringLegacy(BaseLegacyDataTypeMacro, BaseTypeString):
+    @pytest.fixture(scope="class")
+    def macros(self):
+        return {
+            "legacy.sql": macros__legacy_sql
+        }