diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 100521ffbc212..694449982dea6 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -179,7 +179,7 @@ - name: Snowflake destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba dockerRepository: airbyte/destination-snowflake - dockerImageTag: 0.3.24 + dockerImageTag: 0.4.0 documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake icon: snowflake.svg - name: MariaDB ColumnStore diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 266fefa1716d8..a99171e40b825 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3767,7 +3767,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-snowflake:0.3.24" +- dockerImage: "airbyte/destination-snowflake:0.4.0" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake" connectionSpecification: diff --git a/airbyte-integrations/bases/base-normalization/.dockerignore b/airbyte-integrations/bases/base-normalization/.dockerignore index e6fbfb3101a91..a44e07cac6f81 100644 --- a/airbyte-integrations/bases/base-normalization/.dockerignore +++ b/airbyte-integrations/bases/base-normalization/.dockerignore @@ -9,3 +9,4 @@ !dbt-project-template-mysql !dbt-project-template-oracle !dbt-project-template-clickhouse +!dbt-project-template-snowflake diff --git a/airbyte-integrations/bases/base-normalization/.gitignore b/airbyte-integrations/bases/base-normalization/.gitignore index 5e426c453be52..7994f50ee6bea 100644 --- a/airbyte-integrations/bases/base-normalization/.gitignore +++ b/airbyte-integrations/bases/base-normalization/.gitignore @@ -19,13 +19,18 @@ integration_tests/normalization_test_output/**/*.yml # We keep a minimal/restricted subset of sql files for all destinations to avoid noise in diff # Simple Streams !integration_tests/normalization_test_output/**/dedup_exchange_rate*.sql +!integration_tests/normalization_test_output/**/DEDUP_EXCHANGE_RATE*.sql !integration_tests/normalization_test_output/**/exchange_rate.sql +!integration_tests/normalization_test_output/**/EXCHANGE_RATE.sql !integration_tests/normalization_test_output/**/test_simple_streams/first_output/airbyte_views/**/multiple_column_names_conflicts_stg.sql # Nested Streams # Parent table !integration_tests/normalization_test_output/**/nested_stream_with*_names_ab*.sql !integration_tests/normalization_test_output/**/nested_stream_with*_names_scd.sql !integration_tests/normalization_test_output/**/nested_stream_with*_names.sql +!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_NAMES_AB*.sql +!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_NAMES_SCD.sql +!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_NAMES.sql # Nested table !integration_tests/normalization_test_output/**/nested_stream_with_*_partition_ab1.sql !integration_tests/normalization_test_output/**/nested_stream_with_*_data_ab1.sql @@ -33,6 +38,12 @@ integration_tests/normalization_test_output/**/*.yml !integration_tests/normalization_test_output/**/nested_stream_with*_data_scd.sql !integration_tests/normalization_test_output/**/nested_stream_with*_partition.sql !integration_tests/normalization_test_output/**/nested_stream_with*_data.sql +!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH_*_PARTITION_AB1.sql +!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH_*_DATA_AB1.sql +!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_PARTITION_SCD.sql +!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_DATA_SCD.sql +!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_PARTITION.sql +!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_DATA.sql # but we keep all sql files for Postgres !integration_tests/normalization_test_output/postgres/**/*.sql diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index eff597c11a3e1..105dee09c2363 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -73,11 +73,16 @@ task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) { configure buildAirbyteDocker('clickhouse') dependsOn assemble } +task airbyteDockerSnowflake(type: Exec, dependsOn: checkSshScriptCopy) { + configure buildAirbyteDocker('snowflake') + dependsOn assemble +} airbyteDocker.dependsOn(airbyteDockerMSSql) airbyteDocker.dependsOn(airbyteDockerMySql) airbyteDocker.dependsOn(airbyteDockerOracle) airbyteDocker.dependsOn(airbyteDockerClickhouse) +airbyteDocker.dependsOn(airbyteDockerSnowflake) task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) { module = "pytest" diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template-snowflake/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template-snowflake/dbt_project.yml new file mode 100644 index 0000000000000..c22ddc2282c25 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template-snowflake/dbt_project.yml @@ -0,0 +1,64 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: "airbyte_utils" +version: "1.0" +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: "normalize" + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true + # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) + # all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + +transient: false + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + +on_schema_change: sync_all_columns + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +dispatch: + - macro_namespace: dbt_utils + search_order: ["airbyte_utils", "dbt_utils"] diff --git a/airbyte-integrations/bases/base-normalization/docker-compose.build.yaml b/airbyte-integrations/bases/base-normalization/docker-compose.build.yaml index 16948b4a22211..628136ffc33a8 100644 --- a/airbyte-integrations/bases/base-normalization/docker-compose.build.yaml +++ b/airbyte-integrations/bases/base-normalization/docker-compose.build.yaml @@ -36,3 +36,10 @@ services: context: . labels: io.airbyte.git-revision: ${GIT_REVISION} + normalization-snowflake: + image: airbyte/normalization-snowflake:${VERSION} + build: + dockerfile: snowflake.Dockerfile + context: . + labels: + io.airbyte.git-revision: ${GIT_REVISION} diff --git a/airbyte-integrations/bases/base-normalization/docker-compose.yaml b/airbyte-integrations/bases/base-normalization/docker-compose.yaml index 8dd94275765be..23edde345b51b 100644 --- a/airbyte-integrations/bases/base-normalization/docker-compose.yaml +++ b/airbyte-integrations/bases/base-normalization/docker-compose.yaml @@ -12,3 +12,5 @@ services: image: airbyte/normalization-oracle:${VERSION} normalization-clickhouse: image: airbyte/normalization-clickhouse:${VERSION} + normalization-snowflake: + image: airbyte/normalization-snowflake:${VERSION} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py index bce4152dc7b3f..bd67df357b284 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py @@ -374,6 +374,8 @@ def get_normalization_image(destination_type: DestinationType) -> str: return "airbyte/normalization-oracle:dev" elif DestinationType.CLICKHOUSE.value == destination_type.value: return "airbyte/normalization-clickhouse:dev" + elif DestinationType.SNOWFLAKE.value == destination_type.value: + return "airbyte/normalization-snowflake:dev" else: return "airbyte/normalization:dev" @@ -445,6 +447,8 @@ def run_check_dbt_command(normalization_image: str, command: str, cwd: str, forc "Configuration paths exist in your dbt_project.yml", # When no cte / view are generated "Error loading config file: .dockercfg: $HOME is not defined", # ignore warning "depends on a node named 'disabled_test' which was not found", # Tests throwing warning because it is disabled + "The requested image's platform (linux/amd64) does not match the detected host platform " + + "(linux/arm64/v8) and no specific platform was requested", # temporary patch until we publish images for arm64 ]: if except_clause in str_line: is_exception = True diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml old mode 100755 new mode 100644 index 9ad8158759001..c22ddc2282c25 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml @@ -4,13 +4,13 @@ # Name your package! Package names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: 'airbyte_utils' -version: '1.0' +name: "airbyte_utils" +version: "1.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. Profiles contain # database connection information, and should be configured in the ~/.dbt/profiles.yml file -profile: 'normalize' +profile: "normalize" # These configurations specify where dbt should look for different types of files. # The `source-paths` config, for example, states that source models can be found @@ -22,18 +22,18 @@ test-paths: ["tests"] data-paths: ["data"] macro-paths: ["macros"] -target-path: "../build" # directory which will store compiled SQL files -log-path: "../logs" # directory which will store DBT logs -modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies -clean-targets: # directories to be removed by `dbt clean` - - "build" - - "dbt_modules" +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" quoting: database: true -# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) -# all schemas should be unquoted + # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) + # all schemas should be unquoted schema: false identifier: true @@ -41,6 +41,7 @@ quoting: # Using these configurations, you can enable or disable models, change how they # are materialized, and more! models: + +transient: false airbyte_utils: +materialized: table generated: @@ -60,4 +61,4 @@ models: dispatch: - macro_namespace: dbt_utils - search_order: ['airbyte_utils', 'dbt_utils'] + search_order: ["airbyte_utils", "dbt_utils"] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql index dbb7eed19d4e4..070ff60ba2f03 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql @@ -1,6 +1,6 @@ - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as + create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as (select * from( -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql index fe425246856a5..d70e72e47308c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql @@ -1,6 +1,6 @@ - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as + create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as (select * from( with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1 as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql index 4ea5abb0f0f21..b550314ec9c29 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql @@ -1,6 +1,6 @@ - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" as + create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" as (select * from( with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1 as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql index a5e6ac9fdc401..9bd584e0f1d23 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql @@ -1,6 +1,6 @@ - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" as + create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" as (select * from( with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1 as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql index 73631957ce269..3bd7c7e79ee22 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -1,6 +1,6 @@ - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" as + create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" as (select * from( -- depends_on: ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_STG') diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml old mode 100755 new mode 100644 index 9ad8158759001..c22ddc2282c25 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml @@ -4,13 +4,13 @@ # Name your package! Package names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: 'airbyte_utils' -version: '1.0' +name: "airbyte_utils" +version: "1.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. Profiles contain # database connection information, and should be configured in the ~/.dbt/profiles.yml file -profile: 'normalize' +profile: "normalize" # These configurations specify where dbt should look for different types of files. # The `source-paths` config, for example, states that source models can be found @@ -22,18 +22,18 @@ test-paths: ["tests"] data-paths: ["data"] macro-paths: ["macros"] -target-path: "../build" # directory which will store compiled SQL files -log-path: "../logs" # directory which will store DBT logs -modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies -clean-targets: # directories to be removed by `dbt clean` - - "build" - - "dbt_modules" +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" quoting: database: true -# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) -# all schemas should be unquoted + # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) + # all schemas should be unquoted schema: false identifier: true @@ -41,6 +41,7 @@ quoting: # Using these configurations, you can enable or disable models, change how they # are materialized, and more! models: + +transient: false airbyte_utils: +materialized: table generated: @@ -60,4 +61,4 @@ models: dispatch: - macro_namespace: dbt_utils - search_order: ['airbyte_utils', 'dbt_utils'] + search_order: ["airbyte_utils", "dbt_utils"] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql index 253d50b73d9c3..e54fe5cedf121 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql @@ -1,6 +1,6 @@ - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE" as + create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE" as (select * from( -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql index 20ae8e46add52..b65fbb50b75e4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -1,6 +1,6 @@ - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" as + create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" as (select * from( -- depends_on: ref('DEDUP_EXCHANGE_RATE_STG') diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql index 83d0d7cc8fd82..8280ecbf2960e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -1,6 +1,6 @@ - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as + create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as (select * from( with __dbt__cte__EXCHANGE_RATE_AB1 as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql index 83d0d7cc8fd82..8280ecbf2960e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -1,6 +1,6 @@ - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as + create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as (select * from( with __dbt__cte__EXCHANGE_RATE_AB1 as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py b/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py index eaae4a18eda09..287789e378104 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py @@ -134,6 +134,8 @@ def setup_test_dir(integration_type: str) -> str: copy_tree("../dbt-project-template-mysql", test_root_dir) elif integration_type == DestinationType.ORACLE.value: copy_tree("../dbt-project-template-oracle", test_root_dir) + elif integration_type == DestinationType.SNOWFLAKE.value: + copy_tree("../dbt-project-template-snowflake", test_root_dir) return test_root_dir diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py b/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py index b411ae74d278f..0af945ede6305 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py @@ -192,6 +192,9 @@ def setup_test_dir(destination_type: DestinationType, test_resource_name: str) - elif destination_type.value == DestinationType.CLICKHOUSE.value: copy_tree("../dbt-project-template-clickhouse", test_root_dir) dbt_project_yaml = "../dbt-project-template-clickhouse/dbt_project.yml" + elif destination_type.value == DestinationType.SNOWFLAKE.value: + copy_tree("../dbt-project-template-snowflake", test_root_dir) + dbt_project_yaml = "../dbt-project-template-snowflake/dbt_project.yml" dbt_test_utils.copy_replace(dbt_project_yaml, os.path.join(test_root_dir, "dbt_project.yml")) return test_root_dir diff --git a/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile b/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile new file mode 100644 index 0000000000000..aa7349270dd38 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile @@ -0,0 +1,33 @@ +FROM fishtownanalytics/dbt:0.21.1 +COPY --from=airbyte/base-airbyte-protocol-python:0.1.1 /airbyte /airbyte + +# Install SSH Tunneling dependencies +RUN apt-get update && apt-get install -y jq sshpass + +WORKDIR /airbyte +COPY entrypoint.sh . +COPY build/sshtunneling.sh . + +WORKDIR /airbyte/normalization_code +COPY normalization ./normalization +COPY setup.py . +COPY dbt-project-template/ ./dbt-template/ +COPY dbt-project-template-snowflake/* ./dbt-template/ + +# Install python dependencies +WORKDIR /airbyte/base_python_structs +RUN pip install . + +WORKDIR /airbyte/normalization_code +RUN pip install . + +WORKDIR /airbyte/normalization_code/dbt-template/ +# Download external dbt dependencies +RUN dbt deps + +WORKDIR /airbyte +ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" +ENTRYPOINT ["/airbyte/entrypoint.sh"] + +LABEL io.airbyte.version=0.1.62 +LABEL io.airbyte.name=airbyte/normalization-snowflake diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 69794dc352d5a..7d84d1b4ac197 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -79,6 +79,8 @@ public abstract class DestinationAcceptanceTest { + private static final String NORMALIZATION_VERSION = "dev"; + private static final String JOB_ID = "0"; private static final int JOB_ATTEMPT = 0; @@ -462,7 +464,7 @@ public void specNormalizationValueShouldBeCorrect() throws Exception { if (normalizationFromSpec) { boolean normalizationRunnerFactorySupportsDestinationImage; try { - NormalizationRunnerFactory.create(workerConfigs, getImageName(), processFactory); + NormalizationRunnerFactory.create(workerConfigs, getImageName(), processFactory, NORMALIZATION_VERSION); normalizationRunnerFactorySupportsDestinationImage = true; } catch (final IllegalStateException e) { normalizationRunnerFactorySupportsDestinationImage = false; @@ -735,13 +737,14 @@ public void testCustomDbtTransformations() throws Exception { final DbtTransformationRunner runner = new DbtTransformationRunner(workerConfigs, processFactory, NormalizationRunnerFactory.create( workerConfigs, getImageName(), - processFactory)); + processFactory, + NORMALIZATION_VERSION)); runner.start(); final Path transformationRoot = Files.createDirectories(jobRoot.resolve("transform")); final OperatorDbt dbtConfig = new OperatorDbt() .withGitRepoUrl("https://github.com/fishtown-analytics/jaffle_shop.git") .withGitRepoBranch("main") - .withDockerImage(NormalizationRunnerFactory.getNormalizationInfoForConnector(getImageName()).getLeft()); + .withDockerImage(NormalizationRunnerFactory.getNormalizationInfoForConnector(getImageName()).getLeft() + ":" + NORMALIZATION_VERSION); // // jaffle_shop is a fictional ecommerce store maintained by fishtownanalytics/dbt. // @@ -802,7 +805,8 @@ void testCustomDbtTransformationsFailure() throws Exception { final DbtTransformationRunner runner = new DbtTransformationRunner(workerConfigs, processFactory, NormalizationRunnerFactory.create( workerConfigs, getImageName(), - processFactory)); + processFactory, + NORMALIZATION_VERSION)); runner.start(); final Path transformationRoot = Files.createDirectories(jobRoot.resolve("transform")); final OperatorDbt dbtConfig = new OperatorDbt() @@ -993,7 +997,8 @@ private List runSync( final NormalizationRunner runner = NormalizationRunnerFactory.create( workerConfigs, getImageName(), - processFactory); + processFactory, + NORMALIZATION_VERSION); runner.start(); final Path normalizationRoot = Files.createDirectories(jobRoot.resolve("normalize")); if (!runner.normalize(JOB_ID, JOB_ATTEMPT, normalizationRoot, destinationConfig.getDestinationConnectionConfiguration(), diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index 05507749bb026..18e82989befd7 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -18,5 +18,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.24 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java index c8d08cc9fea01..6cf51a5102ffa 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java @@ -4,6 +4,9 @@ package io.airbyte.integrations.destination.snowflake; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Preconditions; @@ -22,6 +25,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import java.io.IOException; import java.nio.file.Path; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; @@ -117,10 +121,17 @@ protected List resolveIdentifier(final String identifier) { return result; } - private List retrieveRecordsFromTable(final String tableName, final String schema) throws SQLException, InterruptedException { + private List retrieveRecordsFromTable(final String tableName, final String schema) throws SQLException { return SnowflakeDatabase.getDatabase(getConfig()).bufferedResultSetQuery( - connection -> connection.createStatement() - .executeQuery(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schema, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)), + connection -> { + final ResultSet tableInfo = connection.createStatement() + .executeQuery(String.format("SHOW TABLES LIKE '%s' IN SCHEMA %s;", tableName, schema)); + assertTrue(tableInfo.next()); + // check that we're creating permanent tables. DBT defaults to transient tables, which have `TRANSIENT` as the value for the `kind` column. + assertEquals("TABLE", tableInfo.getString("kind")); + return connection.createStatement() + .executeQuery(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schema, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)); + }, JdbcUtils.getDefaultSourceOperations()::rowToJson); } @@ -163,7 +174,7 @@ public void testSyncWithBillionRecords(final String messagesFilename, final Stri runSyncAndVerifyStateOutput(config, largeNumberRecords, configuredCatalog, false); } - private T parseConfig(final String path, Class clazz) throws IOException { + private T parseConfig(final String path, final Class clazz) throws IOException { return Jsons.deserialize(MoreResources.readResource(path), clazz); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index 12368661ec5ca..2fee47175bd0e 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -33,16 +33,21 @@ public class NormalizationRunnerFactory { .put("airbyte/destination-postgres", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DestinationType.POSTGRES)) .put("airbyte/destination-postgres-strict-encrypt", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DestinationType.POSTGRES)) .put("airbyte/destination-redshift", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DestinationType.REDSHIFT)) - .put("airbyte/destination-snowflake", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DestinationType.SNOWFLAKE)) + .put("airbyte/destination-snowflake", ImmutablePair.of("airbyte/normalization-snowflake", DestinationType.SNOWFLAKE)) + .put("airbyte/destination-clickhouse", ImmutablePair.of("airbyte/normalization-clickhouse", DestinationType.CLICKHOUSE)) + .put("airbyte/destination-clickhouse-strict-encrypt", ImmutablePair.of("airbyte/normalization-clickhouse", DestinationType.CLICKHOUSE)) .build(); - public static NormalizationRunner create(final WorkerConfigs workerConfigs, final String connectorImageName, final ProcessFactory processFactory) { + public static NormalizationRunner create(final WorkerConfigs workerConfigs, + final String connectorImageName, + final ProcessFactory processFactory, + final String normalizationVersion) { final var valuePair = getNormalizationInfoForConnector(connectorImageName); return new DefaultNormalizationRunner( workerConfigs, valuePair.getRight(), processFactory, - String.format("%s:%s", valuePair.getLeft(), NORMALIZATION_VERSION)); + String.format("%s:%s", valuePair.getLeft(), normalizationVersion)); } public static ImmutablePair getNormalizationInfoForConnector(final String connectorImageName) { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java index 761e5e608eb2c..44716c1c566f9 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java @@ -83,7 +83,7 @@ public Void run(final JobRunConfig jobRunConfig, return fullInput; }; - CheckedSupplier, Exception> workerFactory; + final CheckedSupplier, Exception> workerFactory; if (containerOrchestratorEnabled) { workerFactory = getContainerLauncherWorkerFactory(workerConfigs, destinationLauncherConfig, jobRunConfig); @@ -111,9 +111,10 @@ private CheckedSupplier, Exception> getLegacyWork new DbtTransformationRunner( workerConfigs, jobProcessFactory, NormalizationRunnerFactory.create( - workerConfigs, - destinationLauncherConfig.getDockerImage(), - jobProcessFactory))); + workerConfigs, + destinationLauncherConfig.getDockerImage(), + jobProcessFactory, + NormalizationRunnerFactory.NORMALIZATION_VERSION))); } private CheckedSupplier, Exception> getContainerLauncherWorkerFactory( diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index 3f421bdeaae3e..ee01e9075e676 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -80,7 +80,7 @@ public Void normalize(final JobRunConfig jobRunConfig, return fullInput; }; - CheckedSupplier, Exception> workerFactory; + final CheckedSupplier, Exception> workerFactory; if (containerOrchestratorEnabled) { workerFactory = getContainerLauncherWorkerFactory(workerConfigs, destinationLauncherConfig, jobRunConfig); @@ -108,7 +108,8 @@ private CheckedSupplier, Exception> getLegacyWo NormalizationRunnerFactory.create( workerConfigs, destinationLauncherConfig.getDockerImage(), - jobProcessFactory), + jobProcessFactory, + NormalizationRunnerFactory.NORMALIZATION_VERSION), workerEnvironment); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java index e03b2e5cd6bd2..ca7914e177fe6 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java @@ -19,6 +19,7 @@ class NormalizationRunnerFactoryTest { + public static final String NORMALIZATION_VERSION = "dev"; private ProcessFactory processFactory; @BeforeEach @@ -32,10 +33,11 @@ void testMappings() { assertEquals(entry.getValue().getValue(), ((DefaultNormalizationRunner) NormalizationRunnerFactory.create( new WorkerConfigs(new EnvConfigs()), - String.format("%s:0.1.0", entry.getKey()), processFactory)).getDestinationType()); + String.format("%s:0.1.0", entry.getKey()), processFactory, NORMALIZATION_VERSION)).getDestinationType()); } assertThrows(IllegalStateException.class, - () -> NormalizationRunnerFactory.create(new WorkerConfigs(new EnvConfigs()), "airbyte/destination-csv:0.1.0", processFactory)); + () -> NormalizationRunnerFactory.create(new WorkerConfigs(new EnvConfigs()), "airbyte/destination-csv:0.1.0", processFactory, + NORMALIZATION_VERSION)); } } diff --git a/build.gradle b/build.gradle index 91393c72c57f0..ad8535bbc29f6 100644 --- a/build.gradle +++ b/build.gradle @@ -83,6 +83,7 @@ def createSpotlessTarget = { pattern -> 'dbt-project-template-mysql', 'dbt-project-template-oracle', 'dbt-project-template-clickhouse', + 'dbt-project-template-snowflake', 'dbt_test_config', 'normalization_test_output', 'tools', diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index e93d8b63c5b9b..d2435ccc8e5e9 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -14,6 +14,8 @@ Each stream will be output into its own table in Snowflake. Each table will cont * `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Snowflake is `TIMESTAMP WITH TIME ZONE`. * `_airbyte_data`: a json blob representing with the event data. The column type in Snowflake is `VARIANT`. +Note that Airbyte will create **permanent** tables. If you prefer to create transient tables (see [Snowflake docs](https://docs.snowflake.com/en/user-guide/tables-temp-transient.html) for a comparison), you will want to create a dedicated transient database for Airbyte (`CREATE TRANSIENT DATABASE airbyte_database`). + #### Features | Feature | Supported?\(Yes/No\) | Notes | @@ -196,7 +198,8 @@ Finally, you need to add read/write permissions to your bucket with that email. | Version | Date | Pull Request | Subject | |:--------| :-------- | :----- | :------ | -| 0.3.24 | 2021-12-23 | [#8869](https://github.com/airbytehq/airbyte/pull/8869) | Changed staging approach to Byte-Buffered | | +| 0.4.0 | 2021-12-27 | [#9063](https://github.com/airbytehq/airbyte/pull/9063) | Updated normalization to produce permanent tables | +| 0.3.24 | 2021-12-23 | [#8869](https://github.com/airbytehq/airbyte/pull/8869) | Changed staging approach to Byte-Buffered | | 0.3.23 | 2021-12-22 | [#9039](https://github.com/airbytehq/airbyte/pull/9039) | Added part_size configuration in UI for S3 loading method | | 0.3.22 | 2021-12-21 | [#9006](https://github.com/airbytehq/airbyte/pull/9006) | Updated jdbc schema naming to follow Snowflake Naming Conventions | | 0.3.21 | 2021-12-15 | [#8781](https://github.com/airbytehq/airbyte/pull/8781) | Updated check method to verify permissions to create/drop stage for internal staging; compatibility fix for Java 17 | diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md index a40dab26f56b1..9122b06ea525a 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/understanding-airbyte/basic-normalization.md @@ -50,7 +50,7 @@ CREATE TABLE "cars" ( ## Normalization metadata columns -You'll notice that some metadata are added to keep track of important information about each record. +You'll notice that some metadata are added to keep track of important information about each record. - Some are introduced at the destination connector level: These are propagated by the normalization process from the raw table to the final table - `_airbyte_ab_id`: uuid value assigned by connectors to each row of the data written in the destination. - `_airbyte_emitted_at`: time at which the record was emitted and recorded by destination connector. @@ -329,7 +329,7 @@ Normalization produces tables that are partitioned, clustered, sorted or indexed In general, normalization needs to do lookup on the last emitted_at column to know if a record is freshly produced and need to be incrementally processed or not. But in certain models, such as SCD tables for example, we also need to retrieve older data to update their type 2 SCD end_date and active_row flags, thus a different partitioning scheme is used to optimize that use case. - + On Postgres destination, an additional table suffixed with `_stg` for every stream replicated in [incremental deduped history](connections/incremental-deduped-history.md) needs to be persisted (in a different staging schema) for incremental transformations to work because of a [limitation](https://github.com/dbt-labs/docs.getdbt.com/issues/335#issuecomment-694199569). ## Extending Basic Normalization @@ -350,7 +350,8 @@ Therefore, in order to "upgrade" to the desired normalization version, you need | Airbyte Version | Normalization Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | :--- | -| 0.32.11-alpha | 0.1.61 | 2021-12-02 | [\#8394](https://github.com/airbytehq/airbyte/pull/8394) | Fix incremental queries not updating empty tables | +| 0.32.11-alpha | 0.1.62 | 2021-12-23 | [\#9063](https://github.com/airbytehq/airbyte/pull/9063) | Add Snowflake-specific normalization | +| | 0.1.61 | 2021-12-02 | [\#8394](https://github.com/airbytehq/airbyte/pull/8394) | Fix incremental queries not updating empty tables | | | 0.1.61 | 2021-12-01 | [\#8378](https://github.com/airbytehq/airbyte/pull/8378) | Fix un-nesting queries and add proper ref hints | | 0.32.5-alpha | 0.1.60 | 2021-11-22 | [\#8088](https://github.com/airbytehq/airbyte/pull/8088) | Speed-up incremental queries for SCD table on Snowflake | | 0.30.32-alpha | 0.1.59 | 2021-11-08 | [\#7669](https://github.com/airbytehq/airbyte/pull/7169) | Fix nested incremental dbt |