diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json index d433add5e8071..6c8151fe5c2e4 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "079d5540-f236-4294-ba7c-ade8fd918496", "name": "BigQuery (denormalized typed struct)", "dockerRepository": "airbyte/destination-bigquery-denormalized", - "dockerImageTag": "0.2.4", + "dockerImageTag": "0.2.5", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", "icon": "bigquery.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json index d076c305f7c2f..d968513b3de6e 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "22f6c74f-5699-40ff-833c-4a879ea40133", "name": "BigQuery", "dockerRepository": "airbyte/destination-bigquery", - "dockerImageTag": "0.6.4", + "dockerImageTag": "0.6.5", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", "icon": "bigquery.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index d1584307abe4b..6f120e18eca22 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -13,13 +13,13 @@ - name: BigQuery destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 dockerRepository: airbyte/destination-bigquery - dockerImageTag: 0.6.4 + dockerImageTag: 0.6.5 documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery icon: bigquery.svg - name: BigQuery (denormalized typed struct) destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-bigquery-denormalized - dockerImageTag: 0.2.4 + dockerImageTag: 0.2.5 documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery icon: bigquery.svg - name: Cassandra diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 235c7a9ebb711..6f9cb6856d36c 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -188,7 +188,7 @@ supportsDBT: false supported_destination_sync_modes: - "append" -- dockerImage: "airbyte/destination-bigquery:0.6.4" +- dockerImage: "airbyte/destination-bigquery:0.6.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery" connectionSpecification: @@ -203,10 +203,12 @@ big_query_client_buffer_size_mb: title: "Google BigQuery client chunk size" description: "Google BigQuery client's chunk(buffer) size (MIN=1, MAX =\ - \ 15) for each table. The default 15MiB value is used if not set explicitly.\ - \ It's recommended to decrease value for big data sets migration for less\ - \ HEAP memory consumption and avoiding crashes. For more details refer\ - \ to https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html" + \ 15) for each table. The size that will be written by a single RPC. Written\ + \ data will be buffered and only flushed upon reaching this size or closing\ + \ the channel. The default 15MiB value is used if not set explicitly.\ + \ It's recommended to decrease value for big data sets migration for\ + \ less HEAP memory consumption and avoiding crashes. For more details\ + \ refer to https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html" type: "integer" minimum: 1 maximum: 15 @@ -271,9 +273,12 @@ airbyte_secret: true transformation_priority: type: "string" - description: "When running custom transformations or Basic normalization,\ - \ running queries on interactive mode can hit BQ limits, choosing batch\ - \ will solve those limits." + description: "Interactive run type means that the query is executed as soon\ + \ as possible, and these queries count towards concurrent rate limit and\ + \ daily limit. Batch queries are queued and started as soon as idle resources\ + \ are available in the BigQuery shared resource pool, which usually occurs\ + \ within a few minutes. Batch queries don’t count towards your concurrent\ + \ rate limit." title: "Transformation Query Run Type" default: "interactive" enum: @@ -378,7 +383,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-bigquery-denormalized:0.2.4" +- dockerImage: "airbyte/destination-bigquery-denormalized:0.2.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery" connectionSpecification: @@ -393,9 +398,11 @@ big_query_client_buffer_size_mb: title: "Google BigQuery client chunk size" description: "Google BigQuery client's chunk (buffer) size (MIN = 1, MAX\ - \ = 15) for each table. It defaults to 15MiB. Smaller chunk size means\ + \ = 15) for each table. The size that will be written by a single RPC.\ + \ Written data will be buffered and only flushed upon reaching this size\ + \ or closing the channel. It defaults to 15MiB. Smaller chunk size means\ \ less memory consumption, and is recommended for big data sets. For more\ - \ details refer to the documentation here" type: "integer" minimum: 1 diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index 11d11d4387ad0..e283bc9062d5e 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-bigquery-denormalized COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/spec.json index 829f6c3199a09..ee93cfcf42907 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/spec.json @@ -13,7 +13,7 @@ "properties": { "big_query_client_buffer_size_mb": { "title": "Google BigQuery client chunk size", - "description": "Google BigQuery client's chunk (buffer) size (MIN = 1, MAX = 15) for each table. It defaults to 15MiB. Smaller chunk size means less memory consumption, and is recommended for big data sets. For more details refer to the documentation here", + "description": "Google BigQuery client's chunk (buffer) size (MIN = 1, MAX = 15) for each table. The size that will be written by a single RPC. Written data will be buffered and only flushed upon reaching this size or closing the channel. It defaults to 15MiB. Smaller chunk size means less memory consumption, and is recommended for big data sets. For more details refer to the documentation here", "type": "integer", "minimum": 1, "maximum": 15, diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index ef038a9654030..57decbb9135eb 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-bigquery COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.4 +LABEL io.airbyte.version=0.6.5 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json index 879d4c12c6391..dd86d32e9e077 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json @@ -13,7 +13,7 @@ "properties": { "big_query_client_buffer_size_mb": { "title": "Google BigQuery client chunk size", - "description": "Google BigQuery client's chunk(buffer) size (MIN=1, MAX = 15) for each table. The default 15MiB value is used if not set explicitly. It's recommended to decrease value for big data sets migration for less HEAP memory consumption and avoiding crashes. For more details refer to https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html", + "description": "Google BigQuery client's chunk(buffer) size (MIN=1, MAX = 15) for each table. The size that will be written by a single RPC. Written data will be buffered and only flushed upon reaching this size or closing the channel. The default 15MiB value is used if not set explicitly. It's recommended to decrease value for big data sets migration for less HEAP memory consumption and avoiding crashes. For more details refer to https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html", "type": "integer", "minimum": 1, "maximum": 15, @@ -77,7 +77,7 @@ }, "transformation_priority": { "type": "string", - "description": "When running custom transformations or Basic normalization, running queries on interactive mode can hit BQ limits, choosing batch will solve those limits.", + "description": "Interactive run type means that the query is executed as soon as possible, and these queries count towards concurrent rate limit and daily limit. Batch queries are queued and started as soon as idle resources are available in the BigQuery shared resource pool, which usually occurs within a few minutes. Batch queries don’t count towards your concurrent rate limit.", "title": "Transformation Query Run Type", "default": "interactive", "enum": ["interactive", "batch"] diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index fffbea6c7c4f4..d87b2c73143e4 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -153,6 +153,7 @@ Therefore, Airbyte BigQuery destination will convert any invalid characters into | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.6.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | | 0.6.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | | 0.6.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | | 0.6.2 | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121) | Fixed check method for GCS mode to verify if all roles assigned to user | @@ -173,6 +174,7 @@ Therefore, Airbyte BigQuery destination will convert any invalid characters into | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------| :--- | +| 0.2.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | | 0.2.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | BigQuery/BiqQuery denorm Destinations : Support dataset-id prefixed by project-id | | 0.2.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | | 0.2.2 | 2021-12-22 | [\#9039](https://github.com/airbytehq/airbyte/pull/9039) | Added part_size configuration to UI for GCS staging |