From b7a13c200f2afd8622f173db36e4e2149600139b Mon Sep 17 00:00:00 2001 From: James Zhao <3120325+jzcruiser@users.noreply.github.com> Date: Tue, 21 Dec 2021 14:16:53 +0800 Subject: [PATCH 1/5] Fix clickhouse destination IllegalStateException 2021-12-21 05:48:35 destination > 2021-12-21 05:48:35 INFO r.y.c.ClickHouseDriver():49 - {} - Driver registered 2021-12-21 05:48:35 destination > Exception in thread "main" java.lang.IllegalStateException: jdbc destinations must specify a schema. 2021-12-21 05:48:35 destination > at com.google.common.base.Preconditions.checkState(Preconditions.java:502) 2021-12-21 05:48:35 destination > at io.airbyte.integrations.destination.jdbc.JdbcBufferedConsumerFactory.createWriteConfigs(JdbcBufferedConsumerFactory.java:71) 2021-12-21 05:48:35 destination > at io.airbyte.integrations.destination.jdbc.JdbcBufferedConsumerFactory.create(JdbcBufferedConsumerFactory.java:54) 2021-12-21 05:48:35 destination > at io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination.getConsumer(AbstractJdbcDestination.java:99) 2021-12-21 05:48:35 destination > at io.airbyte.integrations.base.ssh.SshWrappedDestination.getConsumer(SshWrappedDestination.java:58) 2021-12-21 05:48:35 destination > at io.airbyte.integrations.base.IntegrationRunner.run(IntegrationRunner.java:127) 2021-12-21 05:48:35 destination > at io.airbyte.integrations.destination.clickhouse.ClickhouseDestination.main(ClickhouseDestination.java:89) --- .../destination/clickhouse/ClickhouseSqlOperations.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java index b712a8cf0a7a2..046ef74d669a4 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java @@ -28,6 +28,11 @@ public void createSchemaIfNotExists(final JdbcDatabase database, final String sc database.execute(String.format("CREATE DATABASE IF NOT EXISTS %s;\n", schemaName)); } + @Override + public boolean isSchemaRequired() { + return false; + } + @Override public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( From 5d70724fa82fcd5ec84745cb5578650e7c93fd92 Mon Sep 17 00:00:00 2001 From: James Zhao Date: Wed, 22 Dec 2021 00:50:30 +0800 Subject: [PATCH 2/5] bump destination-clickhouse's version to 0.1.1 --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../connectors/destination-clickhouse/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 7f57e644d87b0..e83421ee86016 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -37,7 +37,7 @@ - name: Clickhouse destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 dockerRepository: airbyte/destination-clickhouse - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse - name: DynamoDB destinationDefinitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89 diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index 1a062e9892b0f..01777648251c7 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -7,5 +7,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/destination-clickhouse From fdf38730ac7a403c39e1d8b60704f012e717aab9 Mon Sep 17 00:00:00 2001 From: James Zhao Date: Wed, 22 Dec 2021 00:52:35 +0800 Subject: [PATCH 3/5] bump destination-clickhouse's version to 0.1.1 destination_specs.yaml and standard definition file --- .../ce0d828e-1dc4-496c-b122-2da42e637e48.json | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ce0d828e-1dc4-496c-b122-2da42e637e48.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ce0d828e-1dc4-496c-b122-2da42e637e48.json index 88e582d9c3880..eddc2a62e2bdd 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ce0d828e-1dc4-496c-b122-2da42e637e48.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ce0d828e-1dc4-496c-b122-2da42e637e48.json @@ -2,6 +2,6 @@ "destinationDefinitionId": "ce0d828e-1dc4-496c-b122-2da42e637e48", "name": "Clickhouse", "dockerRepository": "airbyte/destination-clickhouse", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.1", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 37bc777ec4ff1..0eaa102c1838f 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -529,7 +529,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-clickhouse:0.1.0" +- dockerImage: "airbyte/destination-clickhouse:0.1.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: From d5052975f9b05287caec7be452b3c9471319fa8d Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 21 Dec 2021 18:41:46 +0100 Subject: [PATCH 4/5] update changelog --- docs/integrations/destinations/clickhouse.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index 9312e173964b2..8428408667a27 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -79,5 +79,6 @@ Therefore, Airbyte ClickHouse destination will create tables and schemas using t | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.1 | 2021-12-21 | [\#8982](https://github.com/airbytehq/airbyte/pull/8982) | Set isSchemaRequired to false | | 0.1.0 | 2021-11-04 | [\#7620](https://github.com/airbytehq/airbyte/pull/7620) | Add ClickHouse destination | From f9f1c75fdae88fff1fbdac75f4375f849b964b04 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 21 Dec 2021 19:30:28 +0100 Subject: [PATCH 5/5] update destinations specs --- .../resources/seed/destination_specs.yaml | 98 ++++++++++++++++++- 1 file changed, 95 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 0eaa102c1838f..c86872cca2428 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -354,7 +354,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-bigquery-denormalized:0.1.11" +- dockerImage: "airbyte/destination-bigquery-denormalized:0.2.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery" connectionSpecification: @@ -366,6 +366,19 @@ - "dataset_id" additionalProperties: true properties: + big_query_client_buffer_size_mb: + title: "Google BigQuery client chunk size" + description: "Google BigQuery client's chunk (buffer) size (MIN = 1, MAX\ + \ = 15) for each table. It defaults to 15MiB. Smaller chunk size means\ + \ less memory consumption, and is recommended for big data sets. For more\ + \ details refer to the documentation here" + type: "integer" + minimum: 1 + maximum: 15 + default: 15 + examples: + - "15" project_id: type: "string" description: "The GCP project ID for the project containing the target BigQuery\ @@ -422,6 +435,85 @@ \ be used if this field is left empty." title: "Credentials JSON" airbyte_secret: true + loading_method: + type: "object" + title: "Loading Method" + description: "Select the way that data will be uploaded to BigQuery." + oneOf: + - title: "Standard Inserts" + additionalProperties: false + description: "Direct uploading using streams." + required: + - "method" + properties: + method: + type: "string" + const: "Standard" + - title: "GCS Staging" + additionalProperties: false + description: "Writes large batches of records to a file, uploads the file\ + \ to GCS, then uses 
COPY INTO table
to upload the file. Recommended\ + \ for large production workloads for better speed and scalability." + required: + - "method" + - "gcs_bucket_name" + - "gcs_bucket_path" + - "credential" + properties: + method: + type: "string" + const: "GCS Staging" + gcs_bucket_name: + title: "GCS Bucket Name" + type: "string" + description: "The name of the GCS bucket." + examples: + - "airbyte_sync" + gcs_bucket_path: + description: "Directory under the GCS bucket where data will be written." + type: "string" + examples: + - "data_sync/test" + keep_files_in_gcs-bucket: + type: "string" + description: "This upload method is supposed to temporary store records\ + \ in GCS bucket. What do you want to do with data in GCS bucket\ + \ when migration has finished?" + title: "GCS tmp files afterward processing" + default: "Delete all tmp files from GCS" + enum: + - "Delete all tmp files from GCS" + - "Keep all tmp files in GCS" + credential: + title: "Credential" + type: "object" + oneOf: + - title: "HMAC key" + required: + - "credential_type" + - "hmac_key_access_id" + - "hmac_key_secret" + properties: + credential_type: + type: "string" + const: "HMAC_KEY" + hmac_key_access_id: + type: "string" + description: "HMAC key access ID. When linked to a service account,\ + \ this ID is 61 characters long; when linked to a user account,\ + \ it is 24 characters long." + title: "HMAC Key Access ID" + airbyte_secret: true + examples: + - "1234567890abcdefghij1234" + hmac_key_secret: + type: "string" + description: "The corresponding secret for the access ID. It\ + \ is a 40-character base-64 encoded string." + title: "HMAC Key Secret" + airbyte_secret: true + examples: + - "1234567890abcdefghij1234567890ABCDEFGHIJ" supportsIncremental: true supportsNormalization: false supportsDBT: true @@ -1014,7 +1106,7 @@ - "overwrite" - "append" supportsNamespaces: true -- dockerImage: "airbyte/destination-gcs:0.1.16" +- dockerImage: "airbyte/destination-gcs:0.1.17" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/gcs" connectionSpecification: @@ -3259,7 +3351,7 @@ supported_destination_sync_modes: - "append" - "overwrite" -- dockerImage: "airbyte/destination-s3:0.2.1" +- dockerImage: "airbyte/destination-s3:0.2.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/s3" connectionSpecification: