Include tcp port in Clickhouse destination configuration for normaliz…

…ation (#9340) * Include tcp port in Clickhouse destination configuration for normalization * revert custom dbt on clickhouse * Bumpversion of destination connector * Format code
airbytehq · Jan 6, 2022 · 7727b86 · 7727b86
1 parent 5ffc880
commit 7727b86
Show file tree

Hide file tree

Showing 11 changed files with 58 additions and 113 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -37,7 +37,7 @@
 - name: Clickhouse
   destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48
   dockerRepository: airbyte/destination-clickhouse
-  dockerImageTag: 0.1.1
+  dockerImageTag: 0.1.2
   documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse
 - name: DynamoDB
   destinationDefinitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -646,7 +646,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-clickhouse:0.1.1"
+- dockerImage: "airbyte/destination-clickhouse:0.1.2"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse"
     connectionSpecification:
@@ -675,28 +675,38 @@
           examples:
           - "8123"
           order: 1
+        tcp-port:
+          title: "Native Port"
+          description: "Native port (not the JDBC) of the database."
+          type: "integer"
+          minimum: 0
+          maximum: 65536
+          default: 9000
+          examples:
+          - "9000"
+          order: 2
         database:
           title: "DB Name"
           description: "Name of the database."
           type: "string"
-          order: 2
+          order: 3
         username:
           title: "User"
           description: "Username to use to access the database."
           type: "string"
-          order: 3
+          order: 4
         password:
           title: "Password"
           description: "Password associated with the username."
           type: "string"
           airbyte_secret: true
-          order: 4
+          order: 5
         ssl:
           title: "SSL Connection"
           description: "Encrypt data using SSL."
           type: "boolean"
           default: false
-          order: 5
+          order: 6
         tunnel_method:
           type: "object"
           title: "SSH Tunnel Method"

diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py
@@ -277,6 +277,8 @@ def transform_clickhouse(config: Dict[str, Any]):
             "user": config["username"],
             "password": config["password"],
         }
+        if "tcp-port" in config:
+            dbt_config["port"] = config["tcp-port"]
         return dbt_config
 
     @staticmethod

diff --git a/...tegrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java b/...tegrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java
@@ -191,7 +191,7 @@ private JsonNode getObjectNode(final FieldList fields, final JsonNode root) {
 
   @Override
   public Schema getBigQuerySchema(final JsonNode jsonSchema) {
-        final List<Field> fieldList = getSchemaFields(namingResolver, jsonSchema);
+    final List<Field> fieldList = getSchemaFields(namingResolver, jsonSchema);
     if (fieldList.stream().noneMatch(f -> f.getName().equals(JavaBaseConstants.COLUMN_NAME_AB_ID))) {
       fieldList.add(Field.of(JavaBaseConstants.COLUMN_NAME_AB_ID, StandardSQLTypeName.STRING));
     }

diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.version=0.1.2
 LABEL io.airbyte.name=airbyte/destination-clickhouse
diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json
@@ -27,31 +27,41 @@
         "examples": ["8123"],
         "order": 1
       },
+      "tcp-port": {
+        "title": "Native Port",
+        "description": "Native port (not the JDBC) of the database.",
+        "type": "integer",
+        "minimum": 0,
+        "maximum": 65536,
+        "default": 9000,
+        "examples": ["9000"],
+        "order": 2
+      },
       "database": {
         "title": "DB Name",
         "description": "Name of the database.",
         "type": "string",
-        "order": 2
+        "order": 3
       },
       "username": {
         "title": "User",
         "description": "Username to use to access the database.",
         "type": "string",
-        "order": 3
+        "order": 4
       },
       "password": {
         "title": "Password",
         "description": "Password associated with the username.",
         "type": "string",
         "airbyte_secret": true,
-        "order": 4
+        "order": 5
       },
       "ssl": {
         "title": "SSL Connection",
         "description": "Encrypt data using SSL.",
         "type": "boolean",
         "default": false,
-        "order": 5
+        "order": 6
       }
     }
   }

diff --git a/...egrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/...egrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py
@@ -644,4 +644,4 @@ def request_params(
         return params
 
     def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]:
-        yield from response.json().get(self.data_field, {}).get("shippingLabels", [])
+        yield from response.json().get(self.data_field, {}).get("shippingLabels", [])
diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py
@@ -1,6 +1,7 @@
 #
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
+
 from http import HTTPStatus
 from unittest.mock import patch
 

diff --git a/airbyte-integrations/connectors/source-jira/source_jira/source.py b/airbyte-integrations/connectors/source-jira/source_jira/source.py
@@ -107,7 +107,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             **incremental_args,
             additional_fields=config.get("additional_fields", []),
             expand_changelog=config.get("expand_issue_changelog", False),
-            render_fields=render_fields
+            render_fields=render_fields,
         )
         issue_fields_stream = IssueFields(**args)
         experimental_streams = []