From 7727b8635959b6581dd273759cfe62b578ba94c9 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Thu, 6 Jan 2022 18:03:06 +0100 Subject: [PATCH] Include tcp port in Clickhouse destination configuration for normalization (#9340) * Include tcp port in Clickhouse destination configuration for normalization * revert custom dbt on clickhouse * Bumpversion of destination connector * Format code --- .../seed/destination_definitions.yaml | 2 +- .../resources/seed/destination_specs.yaml | 20 ++- .../transform_config/transform.py | 2 + ...ltBigQueryDenormalizedRecordFormatter.java | 2 +- .../destination-clickhouse/Dockerfile | 2 +- .../src/main/resources/spec.json | 18 ++- .../source_amazon_seller_partner/streams.py | 2 +- .../source-github/unit_tests/test_stream.py | 1 + .../source-jira/source_jira/source.py | 2 +- .../source_recurly/schemas/invoices.json | 115 +++--------------- .../NormalizationRunnerFactory.java | 5 +- 11 files changed, 58 insertions(+), 113 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index c03cc2c715d3f..100521ffbc212 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -37,7 +37,7 @@ - name: Clickhouse destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 dockerRepository: airbyte/destination-clickhouse - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse - name: DynamoDB destinationDefinitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89 diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 03dc563b6f39d..266fefa1716d8 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -646,7 +646,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-clickhouse:0.1.1" +- dockerImage: "airbyte/destination-clickhouse:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: @@ -675,28 +675,38 @@ examples: - "8123" order: 1 + tcp-port: + title: "Native Port" + description: "Native port (not the JDBC) of the database." + type: "integer" + minimum: 0 + maximum: 65536 + default: 9000 + examples: + - "9000" + order: 2 database: title: "DB Name" description: "Name of the database." type: "string" - order: 2 + order: 3 username: title: "User" description: "Username to use to access the database." type: "string" - order: 3 + order: 4 password: title: "Password" description: "Password associated with the username." type: "string" airbyte_secret: true - order: 4 + order: 5 ssl: title: "SSL Connection" description: "Encrypt data using SSL." type: "boolean" default: false - order: 5 + order: 6 tunnel_method: type: "object" title: "SSH Tunnel Method" diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py index d45cbc0623fb0..4ed99e1c29b51 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py @@ -277,6 +277,8 @@ def transform_clickhouse(config: Dict[str, Any]): "user": config["username"], "password": config["password"], } + if "tcp-port" in config: + dbt_config["port"] = config["tcp-port"] return dbt_config @staticmethod diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java index a08751b25164f..63e5478c17cfd 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java @@ -191,7 +191,7 @@ private JsonNode getObjectNode(final FieldList fields, final JsonNode root) { @Override public Schema getBigQuerySchema(final JsonNode jsonSchema) { - final List fieldList = getSchemaFields(namingResolver, jsonSchema); + final List fieldList = getSchemaFields(namingResolver, jsonSchema); if (fieldList.stream().noneMatch(f -> f.getName().equals(JavaBaseConstants.COLUMN_NAME_AB_ID))) { fieldList.add(Field.of(JavaBaseConstants.COLUMN_NAME_AB_ID, StandardSQLTypeName.STRING)); } diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index 32402f87805c7..98e08d55920dc 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-clickhouse diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json index 6037b573394d5..1edbbb8465d94 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json @@ -27,31 +27,41 @@ "examples": ["8123"], "order": 1 }, + "tcp-port": { + "title": "Native Port", + "description": "Native port (not the JDBC) of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 9000, + "examples": ["9000"], + "order": 2 + }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", - "order": 2 + "order": 3 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", - "order": 3 + "order": 4 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, - "order": 4 + "order": 5 }, "ssl": { "title": "SSL Connection", "description": "Encrypt data using SSL.", "type": "boolean", "default": false, - "order": 5 + "order": 6 } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index bcf2569a67795..2d8c880cf3e3d 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -644,4 +644,4 @@ def request_params( return params def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - yield from response.json().get(self.data_field, {}).get("shippingLabels", []) \ No newline at end of file + yield from response.json().get(self.data_field, {}).get("shippingLabels", []) diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py index 6f1cbf1d65487..1f06233fd45a2 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py @@ -1,6 +1,7 @@ # # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # + from http import HTTPStatus from unittest.mock import patch diff --git a/airbyte-integrations/connectors/source-jira/source_jira/source.py b/airbyte-integrations/connectors/source-jira/source_jira/source.py index 3689d253b8014..538a9ff94ed35 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/source.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/source.py @@ -107,7 +107,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: **incremental_args, additional_fields=config.get("additional_fields", []), expand_changelog=config.get("expand_issue_changelog", False), - render_fields=render_fields + render_fields=render_fields, ) issue_fields_stream = IssueFields(**args) experimental_streams = [] diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/invoices.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/invoices.json index a7bacc0d38b0d..fecf63de82eb4 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/invoices.json @@ -16,11 +16,7 @@ "title": "Invoice type", "description": "Invoices are either charge, credit, or legacy invoices.", "type": ["null", "string"], - "enum": [ - "charge", - "credit", - "legacy" - ] + "enum": ["charge", "credit", "legacy"] }, "origin": { "title": "Origin", @@ -143,10 +139,7 @@ "description": "An automatic invoice means a corresponding transaction is run using the account's billing information at the same time the invoice is created. Manual invoices are created without a corresponding transaction. The merchant must enter a manual payment transaction or have the customer pay the invoice with an automatic method, like credit card, PayPal, Amazon, or ACH bank payment.", "default": "automatic", "type": ["null", "string"], - "enum": [ - "automatic", - "manual" - ] + "enum": ["automatic", "manual"] }, "po_number": { "type": ["null", "string"], @@ -500,10 +493,7 @@ "title": "Line item type", "description": "Charges are positive line items that debit the account. Credits are negative line items that credit the account.", "type": ["null", "string"], - "enum": [ - "charge", - "credit" - ] + "enum": ["charge", "credit"] }, "item_code": { "type": ["null", "string"], @@ -532,10 +522,7 @@ "title": "Current state of the line item", "description": "Pending line items are charges or credits on an account that have not been applied to an invoice yet. Invoiced line items will always have an `invoice_id` value.", "type": ["null", "string"], - "enum": [ - "invoiced", - "pending" - ] + "enum": ["invoiced", "pending"] }, "legacy_category": { "title": "Legacy category", @@ -1064,11 +1051,7 @@ "type": { "title": "Invoice type", "type": ["null", "string"], - "enum": [ - "charge", - "credit", - "legacy" - ] + "enum": ["charge", "credit", "legacy"] }, "state": { "title": "Invoice state", @@ -1106,11 +1089,7 @@ "type": { "title": "Invoice type", "type": ["null", "string"], - "enum": [ - "charge", - "credit", - "legacy" - ] + "enum": ["charge", "credit", "legacy"] }, "state": { "title": "Invoice state", @@ -1142,13 +1121,7 @@ "title": "Transaction type", "description": "- `authorization` \u2013 verifies billing information and places a hold on money in the customer's account.\n- `capture` \u2013 captures funds held by an authorization and completes a purchase.\n- `purchase` \u2013 combines the authorization and capture in one transaction.\n- `refund` \u2013 returns all or a portion of the money collected in a previous transaction to the customer.\n- `verify` \u2013 a $0 or $1 transaction used to verify billing information which is immediately voided.\n", "type": ["null", "string"], - "enum": [ - "authorization", - "capture", - "purchase", - "refund", - "verify" - ] + "enum": ["authorization", "capture", "purchase", "refund", "verify"] }, "origin": { "title": "Origin of transaction", @@ -1263,10 +1236,7 @@ "collection_method": { "description": "The method by which the payment was collected.", "type": ["null", "string"], - "enum": [ - "automatic", - "manual" - ] + "enum": ["automatic", "manual"] }, "payment_method": { "properties": { @@ -1362,10 +1332,7 @@ "account_type": { "description": "The bank account type. Only present for ACH payment methods.", "type": ["null", "string"], - "enum": [ - "checking", - "savings" - ] + "enum": ["checking", "savings"] }, "routing_number": { "type": ["null", "string"], @@ -1457,16 +1424,7 @@ "title": "CVV check", "description": "When processed, result from checking the CVV/CVC value on the transaction.", "type": ["null", "string"], - "enum": [ - "D", - "I", - "M", - "N", - "P", - "S", - "U", - "X" - ] + "enum": ["D", "I", "M", "N", "P", "S", "U", "X"] }, "avs_check": { "title": "AVS check", @@ -1548,12 +1506,7 @@ "title": "Action", "description": "The action for which the credit was created.", "type": ["null", "string"], - "enum": [ - "payment", - "reduction", - "refund", - "write_off" - ] + "enum": ["payment", "reduction", "refund", "write_off"] }, "account": { "type": "object", @@ -1626,11 +1579,7 @@ "type": { "title": "Invoice type", "type": ["null", "string"], - "enum": [ - "charge", - "credit", - "legacy" - ] + "enum": ["charge", "credit", "legacy"] }, "state": { "title": "Invoice state", @@ -1668,11 +1617,7 @@ "type": { "title": "Invoice type", "type": ["null", "string"], - "enum": [ - "charge", - "credit", - "legacy" - ] + "enum": ["charge", "credit", "legacy"] }, "state": { "title": "Invoice state", @@ -1803,11 +1748,7 @@ "type": { "title": "Invoice type", "type": ["null", "string"], - "enum": [ - "charge", - "credit", - "legacy" - ] + "enum": ["charge", "credit", "legacy"] }, "state": { "title": "Invoice state", @@ -1845,11 +1786,7 @@ "type": { "title": "Invoice type", "type": ["null", "string"], - "enum": [ - "charge", - "credit", - "legacy" - ] + "enum": ["charge", "credit", "legacy"] }, "state": { "title": "Invoice state", @@ -2002,10 +1939,7 @@ "collection_method": { "description": "The method by which the payment was collected.", "type": ["null", "string"], - "enum": [ - "automatic", - "manual" - ] + "enum": ["automatic", "manual"] }, "payment_method": { "type": "object", @@ -2102,10 +2036,7 @@ "account_type": { "description": "The bank account type. Only present for ACH payment methods.", "type": ["null", "string"], - "enum": [ - "checking", - "savings" - ] + "enum": ["checking", "savings"] }, "routing_number": { "type": ["null", "string"], @@ -2116,7 +2047,6 @@ "description": "The bank name of this routing number." } } - }, "ip_address_v4": { "type": ["null", "string"], @@ -2197,16 +2127,7 @@ "title": "CVV check", "description": "When processed, result from checking the CVV/CVC value on the transaction.", "type": ["null", "string"], - "enum": [ - "D", - "I", - "M", - "N", - "P", - "S", - "U", - "X" - ] + "enum": ["D", "I", "M", "N", "P", "S", "U", "X"] }, "avs_check": { "title": "AVS check", diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index adcee9eb3b7a4..12368661ec5ca 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -18,9 +18,12 @@ public class NormalizationRunnerFactory { static final Map> NORMALIZATION_MAPPING = ImmutableMap.>builder() + // map destination connectors (alphabetically) to their expected normalization settings .put("airbyte/destination-bigquery", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DefaultNormalizationRunner.DestinationType.BIGQUERY)) .put("airbyte/destination-bigquery-denormalized", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DefaultNormalizationRunner.DestinationType.BIGQUERY)) + .put("airbyte/destination-clickhouse", ImmutablePair.of("airbyte/normalization-clickhouse", DestinationType.CLICKHOUSE)) + .put("airbyte/destination-clickhouse-strict-encrypt", ImmutablePair.of("airbyte/normalization-clickhouse", DestinationType.CLICKHOUSE)) .put("airbyte/destination-mssql", ImmutablePair.of("airbyte/normalization-mssql", DestinationType.MSSQL)) .put("airbyte/destination-mssql-strict-encrypt", ImmutablePair.of("airbyte/normalization-mssql", DestinationType.MSSQL)) .put("airbyte/destination-mysql", ImmutablePair.of("airbyte/normalization-mysql", DestinationType.MYSQL)) @@ -31,8 +34,6 @@ public class NormalizationRunnerFactory { .put("airbyte/destination-postgres-strict-encrypt", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DestinationType.POSTGRES)) .put("airbyte/destination-redshift", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DestinationType.REDSHIFT)) .put("airbyte/destination-snowflake", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DestinationType.SNOWFLAKE)) - .put("airbyte/destination-clickhouse", ImmutablePair.of("airbyte/normalization-clickhouse", DestinationType.CLICKHOUSE)) - .put("airbyte/destination-clickhouse-strict-encrypt", ImmutablePair.of("airbyte/normalization-clickhouse", DestinationType.CLICKHOUSE)) .build(); public static NormalizationRunner create(final WorkerConfigs workerConfigs, final String connectorImageName, final ProcessFactory processFactory) {