From 80f25edb3d091b3eb62d1ef9f256e728567322a7 Mon Sep 17 00:00:00 2001 From: oneshcheret Date: Thu, 22 Sep 2022 13:07:48 +0300 Subject: [PATCH 01/10] Source clickhouse: add jdbc params --- .../resources/expected_spec.json | 15 ++++++--- .../source/clickhouse/ClickHouseSource.java | 11 ++++++- .../src/main/resources/spec.json | 24 ++++++++++---- .../ClickHouseJdbcSourceAcceptanceTest.java | 32 +++++++++++++++++++ 4 files changed, 70 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json index 985e302f8b276..55ed0472e4eb3 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json @@ -9,7 +9,8 @@ "host": { "description": "The host endpoint of the Clickhouse cluster.", "title": "Host", - "type": "string" + "type": "string", + "order": 0 }, "port": { "description": "The port of the database.", @@ -18,24 +19,28 @@ "minimum": 0, "maximum": 65536, "default": 8123, - "examples": ["8123"] + "examples": ["8123"], + "order": 1 }, "database": { "description": "The name of the database.", "title": "Database", "type": "string", - "examples": ["default"] + "examples": ["default"], + "order": 2 }, "username": { "description": "The username which is used to access the database.", "title": "Username", - "type": "string" + "type": "string", + "order": 3 }, "password": { "description": "The password associated with this username.", "title": "Password", "type": "string", - "airbyte_secret": true + "airbyte_secret": true, + "order": 4 } } } diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java index f68824d2c6800..d3f9630628b39 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java @@ -27,6 +27,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static io.airbyte.db.jdbc.JdbcUtils.AMPERSAND; + public class ClickHouseSource extends AbstractJdbcSource implements Source { /** @@ -90,9 +92,16 @@ public JsonNode toDatabaseConfig(final JsonNode config) { config.get(JdbcUtils.PORT_KEY).asText(), config.get(JdbcUtils.DATABASE_KEY).asText())); + boolean isAdditionalParamsExists = config.get(JdbcUtils.JDBC_URL_PARAMS_KEY) != null && !config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText().isEmpty(); // assume ssl if not explicitly mentioned. + if (isSsl || isAdditionalParamsExists) { + jdbcUrl.append("?"); + } if (isSsl) { - jdbcUrl.append("?").append(SSL_MODE); + jdbcUrl.append(SSL_MODE).append(AMPERSAND); + } + if (isAdditionalParamsExists) { + jdbcUrl.append(config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText()); } final ImmutableMap.Builder configBuilder = ImmutableMap.builder() diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json index 7b28b58ff3593..caf9f40395eda 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/resources/spec.json @@ -9,7 +9,8 @@ "host": { "description": "The host endpoint of the Clickhouse cluster.", "title": "Host", - "type": "string" + "type": "string", + "order": 0 }, "port": { "description": "The port of the database.", @@ -18,30 +19,41 @@ "minimum": 0, "maximum": 65536, "default": 8123, - "examples": ["8123"] + "examples": ["8123"], + "order": 1 }, "database": { "description": "The name of the database.", "title": "Database", "type": "string", - "examples": ["default"] + "examples": ["default"], + "order": 2 }, "username": { "description": "The username which is used to access the database.", "title": "Username", - "type": "string" + "type": "string", + "order": 3 }, "password": { "description": "The password associated with this username.", "title": "Password", "type": "string", - "airbyte_secret": true + "airbyte_secret": true, + "order": 4 + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", + "type": "string", + "order": 5 }, "ssl": { "title": "SSL Connection", "description": "Encrypt data using SSL.", "type": "boolean", - "default": true + "default": true, + "order": 6 } } } diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java index 8aa75368303c9..c760850eeb209 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import static java.time.temporal.ChronoUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.*; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; @@ -18,6 +19,8 @@ import java.sql.SQLException; import java.time.Duration; import java.util.List; + +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -114,4 +117,33 @@ public AbstractJdbcSource getJdbcSource() { return new ClickHouseSource(); } + @Test + public void testEmptyExtraParams() { + final String extraParam = ""; + JsonNode config = buildConfigWithExtraJdbcParameters(extraParam); + final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config); + assertNull(jdbcConfig.get(JdbcUtils.JDBC_URL_PARAMS_KEY)); + } + + @Test + public void testExtraParamsWithSsl() { + final String extraParam = "key1=value1&key2=value2&key3=value3"; + JsonNode config = buildConfigWithExtraJdbcParameters(extraParam); + final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config); + assertNotNull(jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText()); + assertTrue(jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText().endsWith(extraParam)); + } + + private JsonNode buildConfigWithExtraJdbcParameters(String extraParam) { + + return Jsons.jsonNode(com.google.common.collect.ImmutableMap.of( + "host", "localhost", + "port", 8123, + "database", "db", + "username", "username", + "password", "verysecure", + "jdbc_url_params", extraParam, + "ssl", false)); + } + } From 35748b6876dc843685921b73032ac5040453f450 Mon Sep 17 00:00:00 2001 From: oneshcheret Date: Thu, 22 Sep 2022 13:53:22 +0300 Subject: [PATCH 02/10] Source clickhouse: add jdbc params --- .../src/test-integration/resources/expected_spec.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json index 55ed0472e4eb3..b6000d108e350 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/resources/expected_spec.json @@ -41,6 +41,12 @@ "type": "string", "airbyte_secret": true, "order": 4 + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", + "type": "string", + "order": 5 } } } From ab75820139a325f83cf916265de1f8e9b3db41d9 Mon Sep 17 00:00:00 2001 From: oneshcheret Date: Fri, 23 Sep 2022 15:30:44 +0300 Subject: [PATCH 03/10] Source clickhouse: add jdbc params --- .../source/clickhouse/ClickHouseSource.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java index d3f9630628b39..b8ea72f0780e2 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java @@ -20,9 +20,7 @@ import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.SQLException; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,14 +92,17 @@ public JsonNode toDatabaseConfig(final JsonNode config) { boolean isAdditionalParamsExists = config.get(JdbcUtils.JDBC_URL_PARAMS_KEY) != null && !config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText().isEmpty(); // assume ssl if not explicitly mentioned. - if (isSsl || isAdditionalParamsExists) { - jdbcUrl.append("?"); - } + List params = new ArrayList<>(); if (isSsl) { - jdbcUrl.append(SSL_MODE).append(AMPERSAND); + params.add(SSL_MODE); } if (isAdditionalParamsExists) { - jdbcUrl.append(config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText()); + params.add(config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText()); + } + + if (isSsl || isAdditionalParamsExists) { + jdbcUrl.append("?"); + jdbcUrl.append(String.join("&", params)); } final ImmutableMap.Builder configBuilder = ImmutableMap.builder() From ba94e0faeb80a4923c762c84acf87cfbaea5ed67 Mon Sep 17 00:00:00 2001 From: oneshcheret Date: Fri, 23 Sep 2022 17:24:02 +0300 Subject: [PATCH 04/10] Source clickhouse: updated tests --- .../ClickHouseJdbcSourceAcceptanceTest.java | 54 ++++++++++++++++--- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java index c760850eeb209..8fb6c5baefe93 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import static io.airbyte.db.jdbc.JdbcUtils.JDBC_URL_KEY; +import static io.airbyte.integrations.source.clickhouse.ClickHouseSource.SSL_MODE; import static java.time.temporal.ChronoUnit.SECONDS; import static org.junit.jupiter.api.Assertions.*; @@ -20,6 +22,7 @@ import java.time.Duration; import java.util.List; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -118,23 +121,58 @@ public AbstractJdbcSource getJdbcSource() { } @Test - public void testEmptyExtraParams() { + public void testEmptyExtraParamsWithSsl() { final String extraParam = ""; - JsonNode config = buildConfigWithExtraJdbcParameters(extraParam); + JsonNode config = buildConfigWithExtraJdbcParameters(extraParam, true); final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config); - assertNull(jdbcConfig.get(JdbcUtils.JDBC_URL_PARAMS_KEY)); + JsonNode jdbcUrlNode = jdbcConfig.get(JDBC_URL_KEY); + assertNotNull(jdbcUrlNode); + String actualJdbcUrl = jdbcUrlNode.asText(); + assertTrue(actualJdbcUrl.endsWith("?" + SSL_MODE)); + } + + @Test + public void testEmptyExtraParamsWithoutSsl() { + final String extraParam = ""; + JsonNode config = buildConfigWithExtraJdbcParameters(extraParam, false); + final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config); + JsonNode jdbcUrlNode = jdbcConfig.get(JDBC_URL_KEY); + assertNotNull(jdbcUrlNode); + String actualJdbcUrl = jdbcUrlNode.asText(); + assertTrue(actualJdbcUrl.endsWith(config.get("database").asText())); } @Test public void testExtraParamsWithSsl() { final String extraParam = "key1=value1&key2=value2&key3=value3"; - JsonNode config = buildConfigWithExtraJdbcParameters(extraParam); + JsonNode config = buildConfigWithExtraJdbcParameters(extraParam, true); final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config); - assertNotNull(jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText()); - assertTrue(jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText().endsWith(extraParam)); + JsonNode jdbcUrlNode = jdbcConfig.get(JDBC_URL_KEY); + assertNotNull(jdbcUrlNode); + String actualJdbcUrl = jdbcUrlNode.asText(); + assertTrue(actualJdbcUrl.endsWith(getFullExpectedValue(extraParam, SSL_MODE))); + } + + @Test + public void testExtraParamsWithoutSsl() { + final String extraParam = "key1=value1&key2=value2&key3=value3"; + JsonNode config = buildConfigWithExtraJdbcParameters(extraParam, false); + final JsonNode jdbcConfig = new ClickHouseSource().toDatabaseConfig(config); + JsonNode jdbcUrlNode = jdbcConfig.get(JDBC_URL_KEY); + assertNotNull(jdbcUrlNode); + String actualJdbcUrl = jdbcUrlNode.asText(); + assertTrue(actualJdbcUrl.endsWith("?" + extraParam)); + } + + // "?" + extraParam +// "?" + SSL_MODE + + private String getFullExpectedValue(String extraParam, String sslMode) { + StringBuilder expected = new StringBuilder(); + return expected.append("?").append(sslMode).append("&").append(extraParam).toString(); } - private JsonNode buildConfigWithExtraJdbcParameters(String extraParam) { + private JsonNode buildConfigWithExtraJdbcParameters(String extraParam, boolean isSsl) { return Jsons.jsonNode(com.google.common.collect.ImmutableMap.of( "host", "localhost", @@ -143,7 +181,7 @@ private JsonNode buildConfigWithExtraJdbcParameters(String extraParam) { "username", "username", "password", "verysecure", "jdbc_url_params", extraParam, - "ssl", false)); + "ssl", isSsl)); } } From 7a193f9d1f8bcc5feb39f4a81c7ea093c0a7ba38 Mon Sep 17 00:00:00 2001 From: oneshcheret Date: Fri, 23 Sep 2022 17:25:58 +0300 Subject: [PATCH 05/10] Source clickhouse: format --- .../integrations/source/clickhouse/ClickHouseSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java index b8ea72f0780e2..df2b6b782f1ab 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java @@ -91,8 +91,8 @@ public JsonNode toDatabaseConfig(final JsonNode config) { config.get(JdbcUtils.DATABASE_KEY).asText())); boolean isAdditionalParamsExists = config.get(JdbcUtils.JDBC_URL_PARAMS_KEY) != null && !config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText().isEmpty(); - // assume ssl if not explicitly mentioned. List params = new ArrayList<>(); + // assume ssl if not explicitly mentioned. if (isSsl) { params.add(SSL_MODE); } From 827d0592824c850d478124eb1b20d7998aeae21d Mon Sep 17 00:00:00 2001 From: oneshcheret Date: Fri, 23 Sep 2022 17:27:35 +0300 Subject: [PATCH 06/10] Source clickhouse: format --- .../sources/ClickHouseJdbcSourceAcceptanceTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java index 8fb6c5baefe93..924b9fcda4198 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java @@ -164,9 +164,6 @@ public void testExtraParamsWithoutSsl() { assertTrue(actualJdbcUrl.endsWith("?" + extraParam)); } - // "?" + extraParam -// "?" + SSL_MODE - private String getFullExpectedValue(String extraParam, String sslMode) { StringBuilder expected = new StringBuilder(); return expected.append("?").append(sslMode).append("&").append(extraParam).toString(); From 2ae34db58c43aea3dcf0f6c2a769ac310f798043 Mon Sep 17 00:00:00 2001 From: oneshcheret Date: Mon, 26 Sep 2022 20:28:11 +0300 Subject: [PATCH 07/10] Source clickhouse: format --- .../source/clickhouse/ClickHouseSource.java | 10 ++++++---- .../ClickHouseJdbcSourceAcceptanceTest.java | 18 ++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java index df2b6b782f1ab..31b1cfabec040 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java @@ -20,13 +20,14 @@ import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.SQLException; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static io.airbyte.db.jdbc.JdbcUtils.AMPERSAND; - public class ClickHouseSource extends AbstractJdbcSource implements Source { /** @@ -90,7 +91,8 @@ public JsonNode toDatabaseConfig(final JsonNode config) { config.get(JdbcUtils.PORT_KEY).asText(), config.get(JdbcUtils.DATABASE_KEY).asText())); - boolean isAdditionalParamsExists = config.get(JdbcUtils.JDBC_URL_PARAMS_KEY) != null && !config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText().isEmpty(); + boolean isAdditionalParamsExists = + config.get(JdbcUtils.JDBC_URL_PARAMS_KEY) != null && !config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText().isEmpty(); List params = new ArrayList<>(); // assume ssl if not explicitly mentioned. if (isSsl) { diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java index 924b9fcda4198..5ff54a833a59c 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java @@ -21,12 +21,10 @@ import java.sql.SQLException; import java.time.Duration; import java.util.List; - -import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.testcontainers.containers.ClickHouseContainer; import org.testcontainers.containers.wait.strategy.Wait; @@ -172,13 +170,13 @@ private String getFullExpectedValue(String extraParam, String sslMode) { private JsonNode buildConfigWithExtraJdbcParameters(String extraParam, boolean isSsl) { return Jsons.jsonNode(com.google.common.collect.ImmutableMap.of( - "host", "localhost", - "port", 8123, - "database", "db", - "username", "username", - "password", "verysecure", - "jdbc_url_params", extraParam, - "ssl", isSsl)); + "host", "localhost", + "port", 8123, + "database", "db", + "username", "username", + "password", "verysecure", + "jdbc_url_params", extraParam, + "ssl", isSsl)); } } From 76f828ace0de8f83fe7563eb7277d2d5a2e3ec5f Mon Sep 17 00:00:00 2001 From: oneshcheret Date: Mon, 26 Sep 2022 20:59:30 +0300 Subject: [PATCH 08/10] Source clickhouse: format --- .../sources/ClickHouseJdbcSourceAcceptanceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java index 5ff54a833a59c..5a66d267fb11a 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java @@ -7,7 +7,8 @@ import static io.airbyte.db.jdbc.JdbcUtils.JDBC_URL_KEY; import static io.airbyte.integrations.source.clickhouse.ClickHouseSource.SSL_MODE; import static java.time.temporal.ChronoUnit.SECONDS; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; From 13295da27cc071cc3044e7e7c0de23e785978fb2 Mon Sep 17 00:00:00 2001 From: oneshcheret Date: Tue, 27 Sep 2022 12:31:12 +0300 Subject: [PATCH 09/10] Source clickhouse: bump version --- .../connectors/source-clickhouse-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-clickhouse/Dockerfile | 2 +- docs/integrations/sources/clickhouse.md | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile index c22f2e3d29027..582507952ef21 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=0.1.14 LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/source-clickhouse/Dockerfile b/airbyte-integrations/connectors/source-clickhouse/Dockerfile index b41fb06cb776b..6681cc7af3cba 100644 --- a/airbyte-integrations/connectors/source-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=0.1.14 LABEL io.airbyte.name=airbyte/source-clickhouse diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md index 113132059b697..b9fa131fc2d5b 100644 --- a/docs/integrations/sources/clickhouse.md +++ b/docs/integrations/sources/clickhouse.md @@ -78,6 +78,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------|:-----------------------------------------------------------------| +| 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | | 0.1.13 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | | 0.1.12 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | | 0.1.10 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | @@ -95,6 +96,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:---| :--- |:---------------------------------------------------------|:---------------------------------------------------------------------------| +| 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | | 0.1.13 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | | 0.1.9 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | | 0.1.6 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided | From 8956887f0d2574b04f6cf98bb15c4acbec7e4b67 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 27 Sep 2022 10:18:41 +0000 Subject: [PATCH 10/10] auto-bump connector version [ci skip] --- .../main/resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 929b8072502f4..f206b3f6cb06d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -176,7 +176,7 @@ - name: ClickHouse sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c dockerRepository: airbyte/source-clickhouse - dockerImageTag: 0.1.13 + dockerImageTag: 0.1.14 documentationUrl: https://docs.airbyte.io/integrations/sources/clickhouse icon: cliskhouse.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 046567ac9230c..1c5bfef7218b6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1823,7 +1823,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-clickhouse:0.1.13" +- dockerImage: "airbyte/source-clickhouse:0.1.14" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: @@ -1840,6 +1840,7 @@ description: "The host endpoint of the Clickhouse cluster." title: "Host" type: "string" + order: 0 port: description: "The port of the database." title: "Port" @@ -1849,26 +1850,40 @@ default: 8123 examples: - "8123" + order: 1 database: description: "The name of the database." title: "Database" type: "string" examples: - "default" + order: 2 username: description: "The username which is used to access the database." title: "Username" type: "string" + order: 3 password: description: "The password associated with this username." title: "Password" type: "string" airbyte_secret: true + order: 4 + jdbc_url_params: + description: "Additional properties to pass to the JDBC URL string when\ + \ connecting to the database formatted as 'key=value' pairs separated\ + \ by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more\ + \ information read about JDBC URL parameters." + title: "JDBC URL Parameters (Advanced)" + type: "string" + order: 5 ssl: title: "SSL Connection" description: "Encrypt data using SSL." type: "boolean" default: true + order: 6 tunnel_method: type: "object" title: "SSH Tunnel Method"