From 779a2f8a16666fb1955698e7d31360e3ad4101a7 Mon Sep 17 00:00:00 2001 From: Noah Kawasaki Date: Thu, 20 Jan 2022 00:44:08 +0000 Subject: [PATCH 01/16] Add support for jdbc_url_params optional JDBC parameters for Source Snowflake and Destination Snowflake --- .../snowflake/SnowflakeDatabase.java | 7 ++- .../src/main/resources/spec.json | 8 +++- .../SnowflakeSource.java | 43 ++++++++++++------- .../src/main/resources/spec.json | 6 +++ 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java index 76ebade012a98..32229246307a2 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java @@ -24,7 +24,7 @@ public class SnowflakeDatabase { private static final SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer(); public static Connection getConnection(final JsonNode config) throws SQLException { - final String connectUrl = String.format("jdbc:snowflake://%s", config.get("host").asText()); + final String connectUrl = String.format("jdbc:snowflake://%s/?", config.get("host").asText()); final Properties properties = new Properties(); @@ -47,6 +47,11 @@ public static Connection getConnection(final JsonNode config) throws SQLExceptio // https://stackoverflow.com/questions/67409650/snowflake-jdbc-driver-internal-error-fail-to-retrieve-row-count-for-first-arrow properties.put("JDBC_QUERY_RESULT_FORMAT", "JSON"); + // https://docs.snowflake.com/en/user-guide/jdbc-configure.html#jdbc-driver-connection-string + if (config.has("jdbc_url_params")) { + connectUrl.append(config.get("jdbc_url_params").asText()); + } + return DriverManager.getConnection(connectUrl, properties); } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json index de555da19b948..9239cb069e8e6 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json @@ -68,11 +68,17 @@ "title": "Password", "order": 6 }, + "jdbc_url_params": { + "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "title": "JDBC URL Params", + "type": "string", + "order": 7 + }, "loading_method": { "type": "object", "title": "Loading Method", "description": "Loading method used to send data to Snowflake.", - "order": 7, + "order": 8, "oneOf": [ { "title": "[Recommended] Internal Staging", diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java index cdffb1abac6f1..3d4663cee309a 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java @@ -33,23 +33,34 @@ public static void main(final String[] args) throws Exception { @Override public JsonNode toDatabaseConfig(final JsonNode config) { - return Jsons.jsonNode(ImmutableMap.builder() - .put("jdbc_url", String.format("jdbc:snowflake://%s/", - config.get("host").asText())) - .put("host", config.get("host").asText()) + + final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:snowflake://%s/?", + config.get("host").asText()); + + // Add required properties + jdbcUrl.append(String.format("role=%s;warehouse=%s;database=%s;schema=%s;JDBC_QUERY_RESULT_FORMAT=%s;CLIENT_SESSION_KEEP_ALIVE=%s;", + config.get("role").asText(), + config.get("warehouse").asText(), + config.get("database").asText(), + config.get("schema").asText(), + // Needed for JDK17 - see + // https://stackoverflow.com/questions/67409650/snowflake-jdbc-driver-internal-error-fail-to-retrieve-row-count-for-first-arrow + "JSON", + true)); + + // https://docs.snowflake.com/en/user-guide/jdbc-configure.html#jdbc-driver-connection-string + if (config.has("jdbc_url_params")) { + jdbcUrl.append("&").append(config.get("jdbc_url_params").asText()); + } + + LOGGER.warn(jdbcUrl.toString()); + + final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) - .put("password", config.get("password").asText()) - .put("connection_properties", - String.format("role=%s;warehouse=%s;database=%s;schema=%s;JDBC_QUERY_RESULT_FORMAT=%s;CLIENT_SESSION_KEEP_ALIVE=%s;", - config.get("role").asText(), - config.get("warehouse").asText(), - config.get("database").asText(), - config.get("schema").asText(), - // Needed for JDK17 - see - // https://stackoverflow.com/questions/67409650/snowflake-jdbc-driver-internal-error-fail-to-retrieve-row-count-for-first-arrow - "JSON", - true)) - .build()); + .put("password", config.get("password").asText() + .put("jdbc_url", jdbcUrl.toString()); + + return Jsons.jsonNode(configBuilder.build()); } @Override diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json index afc06d871af63..0dcfefc3a0430 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json @@ -63,6 +63,12 @@ "airbyte_secret": true, "title": "Password", "order": 6 + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "title": "JDBC URL Params", + "type": "string", + "order": 7 } } } From c82b9da4761e7ddcd5a4f9b5c77e7df90dc8c847 Mon Sep 17 00:00:00 2001 From: Noah Kawasaki Date: Thu, 20 Jan 2022 00:58:59 +0000 Subject: [PATCH 02/16] Update docs and fix param separator in SnowflakeSource --- .../SnowflakeSource.java | 2 +- docs/integrations/destinations/snowflake.md | 3 ++- docs/integrations/sources/snowflake.md | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java index 3d4663cee309a..ea8cea538ec59 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java @@ -38,7 +38,7 @@ public JsonNode toDatabaseConfig(final JsonNode config) { config.get("host").asText()); // Add required properties - jdbcUrl.append(String.format("role=%s;warehouse=%s;database=%s;schema=%s;JDBC_QUERY_RESULT_FORMAT=%s;CLIENT_SESSION_KEEP_ALIVE=%s;", + jdbcUrl.append(String.format("role=%s&warehouse=%s&database=%s&schema=%s&JDBC_QUERY_RESULT_FORMAT=%s&CLIENT_SESSION_KEEP_ALIVE=%s;", config.get("role").asText(), config.get("warehouse").asText(), config.get("database").asText(), diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index 8c9885b71a862..034f13dae9243 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -113,6 +113,7 @@ You should now have all the requirements needed to configure Snowflake as a dest * **Schema** * **Username** * **Password** +* **JDBC URL Params** (Optional) ## Notes about Snowflake Naming Conventions @@ -198,6 +199,7 @@ Finally, you need to add read/write permissions to your bucket with that email. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----- | :------ | +| 0.4.2 | 2022-01-19 | [#9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | | 0.4.2 | 2022-01-10 | [#9141](https://github.com/airbytehq/airbyte/pull/9141) | Fixed duplicate rows on retries | | 0.4.1 | 2021-01-06 | [#9311](https://github.com/airbytehq/airbyte/pull/9311) | Update сreating schema during check | | 0.4.0 | 2021-12-27 | [#9063](https://github.com/airbytehq/airbyte/pull/9063) | Updated normalization to produce permanent tables | @@ -215,4 +217,3 @@ Finally, you need to add read/write permissions to your bucket with that email. | 0.3.12 | 2021-07-30 | [#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | | 0.3.11 | 2021-07-21 | [#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | | 0.3.10 | 2021-07-12 | [#4713](https://github.com/airbytehq/airbyte/pull/4713)| Tag traffic with `airbyte` label to enable optimization opportunities from Snowflake | - diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md index 17a25a3a377e8..439e0eb685d46 100644 --- a/docs/integrations/sources/snowflake.md +++ b/docs/integrations/sources/snowflake.md @@ -30,7 +30,8 @@ The Snowflake source does not alter the schema present in your warehouse. Depend 6. **Schema** 7. **Username** 8. **Password** -9. Create a dedicated read-only Airbyte user and role with access to all schemas needed for replication. +9. **JDBC URL Params** (Optional) +10. Create a dedicated read-only Airbyte user and role with access to all schemas needed for replication. ### Setup guide @@ -75,9 +76,9 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.6 | 2022-01-19 | [#9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | | 0.1.5 | 2022-01-19 | [9567](https://github.com/airbytehq/airbyte/pull/9567) | Added parameter for keeping JDBC session alive | | 0.1.4 | 2021-12-30 | [9203](https://github.com/airbytehq/airbyte/pull/9203) | Update connector fields title/description | | 0.1.3 | 2021-01-11 | [9304](https://github.com/airbytehq/airbyte/pull/9304) | Upgrade version of JDBC driver | | 0.1.2 | 2021-10-21 | [7257](https://github.com/airbytehq/airbyte/pull/7257) | Fixed parsing of extreme values for FLOAT and NUMBER data types | | 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | - From a42aec655d886fbe9b7e52e11f61ed7cef63b329 Mon Sep 17 00:00:00 2001 From: Noah Kawasaki Date: Thu, 20 Jan 2022 01:02:00 +0000 Subject: [PATCH 03/16] Small markdown fixes --- docs/integrations/destinations/snowflake.md | 2 +- docs/integrations/sources/snowflake.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index 034f13dae9243..f5f880127e0fc 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -199,7 +199,7 @@ Finally, you need to add read/write permissions to your bucket with that email. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----- | :------ | -| 0.4.2 | 2022-01-19 | [#9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | +| 0.4.3 | 2022-01-19 | [#9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | | 0.4.2 | 2022-01-10 | [#9141](https://github.com/airbytehq/airbyte/pull/9141) | Fixed duplicate rows on retries | | 0.4.1 | 2021-01-06 | [#9311](https://github.com/airbytehq/airbyte/pull/9311) | Update сreating schema during check | | 0.4.0 | 2021-12-27 | [#9063](https://github.com/airbytehq/airbyte/pull/9063) | Updated normalization to produce permanent tables | diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md index 439e0eb685d46..4628d30ae4806 100644 --- a/docs/integrations/sources/snowflake.md +++ b/docs/integrations/sources/snowflake.md @@ -76,7 +76,7 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.6 | 2022-01-19 | [#9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | +| 0.1.6 | 2022-01-19 | [9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | | 0.1.5 | 2022-01-19 | [9567](https://github.com/airbytehq/airbyte/pull/9567) | Added parameter for keeping JDBC session alive | | 0.1.4 | 2021-12-30 | [9203](https://github.com/airbytehq/airbyte/pull/9203) | Update connector fields title/description | | 0.1.3 | 2021-01-11 | [9304](https://github.com/airbytehq/airbyte/pull/9304) | Upgrade version of JDBC driver | From e0e800bec581aa62d068ff19d51703e778abf217 Mon Sep 17 00:00:00 2001 From: Noah Kawasaki Date: Thu, 20 Jan 2022 16:04:40 +0000 Subject: [PATCH 04/16] Fix Java syntax --- .../destination/snowflake/SnowflakeDatabase.java | 8 +++++--- .../SnowflakeSource.java | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java index 32229246307a2..7176aa89d4095 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java @@ -24,7 +24,9 @@ public class SnowflakeDatabase { private static final SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer(); public static Connection getConnection(final JsonNode config) throws SQLException { - final String connectUrl = String.format("jdbc:snowflake://%s/?", config.get("host").asText()); + + final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:snowflake://%s/?", + config.get("host").asText())); final Properties properties = new Properties(); @@ -49,10 +51,10 @@ public static Connection getConnection(final JsonNode config) throws SQLExceptio // https://docs.snowflake.com/en/user-guide/jdbc-configure.html#jdbc-driver-connection-string if (config.has("jdbc_url_params")) { - connectUrl.append(config.get("jdbc_url_params").asText()); + jdbcUrl.append(config.get("jdbc_url_params").asText()); } - return DriverManager.getConnection(connectUrl, properties); + return DriverManager.getConnection(jdbcUrl.toString(), properties); } public static JdbcDatabase getDatabase(final JsonNode config) { diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java index ea8cea538ec59..97ae591e50f58 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java @@ -35,7 +35,7 @@ public static void main(final String[] args) throws Exception { public JsonNode toDatabaseConfig(final JsonNode config) { final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:snowflake://%s/?", - config.get("host").asText()); + config.get("host").asText())); // Add required properties jdbcUrl.append(String.format("role=%s&warehouse=%s&database=%s&schema=%s&JDBC_QUERY_RESULT_FORMAT=%s&CLIENT_SESSION_KEEP_ALIVE=%s;", @@ -57,7 +57,7 @@ public JsonNode toDatabaseConfig(final JsonNode config) { final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) - .put("password", config.get("password").asText() + .put("password", config.get("password").asText()) .put("jdbc_url", jdbcUrl.toString()); return Jsons.jsonNode(configBuilder.build()); From b01a03830a191e88739bbed636230bc5a0705fb9 Mon Sep 17 00:00:00 2001 From: Noah Kawasaki Date: Thu, 20 Jan 2022 21:07:11 +0000 Subject: [PATCH 05/16] Fix one ; --- .../SnowflakeSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java index 97ae591e50f58..722e72479939f 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java @@ -38,7 +38,7 @@ public JsonNode toDatabaseConfig(final JsonNode config) { config.get("host").asText())); // Add required properties - jdbcUrl.append(String.format("role=%s&warehouse=%s&database=%s&schema=%s&JDBC_QUERY_RESULT_FORMAT=%s&CLIENT_SESSION_KEEP_ALIVE=%s;", + jdbcUrl.append(String.format("role=%s&warehouse=%s&database=%s&schema=%s&JDBC_QUERY_RESULT_FORMAT=%s&CLIENT_SESSION_KEEP_ALIVE=%s", config.get("role").asText(), config.get("warehouse").asText(), config.get("database").asText(), From 80af2f770b6d297bcf98bc9c18900899d03f5784 Mon Sep 17 00:00:00 2001 From: Noah Kawasaki Date: Thu, 20 Jan 2022 21:42:52 +0000 Subject: [PATCH 06/16] Add logger for JDBC string --- .../destination/snowflake/SnowflakeDatabase.java | 5 +++++ .../SnowflakeSource.java | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java index 7176aa89d4095..57790906c4b00 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java @@ -13,6 +13,8 @@ import java.sql.SQLException; import java.time.Duration; import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * SnowflakeDatabase contains helpers to create connections to and run queries on Snowflake. @@ -22,6 +24,7 @@ public class SnowflakeDatabase { private static final Duration NETWORK_TIMEOUT = Duration.ofMinutes(1); private static final Duration QUERY_TIMEOUT = Duration.ofHours(3); private static final SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer(); + private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeDatabase.class); public static Connection getConnection(final JsonNode config) throws SQLException { @@ -54,6 +57,8 @@ public static Connection getConnection(final JsonNode config) throws SQLExceptio jdbcUrl.append(config.get("jdbc_url_params").asText()); } + LOGGER.info(jdbcUrl.toString()); + return DriverManager.getConnection(jdbcUrl.toString(), properties); } diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java index 722e72479939f..b404e4fc3b3b3 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java @@ -53,7 +53,7 @@ public JsonNode toDatabaseConfig(final JsonNode config) { jdbcUrl.append("&").append(config.get("jdbc_url_params").asText()); } - LOGGER.warn(jdbcUrl.toString()); + LOGGER.info(jdbcUrl.toString()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) From 7b1ddf7d4bd61dd932eeb9d6325dc801a75740aa Mon Sep 17 00:00:00 2001 From: Noah Kawasaki Date: Thu, 20 Jan 2022 21:50:22 +0000 Subject: [PATCH 07/16] add MIT license under docs/integrations --- docs/integrations/LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 docs/integrations/LICENSE diff --git a/docs/integrations/LICENSE b/docs/integrations/LICENSE new file mode 100644 index 0000000000000..ec45d182fcb90 --- /dev/null +++ b/docs/integrations/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Airbyte, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From cdfc867d638de9779bf668913c10ce2e81169aa4 Mon Sep 17 00:00:00 2001 From: Noah Kawasaki Date: Thu, 20 Jan 2022 21:52:39 +0000 Subject: [PATCH 08/16] Bump changelog patch version --- docs/integrations/destinations/snowflake.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index f3b9906a9d8aa..45570e4ab1d7a 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -218,7 +218,7 @@ Finally, you need to add read/write permissions to your bucket with that email. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----- | :------ | -| 0.4.3 | 2022-01-20 | [#9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | +| 0.4.4 | 2022-01-20 | [#9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | | 0.4.3 | 2022-01-20 | [#9531](https://github.com/airbytehq/airbyte/pull/9531) | Start using new S3StreamCopier and expose the purgeStagingData option | | 0.4.2 | 2022-01-10 | [#9141](https://github.com/airbytehq/airbyte/pull/9141) | Fixed duplicate rows on retries | | 0.4.1 | 2021-01-06 | [#9311](https://github.com/airbytehq/airbyte/pull/9311) | Update сreating schema during check | From febf9041a6a7a81379ceaecc72d72b09d71e94ba Mon Sep 17 00:00:00 2001 From: Noah Kawasaki Date: Fri, 21 Jan 2022 17:21:01 +0000 Subject: [PATCH 09/16] Bump all versions, remove LICENSE --- .../424892c4-daac-4491-b35d-c6688ba547ba.json | 2 +- .../e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json | 2 +- .../seed/destination_definitions.yaml | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../destination-snowflake/Dockerfile | 2 +- .../connectors/source-snowflake/Dockerfile | 2 +- docs/integrations/LICENSE | 21 ------------------- 7 files changed, 6 insertions(+), 27 deletions(-) delete mode 100644 docs/integrations/LICENSE diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json index acf0a7d031470..bc563225389e8 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba", "name": "Snowflake", "dockerRepository": "airbyte/destination-snowflake", - "dockerImageTag": "0.4.3", + "dockerImageTag": "0.4.4", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake", "icon": "snowflake.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json index 2fccee5cd4e1c..392053888a1b7 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2", "name": "Snowflake", "dockerRepository": "airbyte/source-snowflake", - "dockerImageTag": "0.1.5", + "dockerImageTag": "0.1.6", "documentationUrl": "https://docs.airbyte.io/integrations/sources/snowflake", "icon": "snowflake.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 712996128cd71..dc4f0cf342adf 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -179,7 +179,7 @@ - name: Snowflake destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba dockerRepository: airbyte/destination-snowflake - dockerImageTag: 0.4.3 + dockerImageTag: 0.4.4 documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake icon: snowflake.svg - name: MariaDB ColumnStore diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index aed21d4979a6b..acf770c5f7787 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -676,7 +676,7 @@ - name: Snowflake sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2 dockerRepository: airbyte/source-snowflake - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/sources/snowflake icon: snowflake.svg sourceType: database diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index b0e293930d9cc..c7d1bac24852c 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -18,5 +18,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.4.3 +LABEL io.airbyte.version=0.4.4 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile index 22356354f9002..3c35d62e1d0ea 100644 --- a/airbyte-integrations/connectors/source-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/source-snowflake/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-snowflake COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/source-snowflake diff --git a/docs/integrations/LICENSE b/docs/integrations/LICENSE deleted file mode 100644 index ec45d182fcb90..0000000000000 --- a/docs/integrations/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Airbyte, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. From df0b3f84e2f8ff7331b32ff9fb8397bc9ee02396 Mon Sep 17 00:00:00 2001 From: Noah Kawasaki Date: Thu, 27 Jan 2022 00:24:29 +0000 Subject: [PATCH 10/16] JDBC URL string --- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- .../source-java-jdbc/src/main/resources/spec.json.hbs | 2 +- .../destination-snowflake/src/main/resources/spec.json | 2 +- .../src/test/resources/expected_spec.json | 2 +- .../connectors/source-mysql/src/main/resources/spec.json | 2 +- .../source-scaffold-java-jdbc/src/main/resources/spec.json | 2 +- .../connectors/source-snowflake/src/main/resources/spec.json | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 4403662e9a770..22adfe7088d00 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4963,7 +4963,7 @@ airbyte_secret: true order: 4 jdbc_url_params: - description: "Additional properties to pass to the jdbc url string when\ + description: "Additional properties to pass to the JDBC URL string when\ \ connecting to the database formatted as 'key=value' pairs separated\ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." title: "JDBC URL Params" diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs index 9588875eb6255..690741787f929 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs @@ -38,7 +38,7 @@ "order": 4 }, "jdbc_url_params": { - "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", "type": "string", "order": 5 }, diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json index 7b740ed13366c..64902908b80b1 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json @@ -69,7 +69,7 @@ "order": 6 }, "jdbc_url_params": { - "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", "order": 7 diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json index 9c811ed82bf64..1aac8a064b23e 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json @@ -43,7 +43,7 @@ "order": 4 }, "jdbc_url_params": { - "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", "order": 5 diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index 050cf00ec6d05..c6a652f3e48ef 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -43,7 +43,7 @@ "order": 4 }, "jdbc_url_params": { - "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", "order": 5 diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json index 62de3fc0db353..fb75d79434a2d 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json @@ -38,7 +38,7 @@ "order": 4 }, "jdbc_url_params": { - "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", "type": "string", "order": 5 }, diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json index 0dcfefc3a0430..95b989811537c 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json @@ -65,7 +65,7 @@ "order": 6 }, "jdbc_url_params": { - "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", "order": 7 From 607e54f9e2488c4e8f275ce3b695402750f6f20f Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 28 Jan 2022 16:20:49 +0100 Subject: [PATCH 11/16] fix typo --- .../destination-snowflake/src/main/resources/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json index f6830cfd21bdf..d2179cc994132 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json @@ -77,7 +77,7 @@ "loading_method": { "type": "object", "title": "Loading Method", - "description": "The loading method used to send data to Snowflake.", + "description": "Loading method used to send data to Snowflake.", "order": 8, "oneOf": [ { From be732ffbd2c1456ebc3a257695d93d40f95790f2 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 28 Jan 2022 16:22:02 +0100 Subject: [PATCH 12/16] update dest spec --- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 307608e872d72..0d76e9fd9d786 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3787,7 +3787,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-snowflake:6" +- dockerImage: "airbyte/destination-snowflake:0.4.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake" connectionSpecification: From d40931a0e19637d17858e6ba6d1dee8e50b03640 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 28 Jan 2022 17:05:26 +0100 Subject: [PATCH 13/16] fix typo --- airbyte-integrations/connectors/destination-snowflake/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index a2eb85057f6d9..f490896f558d5 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -19,4 +19,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 LABEL io.airbyte.version=0.4.6 + LABEL io.airbyte.name=airbyte/destination-snowflake From 4fb3ea8bc3588e2bd355b8b3159cc766f1a126eb Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 28 Jan 2022 17:06:53 +0100 Subject: [PATCH 14/16] fix wrong change --- .../destination-snowflake/src/main/resources/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json index d2179cc994132..f6830cfd21bdf 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json @@ -77,7 +77,7 @@ "loading_method": { "type": "object", "title": "Loading Method", - "description": "Loading method used to send data to Snowflake.", + "description": "The loading method used to send data to Snowflake.", "order": 8, "oneOf": [ { From 2b44ce0b63c7b7a909bbe48566be194f6fd619b6 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 28 Jan 2022 19:23:25 +0100 Subject: [PATCH 15/16] update source_specs.yaml --- .../init/src/main/resources/seed/source_specs.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index fe4ccaf06f2e9..40a16d8462613 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -7339,7 +7339,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-snowflake:0.1.5" +- dockerImage: "airbyte/source-snowflake:0.1.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/snowflake" connectionSpecification: @@ -7405,6 +7405,13 @@ airbyte_secret: true title: "Password" order: 6 + jdbc_url_params: + description: "Additional properties to pass to the JDBC URL string when\ + \ connecting to the database formatted as 'key=value' pairs separated\ + \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." + title: "JDBC URL Params" + type: "string" + order: 7 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] From 52fb5404ff4336bf53a244e7d398a5347d7727eb Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 28 Jan 2022 19:34:44 +0100 Subject: [PATCH 16/16] update destinations_specs.yaml --- .../init/src/main/resources/seed/destination_specs.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 0d76e9fd9d786..ebd87e918edb6 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3856,11 +3856,18 @@ airbyte_secret: true title: "Password" order: 6 + jdbc_url_params: + description: "Additional properties to pass to the JDBC URL string when\ + \ connecting to the database formatted as 'key=value' pairs separated\ + \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." + title: "JDBC URL Params" + type: "string" + order: 7 loading_method: type: "object" title: "Loading Method" description: "The loading method used to send data to Snowflake." - order: 7 + order: 8 oneOf: - title: "[Recommended] Internal Staging" additionalProperties: false