diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index fd7b5772272cc..72d7d05a87247 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -37,7 +37,7 @@ - name: Clickhouse destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 dockerRepository: airbyte/destination-clickhouse - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse - name: DynamoDB destinationDefinitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89 @@ -108,7 +108,7 @@ - name: MS SQL Server destinationDefinitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf dockerRepository: airbyte/destination-mssql - dockerImageTag: 0.1.14 + dockerImageTag: 0.1.15 documentationUrl: https://docs.airbyte.io/integrations/destinations/mssql icon: mssql.svg - name: MeiliSearch @@ -126,19 +126,19 @@ - name: MySQL destinationDefinitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 dockerRepository: airbyte/destination-mysql - dockerImageTag: 0.1.17 + dockerImageTag: 0.1.18 documentationUrl: https://docs.airbyte.io/integrations/destinations/mysql icon: mysql.svg - name: Oracle destinationDefinitionId: 3986776d-2319-4de9-8af8-db14c0996e72 dockerRepository: airbyte/destination-oracle - dockerImageTag: 0.1.13 + dockerImageTag: 0.1.15 documentationUrl: https://docs.airbyte.io/integrations/destinations/oracle icon: oracle.svg - name: Postgres destinationDefinitionId: 25c5221d-dce2-4163-ade9-739ef790f503 dockerRepository: airbyte/destination-postgres - dockerImageTag: 0.3.14 + dockerImageTag: 0.3.15 documentationUrl: https://docs.airbyte.io/integrations/destinations/postgres icon: postgresql.svg - name: Pulsar @@ -162,7 +162,7 @@ - name: Redshift destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc dockerRepository: airbyte/destination-redshift - dockerImageTag: 0.3.26 + dockerImageTag: 0.3.27 documentationUrl: https://docs.airbyte.io/integrations/destinations/redshift icon: redshift.svg - name: Rockset @@ -185,7 +185,7 @@ - name: Snowflake destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba dockerRepository: airbyte/destination-snowflake - dockerImageTag: 0.4.16 + dockerImageTag: 0.4.17 documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake icon: snowflake.svg resourceRequirements: @@ -207,7 +207,7 @@ - name: MariaDB ColumnStore destinationDefinitionId: 294a4790-429b-40ae-9516-49826b9702e1 dockerRepository: airbyte/destination-mariadb-columnstore - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/destinations/mariadb-columnstore icon: mariadb.svg - name: Streamr diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 091d757ed3a3b..1d54b454797d5 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -667,7 +667,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-clickhouse:0.1.3" +- dockerImage: "airbyte/destination-clickhouse:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: @@ -2077,7 +2077,7 @@ supportsDBT: false supported_destination_sync_modes: - "append" -- dockerImage: "airbyte/destination-mssql:0.1.14" +- dockerImage: "airbyte/destination-mssql:0.1.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql" connectionSpecification: @@ -2464,7 +2464,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-mysql:0.1.17" +- dockerImage: "airbyte/destination-mysql:0.1.18" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mysql" connectionSpecification: @@ -2629,7 +2629,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-oracle:0.1.13" +- dockerImage: "airbyte/destination-oracle:0.1.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/oracle" connectionSpecification: @@ -2853,11 +2853,11 @@ order: 4 supportsIncremental: true supportsNormalization: false - supportsDBT: true + supportsDBT: false supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-postgres:0.3.14" +- dockerImage: "airbyte/destination-postgres:0.3.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/postgres" connectionSpecification: @@ -3272,7 +3272,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-redshift:0.3.26" +- dockerImage: "airbyte/destination-redshift:0.3.27" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/redshift" connectionSpecification: @@ -3825,7 +3825,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-snowflake:0.4.16" +- dockerImage: "airbyte/destination-snowflake:0.4.17" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake" connectionSpecification: @@ -4078,7 +4078,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-mariadb-columnstore:0.1.3" +- dockerImage: "airbyte/destination-mariadb-columnstore:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mariadb-columnstore" connectionSpecification: diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java index e25bd240e2a05..3fb65f85f73b5 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java @@ -4,6 +4,7 @@ package io.airbyte.db; +import com.google.common.collect.Maps; import io.airbyte.commons.lang.Exceptions; import io.airbyte.db.bigquery.BigQueryDatabase; import io.airbyte.db.jdbc.DefaultJdbcDatabase; @@ -15,7 +16,6 @@ import io.airbyte.db.mongodb.MongoDatabase; import java.io.IOException; import java.util.Map; -import java.util.Optional; import java.util.function.Function; import lombok.val; import org.apache.commons.dbcp2.BasicDataSource; @@ -41,7 +41,7 @@ public static Database createPostgresDatabaseWithRetry(final String username, try { val infinity = Integer.MAX_VALUE; database = createPostgresDatabaseWithRetryTimeout(username, password, jdbcConnectionString, isDbReady, infinity); - } catch (IOException e) { + } catch (final IOException e) { // This should theoretically never happen since we set the timeout to be a very high number. } } @@ -131,9 +131,9 @@ public static Database createDatabase(final String username, final String jdbcConnectionString, final String driverClassName, final SQLDialect dialect, - final String connectionProperties) { + final Map connectionProperties) { final BasicDataSource connectionPool = - createBasicDataSource(username, password, jdbcConnectionString, driverClassName, Optional.ofNullable(connectionProperties)); + createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); return new Database(connectionPool, dialect); } @@ -159,7 +159,7 @@ public static JdbcDatabase createJdbcDatabase(final String username, final String password, final String jdbcConnectionString, final String driverClassName, - final String connectionProperties) { + final Map connectionProperties) { return createJdbcDatabase(username, password, jdbcConnectionString, driverClassName, connectionProperties, JdbcUtils.getDefaultSourceOperations()); } @@ -168,10 +168,10 @@ public static JdbcDatabase createJdbcDatabase(final String username, final String password, final String jdbcConnectionString, final String driverClassName, - final String connectionProperties, + final Map connectionProperties, final JdbcCompatibleSourceOperations sourceOperations) { final BasicDataSource connectionPool = - createBasicDataSource(username, password, jdbcConnectionString, driverClassName, Optional.ofNullable(connectionProperties)); + createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); return new DefaultJdbcDatabase(connectionPool, sourceOperations); } @@ -181,10 +181,10 @@ public static JdbcDatabase createStreamingJdbcDatabase(final String username, final String jdbcConnectionString, final String driverClassName, final JdbcStreamingQueryConfiguration jdbcStreamingQuery, - final String connectionProperties, + final Map connectionProperties, final JdbcCompatibleSourceOperations sourceOperations) { final BasicDataSource connectionPool = - createBasicDataSource(username, password, jdbcConnectionString, driverClassName, Optional.ofNullable(connectionProperties)); + createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); return new StreamingJdbcDatabase(connectionPool, sourceOperations, jdbcStreamingQuery); } @@ -194,27 +194,7 @@ private static BasicDataSource createBasicDataSource(final String username, final String jdbcConnectionString, final String driverClassName) { return createBasicDataSource(username, password, jdbcConnectionString, driverClassName, - Optional.empty()); - } - - /** - * Prefer to use the method that takes in the connection properties as a map. - */ - @Deprecated - private static BasicDataSource createBasicDataSource(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final Optional connectionProperties) { - final BasicDataSource connectionPool = new BasicDataSource(); - connectionPool.setDriverClassName(driverClassName); - connectionPool.setUsername(username); - connectionPool.setPassword(password); - connectionPool.setInitialSize(0); - connectionPool.setMaxTotal(5); - connectionPool.setUrl(jdbcConnectionString); - connectionProperties.ifPresent(connectionPool::setConnectionProperties); - return connectionPool; + Maps.newHashMap()); } public static BasicDataSource createBasicDataSource(final String username, diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index a7d681857aae6..05caf59336b02 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -4,6 +4,10 @@ package io.airbyte.db.jdbc; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Maps; +import java.util.HashMap; +import java.util.Map; import org.jooq.JSONFormat; public class JdbcUtils { @@ -24,4 +28,30 @@ public static String getFullyQualifiedTableName(final String schemaName, final S return schemaName != null ? schemaName + "." + tableName : tableName; } + public static Map parseJdbcParameters(final JsonNode config, final String jdbcUrlParamsKey) { + if (config.has(jdbcUrlParamsKey)) { + return parseJdbcParameters(config.get(jdbcUrlParamsKey).asText()); + } else { + return Maps.newHashMap(); + } + } + + public static Map parseJdbcParameters(final String jdbcPropertiesString) { + final Map parameters = new HashMap<>(); + if (!jdbcPropertiesString.isBlank()) { + final String[] keyValuePairs = jdbcPropertiesString.split("&"); + for (final String kv : keyValuePairs) { + final String[] split = kv.split("="); + if (split.length == 2) { + parameters.put(split[0], split[1]); + } else { + throw new IllegalArgumentException( + "jdbc_url_params must be formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). Got " + + jdbcPropertiesString); + } + } + } + return parameters; + } + } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile index f6069d7e1c60f..9fc1aa0f33db4 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json index 3d15378c40808..d27dea83a7f41 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json @@ -3,12 +3,21 @@ "supportsIncremental": true, "supportsNormalization": true, "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], + "supported_destination_sync_modes": [ + "overwrite", + "append", + "append_dedup" + ], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "ClickHouse Destination Spec", "type": "object", - "required": ["host", "port", "database", "username"], + "required": [ + "host", + "port", + "database", + "username" + ], "additionalProperties": true, "properties": { "host": { @@ -24,27 +33,41 @@ "minimum": 0, "maximum": 65536, "default": 8123, - "examples": ["8123"], + "examples": [ + "8123" + ], "order": 1 }, + "tcp-port": { + "title": "Native Port", + "description": "Native port (not the JDBC) of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 9000, + "examples": [ + "9000" + ], + "order": 2 + }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", - "order": 2 + "order": 3 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", - "order": 3 + "order": 4 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, - "order": 4 + "order": 5 }, "tunnel_method": { "type": "object", @@ -53,7 +76,9 @@ "oneOf": [ { "title": "No Tunnel", - "required": ["tunnel_method"], + "required": [ + "tunnel_method" + ], "properties": { "tunnel_method": { "description": "No ssh tunnel needed to connect to database", @@ -92,7 +117,9 @@ "minimum": 0, "maximum": 65536, "default": 22, - "examples": ["22"], + "examples": [ + "22" + ], "order": 2 }, "tunnel_user": { @@ -140,7 +167,9 @@ "minimum": 0, "maximum": 65536, "default": 22, - "examples": ["22"], + "examples": [ + "22" + ], "order": 2 }, "tunnel_user": { diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index f62452f078213..dd60f557cc1c9 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-clickhouse diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index 95c0b767b1436..b77459a8ee62b 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -15,8 +15,9 @@ import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; -import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,6 +32,10 @@ public class ClickhouseDestination extends AbstractJdbcDestination implements De private static final String PASSWORD = "password"; + static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( + "ssl", "true", + "sslmode", "none"); + public static Destination sshWrappedDestination() { return new SshWrappedDestination(new ClickhouseDestination(), HOST_KEY, PORT_KEY); } @@ -41,25 +46,14 @@ public ClickhouseDestination() { @Override public JsonNode toJdbcConfig(final JsonNode config) { - final List additionalParameters = new ArrayList<>(); - - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:clickhouse://%s:%s/%s?", + final String jdbcUrl = String.format("jdbc:clickhouse://%s:%s/%s?", config.get("host").asText(), config.get("port").asText(), - config.get("database").asText())); - - if (!config.has("ssl") || config.get("ssl").asBoolean()) { - additionalParameters.add("ssl=true"); - additionalParameters.add("sslmode=none"); - } - - if (!additionalParameters.isEmpty()) { - additionalParameters.forEach(x -> jdbcUrl.append(x).append("&")); - } + config.get("database").asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) - .put("jdbc_url", jdbcUrl.toString()); + .put("jdbc_url", jdbcUrl); if (config.has(PASSWORD)) { configBuilder.put(PASSWORD, config.get(PASSWORD).asText()); @@ -68,6 +62,10 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } + private boolean useSsl(final JsonNode config) { + return !config.has("ssl") || config.get("ssl").asBoolean(); + } + @Override public AirbyteConnectionStatus check(final JsonNode config) { try (final JdbcDatabase database = getDatabase(config)) { @@ -83,7 +81,17 @@ public AirbyteConnectionStatus check(final JsonNode config) { } } - public static void main(String[] args) throws Exception { + @Override + protected Map getDefaultConnectionProperties(final JsonNode config) { + if (useSsl(config)) { + return SSL_JDBC_PARAMETERS; + } else { + // No need for any parameters if the connection doesn't use SSL + return new HashMap<>(); + } + } + + public static void main(final String[] args) throws Exception { final Destination destination = ClickhouseDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", ClickhouseDestination.class); new IntegrationRunner(destination).run(args); diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java index 36fcb20ac7b40..75e5d96637422 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.map.MoreMaps; import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; @@ -25,7 +26,9 @@ import io.airbyte.protocol.models.JsonSchemaType; import java.time.Instant; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.junit.jupiter.api.AfterAll; @@ -44,6 +47,17 @@ public class ClickhouseDestinationTest { private static ConfiguredAirbyteCatalog catalog; private static JsonNode config; + private static final Map CONFIG_WITH_SSL = ImmutableMap.of( + "host", "localhost", + "port", "1337", + "username", "user", + "database", "db"); + + private static final Map CONFIG_NO_SSL = MoreMaps.merge( + CONFIG_WITH_SSL, + ImmutableMap.of( + "ssl", "false")); + @BeforeAll static void init() { db = new ClickHouseContainer("yandex/clickhouse-server"); @@ -76,6 +90,20 @@ static void cleanUp() { db.close(); } + @Test + void testDefaultParamsNoSSL() { + final Map defaultProperties = new ClickhouseDestination().getDefaultConnectionProperties( + Jsons.jsonNode(CONFIG_NO_SSL)); + assertEquals(new HashMap<>(), defaultProperties); + } + + @Test + void testDefaultParamsWithSSL() { + final Map defaultProperties = new ClickhouseDestination().getDefaultConnectionProperties( + Jsons.jsonNode(CONFIG_WITH_SSL)); + assertEquals(ClickhouseDestination.SSL_JDBC_PARAMETERS, defaultProperties); + } + @Test void sanityTest() throws Exception { final Destination dest = new ClickhouseDestination(); diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java index 4a58e8299a8fc..d8afb121b1526 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.destination.jdbc; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.map.MoreMaps; import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; @@ -17,6 +18,8 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.Map; +import java.util.Objects; import java.util.UUID; import java.util.function.Consumer; import org.slf4j.Logger; @@ -26,14 +29,12 @@ public abstract class AbstractJdbcDestination extends BaseConnector implements D private static final Logger LOGGER = LoggerFactory.getLogger(AbstractJdbcDestination.class); + public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params"; + private final String driverClass; private final NamingConventionTransformer namingResolver; private final SqlOperations sqlOperations; - protected String getDriverClass() { - return driverClass; - } - protected NamingConventionTransformer getNamingResolver() { return namingResolver; } @@ -89,9 +90,28 @@ protected JdbcDatabase getDatabase(final JsonNode config) { jdbcConfig.get("username").asText(), jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, jdbcConfig.get("jdbc_url").asText(), - driverClass); + driverClass, + getConnectionProperties(config)); + } + + protected Map getConnectionProperties(final JsonNode config) { + final Map customProperties = JdbcUtils.parseJdbcParameters(config, JDBC_URL_PARAMS_KEY); + final Map defaultProperties = getDefaultConnectionProperties(config); + assertCustomParametersDontOverwriteDefaultParameters(customProperties, defaultProperties); + return MoreMaps.merge(customProperties, defaultProperties); } + private void assertCustomParametersDontOverwriteDefaultParameters(final Map customParameters, + final Map defaultParameters) { + for (final String key : defaultParameters.keySet()) { + if (customParameters.containsKey(key) && !Objects.equals(customParameters.get(key), defaultParameters.get(key))) { + throw new IllegalArgumentException("Cannot overwrite default JDBC parameter " + key); + } + } + } + + protected abstract Map getDefaultConnectionProperties(final JsonNode config); + public abstract JsonNode toJdbcConfig(JsonNode config); @Override diff --git a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java new file mode 100644 index 0000000000000..a62128c41fc8c --- /dev/null +++ b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.jdbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.StandardNameTransformer; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +public class AbstractJdbcDestinationTest { + + private JsonNode buildConfigNoJdbcParameters() { + return Jsons.jsonNode(ImmutableMap.of( + "host", "localhost", + "port", 1337, + "username", "user", + "database", "db")); + } + + private JsonNode buildConfigWithExtraJdbcParameters(final String extraParam) { + return Jsons.jsonNode(ImmutableMap.of( + "host", "localhost", + "port", 1337, + "username", "user", + "database", "db", + "jdbc_url_params", extraParam)); + } + + @Test + void testNoExtraParamsNoDefault() { + final Map connectionProperties = new TestJdbcDestination().getConnectionProperties(buildConfigNoJdbcParameters()); + + final Map expectedProperties = ImmutableMap.of(); + assertEquals(expectedProperties, connectionProperties); + } + + @Test + void testNoExtraParamsWithDefault() { + final Map defaultProperties = ImmutableMap.of("A_PARAMETER", "A_VALUE"); + + final Map connectionProperties = new TestJdbcDestination(defaultProperties).getConnectionProperties( + buildConfigNoJdbcParameters()); + + assertEquals(defaultProperties, connectionProperties); + } + + @Test + void testExtraParamNoDefault() { + final String extraParam = "key1=value1&key2=value2&key3=value3"; + final Map connectionProperties = new TestJdbcDestination().getConnectionProperties( + buildConfigWithExtraJdbcParameters(extraParam)); + final Map expectedProperties = ImmutableMap.of( + "key1", "value1", + "key2", "value2", + "key3", "value3"); + assertEquals(expectedProperties, connectionProperties); + } + + @Test + void testExtraParamWithDefault() { + final Map defaultProperties = ImmutableMap.of("A_PARAMETER", "A_VALUE"); + final String extraParam = "key1=value1&key2=value2&key3=value3"; + final Map connectionProperties = new TestJdbcDestination(defaultProperties).getConnectionProperties( + buildConfigWithExtraJdbcParameters(extraParam)); + final Map expectedProperties = ImmutableMap.of( + "A_PARAMETER", "A_VALUE", + "key1", "value1", + "key2", "value2", + "key3", "value3"); + assertEquals(expectedProperties, connectionProperties); + } + + @Test + void testExtraParameterEqualToDefault() { + final Map defaultProperties = ImmutableMap.of("key1", "value1"); + final String extraParam = "key1=value1&key2=value2&key3=value3"; + final Map connectionProperties = new TestJdbcDestination(defaultProperties).getConnectionProperties( + buildConfigWithExtraJdbcParameters(extraParam)); + final Map expectedProperties = ImmutableMap.of( + "key1", "value1", + "key2", "value2", + "key3", "value3"); + assertEquals(expectedProperties, connectionProperties); + } + + @Test + void testExtraParameterDiffersFromDefault() { + final Map defaultProperties = ImmutableMap.of("key1", "value0"); + final String extraParam = "key1=value1&key2=value2&key3=value3"; + + assertThrows(IllegalArgumentException.class, () -> new TestJdbcDestination(defaultProperties).getConnectionProperties( + buildConfigWithExtraJdbcParameters(extraParam))); + } + + @Test + void testInvalidExtraParam() { + final String extraParam = "key1=value1&sdf&"; + assertThrows(IllegalArgumentException.class, + () -> new TestJdbcDestination().getConnectionProperties(buildConfigWithExtraJdbcParameters(extraParam))); + } + + static class TestJdbcDestination extends AbstractJdbcDestination { + + private final Map defaultProperties; + + public TestJdbcDestination() { + this(new HashMap<>()); + } + + public TestJdbcDestination(final Map defaultProperties) { + super("", new StandardNameTransformer(), new TestJdbcSqlOperations()); + this.defaultProperties = defaultProperties; + } + + @Override + protected Map getDefaultConnectionProperties(final JsonNode config) { + return defaultProperties; + } + + @Override + public JsonNode toJdbcConfig(final JsonNode config) { + return config; + } + + } + +} diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile b/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile index 0919436b17ef8..f33e8ba0b8d4d 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mariadb-columnstore COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-mariadb-columnstore diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java index 94568fcaf874a..f8d6ed49025fc 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; @@ -17,6 +16,7 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; import java.util.List; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,6 +27,9 @@ public class MariadbColumnstoreDestination extends AbstractJdbcDestination imple public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); + static final Map DEFAULT_JDBC_PARAMETERS = ImmutableMap.of( + "allowLoadLocalInfile", "true"); + public static Destination sshWrappedDestination() { return new SshWrappedDestination(new MariadbColumnstoreDestination(), HOST_KEY, PORT_KEY); } @@ -36,7 +39,7 @@ public MariadbColumnstoreDestination() { } @Override - public AirbyteConnectionStatus check(JsonNode config) { + public AirbyteConnectionStatus check(final JsonNode config) { try (final JdbcDatabase database = getDatabase(config)) { final MariadbColumnstoreSqlOperations mariadbColumnstoreSqlOperations = (MariadbColumnstoreSqlOperations) getSqlOperations(); final String outputSchema = getNamingResolver().getIdentifier(config.get("database").asText()); @@ -66,27 +69,20 @@ public AirbyteConnectionStatus check(JsonNode config) { } @Override - protected JdbcDatabase getDatabase(final JsonNode config) { - final JsonNode jdbcConfig = toJdbcConfig(config); - - return Databases.createJdbcDatabase( - jdbcConfig.get("username").asText(), - jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, - jdbcConfig.get("jdbc_url").asText(), - getDriverClass(), - "allowLoadLocalInfile=true"); + protected Map getDefaultConnectionProperties(final JsonNode config) { + return DEFAULT_JDBC_PARAMETERS; } @Override public JsonNode toJdbcConfig(final JsonNode config) { - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:mariadb://%s:%s/%s", + final String jdbcUrl = String.format("jdbc:mariadb://%s:%s/%s", config.get("host").asText(), config.get("port").asText(), - config.get("database").asText())); + config.get("database").asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) - .put("jdbc_url", jdbcUrl.toString()); + .put("jdbc_url", jdbcUrl); if (config.has("password")) { configBuilder.put("password", config.get("password").asText()); @@ -95,7 +91,7 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } - public static void main(String[] args) throws Exception { + public static void main(final String[] args) throws Exception { final Destination destination = MariadbColumnstoreDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", MariadbColumnstoreDestination.class); new IntegrationRunner(destination).run(args); diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile index 7cc7a7f59db19..7b094ef7293bf 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/destination-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mssql/Dockerfile b/airbyte-integrations/connectors/destination-mssql/Dockerfile index a48db180356c4..027c0699ecf1d 100644 --- a/airbyte-integrations/connectors/destination-mssql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.version=0.1.15 LABEL io.airbyte.name=airbyte/destination-mssql diff --git a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java index 86aeb9dc35477..ca8a95c7dcf25 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java +++ b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java @@ -12,8 +12,9 @@ import io.airbyte.integrations.base.ssh.SshWrappedDestination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import java.io.File; -import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,27 +31,44 @@ public MSSQLDestination() { super(DRIVER_CLASS, new MSSQLNameTransformer(), new SqlServerOperations()); } + @Override + protected Map getDefaultConnectionProperties(final JsonNode config) { + final HashMap properties = new HashMap<>(); + if (config.has("ssl_method")) { + switch (config.get("ssl_method").asText()) { + case "unencrypted" -> properties.put("encrypt", "false"); + case "encrypted_trust_server_certificate" -> { + properties.put("encrypt", "true"); + properties.put("trustServerCertificate", "true"); + } + case "encrypted_verify_certificate" -> { + properties.put("encrypt", "true"); + properties.put("trustStore", getTrustStoreLocation()); + final String trustStorePassword = System.getProperty("javax.net.ssl.trustStorePassword"); + if (trustStorePassword != null && !trustStorePassword.isEmpty()) { + properties.put("trustStorePassword", config.get("trustStorePassword").asText()); + } + if (config.has("hostNameInCertificate")) { + properties.put("hostNameInCertificate", config.get("hostNameInCertificate").asText()); + } + } + } + } + + return properties; + } + @Override public JsonNode toJdbcConfig(final JsonNode config) { final String schema = Optional.ofNullable(config.get("schema")).map(JsonNode::asText).orElse("public"); - final List additionalParameters = new ArrayList<>(); - - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:sqlserver://%s:%s;databaseName=%s;", + final String jdbcUrl = String.format("jdbc:sqlserver://%s:%s;databaseName=%s;", config.get("host").asText(), config.get("port").asText(), - config.get("database").asText())); - - if (config.has("ssl_method")) { - readSsl(config, additionalParameters); - } - - if (!additionalParameters.isEmpty()) { - jdbcUrl.append(String.join(";", additionalParameters)); - } + config.get("database").asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() - .put("jdbc_url", jdbcUrl.toString()) + .put("jdbc_url", jdbcUrl) .put("username", config.get("username").asText()) .put("password", config.get("password").asText()) .put("schema", schema); @@ -58,37 +76,16 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } - private void readSsl(final JsonNode config, final List additionalParameters) { - switch (config.get("ssl_method").asText()) { - case "unencrypted": - additionalParameters.add("encrypt=false"); - break; - case "encrypted_trust_server_certificate": - additionalParameters.add("encrypt=true"); - additionalParameters.add("trustServerCertificate=true"); - break; - case "encrypted_verify_certificate": - additionalParameters.add("encrypt=true"); - - // trust store location code found at https://stackoverflow.com/a/56570588 - final String trustStoreLocation = Optional.ofNullable(System.getProperty("javax.net.ssl.trustStore")) - .orElseGet(() -> System.getProperty("java.home") + "/lib/security/cacerts"); - final File trustStoreFile = new File(trustStoreLocation); - if (!trustStoreFile.exists()) { - throw new RuntimeException("Unable to locate the Java TrustStore: the system property javax.net.ssl.trustStore is undefined or " - + trustStoreLocation + " does not exist."); - } - final String trustStorePassword = System.getProperty("javax.net.ssl.trustStorePassword"); - - additionalParameters.add("trustStore=" + trustStoreLocation); - if (trustStorePassword != null && !trustStorePassword.isEmpty()) { - additionalParameters.add("trustStorePassword=" + config.get("trustStorePassword").asText()); - } - if (config.has("hostNameInCertificate")) { - additionalParameters.add("hostNameInCertificate=" + config.get("hostNameInCertificate").asText()); - } - break; + private String getTrustStoreLocation() { + // trust store location code found at https://stackoverflow.com/a/56570588 + final String trustStoreLocation = Optional.ofNullable(System.getProperty("javax.net.ssl.trustStore")) + .orElseGet(() -> System.getProperty("java.home") + "/lib/security/cacerts"); + final File trustStoreFile = new File(trustStoreLocation); + if (!trustStoreFile.exists()) { + throw new RuntimeException("Unable to locate the Java TrustStore: the system property javax.net.ssl.trustStore is undefined or " + + trustStoreLocation + " does not exist."); } + return trustStoreLocation; } public static Destination sshWrappedDestination() { diff --git a/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java b/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java new file mode 100644 index 0000000000000..88697030ffa89 --- /dev/null +++ b/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java @@ -0,0 +1,167 @@ +package io.airbyte.integrations.destination.mssql; + +import static java.lang.System.getProperty; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.map.MoreMaps; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; + +public class MSSQLDestinationTest { + + private Map existingProperties; + + private JsonNode createConfig(final String sslMethod) { + return createConfig(sslMethod, new HashMap<>()); + } + + private JsonNode createConfig(final String sslMethod, final Map additionalConfigs) { + return Jsons.jsonNode(MoreMaps.merge(baseParameters(sslMethod), additionalConfigs)); + } + + private Map baseParameters(final String sslMethod) { + return ImmutableMap.builder() + .put("ssl_method", sslMethod) + .put("host", "localhost") + .put("port", "1773") + .put("database", "db") + .put("username", "username") + .put("password", "verysecure") + .build(); + } + + @BeforeEach + public void setUp() { + existingProperties = new HashMap<>(); + } + + @AfterEach + public void tearDown() { + resetProperties(); + } + + @Test + public void testNoSsl() { + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = Jsons.jsonNode(ImmutableMap.of()); + final Map properties = destination.getDefaultConnectionProperties(config); + assertTrue(properties.isEmpty()); + } + + @Test + public void testUnencrypted() { + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = createConfig("unencrypted"); + final Map properties = destination.getDefaultConnectionProperties(config); + assertEquals(properties.get("encrypt"), "false"); + } + + @Test + public void testEncryptedTrustServerCertificate() { + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = createConfig("encrypted_trust_server_certificate"); + final Map properties = destination.getDefaultConnectionProperties(config); + assertEquals(properties.get("encrypt"), "true"); + assertEquals(properties.get("trustServerCertificate"), "true"); + } + + @Test + public void testEncryptedVerifyCertificate() { + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = createConfig("encrypted_verify_certificate"); + + final Map properties = destination.getDefaultConnectionProperties(config); + assertEquals(properties.get("encrypt"), "true"); + + final String trustStoreLocation = getProperty("java.home") + "/lib/security/cacerts"; + assertEquals(properties.get("trustStore"), trustStoreLocation); + assertNull(properties.get("trustStorePassword")); + assertNull(properties.get("hostNameInCertificate")); //TODO: add test with hostname in certificate + } + + @Test + public void testInvalidTrustStoreFile() { + setProperty("javax.net.ssl.trustStore", "/NOT_A_TRUST_STORE"); + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = createConfig("encrypted_verify_certificate"); + + assertThrows(RuntimeException.class, () -> + destination.getDefaultConnectionProperties(config) + ); + } + + @Test + public void testEncryptedVerifyCertificateWithEmptyTrustStorePassword() { + setProperty("javax.net.ssl.trustStorePassword", ""); + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = createConfig("encrypted_verify_certificate", ImmutableMap.of("trustStorePassword", "")); + + final Map properties = destination.getDefaultConnectionProperties(config); + assertEquals(properties.get("encrypt"), "true"); + + final String trustStoreLocation = getProperty("java.home") + "/lib/security/cacerts"; + assertEquals(properties.get("trustStore"), trustStoreLocation); + assertNull(properties.get("trustStorePassword")); + assertNull(properties.get("hostNameInCertificate")); + } + + @Test + public void testEncryptedVerifyCertificateWithNonEmptyTrustStorePassword() { + final String TRUST_STORE_PASSWORD = "TRUSTSTOREPASSWORD"; + setProperty("javax.net.ssl.trustStorePassword", TRUST_STORE_PASSWORD); + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = createConfig("encrypted_verify_certificate", ImmutableMap.of("trustStorePassword", TRUST_STORE_PASSWORD)); + + final Map properties = destination.getDefaultConnectionProperties(config); + assertEquals(properties.get("encrypt"), "true"); + + final String trustStoreLocation = getProperty("java.home") + "/lib/security/cacerts"; + assertEquals(properties.get("trustStore"), trustStoreLocation); + assertEquals(properties.get("trustStorePassword"), TRUST_STORE_PASSWORD); + assertNull(properties.get("hostNameInCertificate")); + } + + @Test + public void testEncryptedVerifyCertificateWithHostNameInCertificate() { + final MSSQLDestination destination = new MSSQLDestination(); + final String HOSTNAME_IN_CERTIFICATE = "HOSTNAME_IN_CERTIFICATE"; + final JsonNode config = createConfig("encrypted_verify_certificate", ImmutableMap.of("hostNameInCertificate", HOSTNAME_IN_CERTIFICATE)); + + final Map properties = destination.getDefaultConnectionProperties(config); + assertEquals(properties.get("encrypt"), "true"); + + final String trustStoreLocation = getProperty("java.home") + "/lib/security/cacerts"; + assertEquals(properties.get("trustStore"), trustStoreLocation); + assertNull(properties.get("trustStorePassword")); + + assertEquals(properties.get("hostNameInCertificate"), HOSTNAME_IN_CERTIFICATE); + } + + private void setProperty(final String key, final String value) { + existingProperties.put(key, System.getProperty(key)); + System.setProperty(key, value); + } + + private void resetProperties() { + existingProperties.forEach((k, v) -> resetProperty(k)); + } + + private void resetProperty(final String key) { + final String value = existingProperties.get(key); + if (value != null) { + System.setProperty(key, value); + } else { + System.clearProperty(key); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile index 530eb36c0b61b..ad65ba60ef064 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mysql/Dockerfile b/airbyte-integrations/connectors/destination-mysql/Dockerfile index b137bdd41e32b..bc324b2bff11f 100644 --- a/airbyte-integrations/connectors/destination-mysql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.17 +LABEL io.airbyte.version=0.1.18 LABEL io.airbyte.name=airbyte/destination-mysql diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java index ad2b0505172e9..7d4720e90c379 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java @@ -6,10 +6,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Streams; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; @@ -18,13 +16,8 @@ import io.airbyte.integrations.destination.mysql.MySQLSqlOperations.VersionCompatibility; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; -import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,12 +36,20 @@ public class MySQLDestination extends AbstractJdbcDestination implements Destina public static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; - static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( - "useSSL", "true", - "requireSSL", "true", - "verifyServerCertificate", "false"); static final Map DEFAULT_JDBC_PARAMETERS = ImmutableMap.of( - "zeroDateTimeBehavior", "convertToNull"); + // zero dates by default cannot be parsed into java date objects (they will throw an error) + // in addition, users don't always have agency in fixing them e.g: maybe they don't own the database + // and can't + // remove zero date values. + // since zero dates are placeholders, we convert them to null by default + "zeroDateTimeBehavior", "convertToNull", + "allowLoadLocalInfile", "true"); + + static final Map DEFAULT_SSL_JDBC_PARAMETERS = MoreMaps.merge(ImmutableMap.of( + "useSSL", "true", + "requireSSL", "true", + "verifyServerCertificate", "false"), + DEFAULT_JDBC_PARAMETERS); public static Destination sshWrappedDestination() { return new SshWrappedDestination(new MySQLDestination(), List.of(HOST_KEY), List.of(PORT_KEY)); @@ -86,101 +87,37 @@ public MySQLDestination() { } @Override - protected JdbcDatabase getDatabase(final JsonNode config) { - final JsonNode jdbcConfig = toJdbcConfig(config); - - return Databases.createJdbcDatabase( - jdbcConfig.get(USERNAME_KEY).asText(), - jdbcConfig.has(PASSWORD_KEY) ? jdbcConfig.get(PASSWORD_KEY).asText() : null, - jdbcConfig.get(JDBC_URL_KEY).asText(), - getDriverClass(), - "allowLoadLocalInfile=true"); + protected Map getDefaultConnectionProperties(final JsonNode config) { + if (useSSL(config)) { + return DEFAULT_SSL_JDBC_PARAMETERS; + } else { + return DEFAULT_JDBC_PARAMETERS; + } + } + + private boolean useSSL(final JsonNode config) { + return !config.has(SSL_KEY) || config.get(SSL_KEY).asBoolean(); } @Override public JsonNode toJdbcConfig(final JsonNode config) { - final List additionalParameters = getAdditionalParameters(config); - - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:mysql://%s:%s/%s", + final String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s", config.get(HOST_KEY).asText(), config.get(PORT_KEY).asText(), - config.get(DATABASE_KEY).asText())); - // zero dates by default cannot be parsed into java date objects (they will throw an error) - // in addition, users don't always have agency in fixing them e.g: maybe they don't own the database - // and can't - // remove zero date values. - // since zero dates are placeholders, we convert them to null by default - if (!additionalParameters.isEmpty()) { - jdbcUrl.append("?"); - jdbcUrl.append(String.join("&", additionalParameters)); - } + config.get(DATABASE_KEY).asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put(USERNAME_KEY, config.get(USERNAME_KEY).asText()) - .put(JDBC_URL_KEY, jdbcUrl.toString()); + .put(JDBC_URL_KEY, jdbcUrl); if (config.has(PASSWORD_KEY)) { configBuilder.put(PASSWORD_KEY, config.get(PASSWORD_KEY).asText()); } - - return Jsons.jsonNode(configBuilder.build()); - } - - private List getAdditionalParameters(final JsonNode config) { - final Map customParameters = getCustomJdbcParameters(config); - - if (useSSL(config)) { - return convertToJdbcStrings(customParameters, MoreMaps.merge(DEFAULT_JDBC_PARAMETERS, SSL_JDBC_PARAMETERS)); - } else { - return convertToJdbcStrings(customParameters, DEFAULT_JDBC_PARAMETERS); - } - } - - private List convertToJdbcStrings(final Map customParameters, final Map defaultParametersMap) { - assertCustomParametersDontOverwriteDefaultParameters(customParameters, defaultParametersMap); - return Streams.concat(Stream.of(customParameters, defaultParametersMap)) - .map(Map::entrySet) - .flatMap(Collection::stream) - .map(entry -> formatParameter(entry.getKey(), entry.getValue())) - .collect(Collectors.toList()); - } - - private void assertCustomParametersDontOverwriteDefaultParameters(final Map customParameters, - final Map defaultParameters) { - for (final String key : defaultParameters.keySet()) { - if (customParameters.containsKey(key) && !Objects.equals(customParameters.get(key), defaultParameters.get(key))) { - throw new IllegalArgumentException("Cannot overwrite default JDBC parameter " + key); - } - } - } - - private Map getCustomJdbcParameters(final JsonNode config) { - final Map parameters = new HashMap<>(); if (config.has(JDBC_URL_PARAMS_KEY)) { - final String jdbcParams = config.get(JDBC_URL_PARAMS_KEY).asText(); - if (!jdbcParams.isBlank()) { - final String[] keyValuePairs = jdbcParams.split("&"); - for (final String kv : keyValuePairs) { - final String[] split = kv.split("="); - if (split.length == 2) { - parameters.put(split[0], split[1]); - } else { - throw new IllegalArgumentException( - "jdbc_url_params must be formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). Got " - + jdbcParams); - } - } - } + configBuilder.put(JDBC_URL_PARAMS_KEY, config.get(JDBC_URL_PARAMS_KEY)); } - return parameters; - } - - private boolean useSSL(final JsonNode config) { - return !config.has(SSL_KEY) || config.get(SSL_KEY).asBoolean(); - } - static String formatParameter(final String key, final String value) { - return String.format("%s=%s", key, value); + return Jsons.jsonNode(configBuilder.build()); } public static void main(final String[] args) throws Exception { diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java index a9f491059c558..6016ca02c3be0 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java @@ -189,8 +189,7 @@ protected void tearDown(final TestDestinationEnv testEnv) { public void testCustomDbtTransformations() throws Exception { // We need to create view for testing custom dbt transformations executeQuery("GRANT CREATE VIEW ON *.* TO " + db.getUsername() + "@'%';"); - // overrides test with a no-op until https://github.com/dbt-labs/jaffle_shop/pull/8 is merged - // super.testCustomDbtTransformations(); + super.testCustomDbtTransformations(); } @Test diff --git a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java index b86977f55c680..6a98e8f3fa66e 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java @@ -4,130 +4,74 @@ package io.airbyte.integrations.destination.mysql; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.spy; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.map.MoreMaps; import java.util.Map; -import java.util.Map.Entry; import org.junit.jupiter.api.Test; public class MySQLDestinationTest { - private MySQLDestination getDestination() { - final MySQLDestination result = spy(MySQLDestination.class); - return result; - } + public static final String JDBC_URL = "jdbc:mysql://localhost:1337/db"; private JsonNode buildConfigNoJdbcParameters() { - final JsonNode config = Jsons.jsonNode(ImmutableMap.of( + return Jsons.jsonNode(ImmutableMap.of( "host", "localhost", "port", 1337, "username", "user", "database", "db")); - return config; } private JsonNode buildConfigWithExtraJdbcParameters(final String extraParam) { - final JsonNode config = Jsons.jsonNode(ImmutableMap.of( - "host", "localhost", - "port", 1337, - "username", "user", - "database", "db", - "jdbc_url_params", extraParam)); - return config; - } - - private JsonNode buildConfigWithExtraJdbcParametersWithNoSsl(final String extraParam) { - final JsonNode config = Jsons.jsonNode(ImmutableMap.of( + return Jsons.jsonNode(ImmutableMap.of( "host", "localhost", "port", 1337, "username", "user", "database", "db", - "ssl", false, "jdbc_url_params", extraParam)); - return config; } private JsonNode buildConfigNoExtraJdbcParametersWithoutSsl() { - final JsonNode config = Jsons.jsonNode(ImmutableMap.of( + return Jsons.jsonNode(ImmutableMap.of( "host", "localhost", "port", 1337, "username", "user", "database", "db", "ssl", false)); - return config; } @Test void testNoExtraParams() { - final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigNoJdbcParameters()); - final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals("jdbc:mysql://localhost:1337/db?verifyServerCertificate=false&zeroDateTimeBehavior=convertToNull&requireSSL=true&useSSL=true", url); + final JsonNode config = buildConfigNoJdbcParameters(); + final JsonNode jdbcConfig = new MySQLDestination().toJdbcConfig(config); + assertEquals(JDBC_URL, jdbcConfig.get("jdbc_url").asText()); } @Test void testEmptyExtraParams() { - final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters("")); - final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals("jdbc:mysql://localhost:1337/db?verifyServerCertificate=false&zeroDateTimeBehavior=convertToNull&requireSSL=true&useSSL=true", url); + final JsonNode jdbcConfig = new MySQLDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters("")); + assertEquals(JDBC_URL, jdbcConfig.get("jdbc_url").asText()); } @Test void testExtraParams() { final String extraParam = "key1=value1&key2=value2&key3=value3"; - final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam)); - final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals( - "jdbc:mysql://localhost:1337/db?key1=value1&key2=value2&key3=value3&verifyServerCertificate=false&zeroDateTimeBehavior=convertToNull&requireSSL=true&useSSL=true", - url); - } - - @Test - void testExtraParamsWithDefaultParameter() { - final Map allDefaultParameters = MoreMaps.merge(MySQLDestination.SSL_JDBC_PARAMETERS, - MySQLDestination.DEFAULT_JDBC_PARAMETERS); - for (final Entry entry : allDefaultParameters.entrySet()) { - final String identicalParameter = MySQLDestination.formatParameter(entry.getKey(), entry.getValue()); - final String overridingParameter = MySQLDestination.formatParameter(entry.getKey(), "DIFFERENT_VALUE"); - - // Do not throw an exception if the values are equal - assertDoesNotThrow(() -> getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(identicalParameter)).get("jdbc_url").asText()); - // Throw an exception if the values are different - assertThrows(IllegalArgumentException.class, () -> getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(overridingParameter))); - } - } - - @Test - void testExtraParameterNoSsl() { - final String extraParam = "key1=value1&key2=value2&key3=value3"; - final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigWithExtraJdbcParametersWithNoSsl(extraParam)); - final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals( - "jdbc:mysql://localhost:1337/db?key1=value1&key2=value2&key3=value3&zeroDateTimeBehavior=convertToNull", - url); + final JsonNode jdbcConfig = new MySQLDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam)); + assertEquals(JDBC_URL, jdbcConfig.get("jdbc_url").asText()); } @Test - void testNoExtraParameterNoSsl() { - final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigNoExtraJdbcParametersWithoutSsl()); - final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals( - "jdbc:mysql://localhost:1337/db?zeroDateTimeBehavior=convertToNull", - url); + void testDefaultParamsNoSSL() { + final Map defaultProperties = new MySQLDestination().getDefaultConnectionProperties(buildConfigNoExtraJdbcParametersWithoutSsl()); + assertEquals(MySQLDestination.DEFAULT_JDBC_PARAMETERS, defaultProperties); } @Test - void testInvalidExtraParam() { - final String extraParam = "key1=value1&sdf&"; - assertThrows(IllegalArgumentException.class, () -> { - getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam)); - }); + void testDefaultParamsWithSSL() { + final Map defaultProperties = new MySQLDestination().getDefaultConnectionProperties(buildConfigNoJdbcParameters()); + assertEquals(MySQLDestination.DEFAULT_SSL_JDBC_PARAMETERS, defaultProperties); } } diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile index bfbd7c875274e..f7ed56838c5fd 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-oracle-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java index 05b2a6d9eb435..04ab33055b98d 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java @@ -15,6 +15,7 @@ import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.destination.oracle.OracleDestination; import io.airbyte.integrations.destination.oracle.OracleNameTransformer; @@ -77,7 +78,7 @@ protected boolean implementsNamespaces() { @Override protected boolean supportsDBT() { - return true; + return false; } @Override @@ -113,8 +114,8 @@ private List retrieveRecordsFromTable(final String tableName, final St throws SQLException { final List result = getDatabase(config) .query(ctx -> ctx.fetch( - String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, - OracleDestination.COLUMN_NAME_EMITTED_AT)) + String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, + OracleDestination.COLUMN_NAME_EMITTED_AT)) .stream() .collect(Collectors.toList())); return result @@ -176,9 +177,9 @@ public void testEncryption() throws SQLException { config.get("port").asText(), config.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", - "oracle.net.encryption_client=REQUIRED;" + + JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + "oracle.net.encryption_types_client=( " - + algorithm + " )"); + + algorithm + " )")); final String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -202,9 +203,9 @@ public void testCheckProtocol() throws SQLException { clone.get("port").asText(), clone.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", - "oracle.net.encryption_client=REQUIRED;" + + JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + "oracle.net.encryption_types_client=( " - + algorithm + " )"); + + algorithm + " )")); final String network_service_banner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.query(network_service_banner).collect(Collectors.toList()); diff --git a/airbyte-integrations/connectors/destination-oracle/Dockerfile b/airbyte-integrations/connectors/destination-oracle/Dockerfile index 408b559fa4fa2..4bf980d8c5c64 100644 --- a/airbyte-integrations/connectors/destination-oracle/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-oracle COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.version=0.1.15 LABEL io.airbyte.name=airbyte/destination-oracle diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index 6b7ea9eeb4e1a..795882a30ad99 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -14,8 +14,9 @@ import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import java.io.IOException; import java.io.PrintWriter; -import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; @@ -36,8 +37,10 @@ public class OracleDestination extends AbstractJdbcDestination implements Destin public static final String COLUMN_NAME_EMITTED_AT = "\"" + JavaBaseConstants.COLUMN_NAME_EMITTED_AT.toUpperCase() + "\""; - private static final String KEY_STORE_FILE_PATH = "clientkeystore.jks"; + protected static final String KEY_STORE_FILE_PATH = "clientkeystore.jks"; private static final String KEY_STORE_PASS = RandomStringUtils.randomAlphanumeric(8); + public static final String ENCRYPTION_KEY = "encryption"; + public static final String ENCRYPTION_METHOD_KEY = "encryption_method"; enum Protocol { TCP, @@ -54,12 +57,36 @@ public static Destination sshWrappedDestination() { } @Override - public JsonNode toJdbcConfig(final JsonNode config) { - final List additionalParameters = new ArrayList<>(); + protected Map getDefaultConnectionProperties(final JsonNode config) { + final HashMap properties = new HashMap<>(); + if (config.has(ENCRYPTION_KEY)) { + final JsonNode encryption = config.get(ENCRYPTION_KEY); + final String encryptionMethod = encryption.get(ENCRYPTION_METHOD_KEY).asText(); + switch (encryptionMethod) { + case "unencrypted" -> { + + } + case "client_nne" -> { + final String algorithm = encryption.get("encryption_algorithm").asText(); + properties.put("oracle.net.encryption_client", "REQUIRED"); + properties.put("oracle.net.encryption_types_client", "( " + algorithm + " )"); + } + case "encrypted_verify_certificate" -> { + tryConvertAndImportCertificate(encryption.get("ssl_certificate").asText()); + properties.put("javax.net.ssl.trustStore", KEY_STORE_FILE_PATH); + properties.put("javax.net.ssl.trustStoreType", "JKS"); + properties.put("javax.net.ssl.trustStorePassword", KEY_STORE_PASS); + } + default -> throw new RuntimeException("Failed to obtain connection protocol from config " + encryption.asText()); + } - final Protocol protocol = config.has("encryption") - ? obtainConnectionProtocol(config.get("encryption"), additionalParameters) - : Protocol.TCP; + } + return properties; + } + + @Override + public JsonNode toJdbcConfig(final JsonNode config) { + final Protocol protocol = obtainConnectionProtocol(config); final String connectionString = String.format( "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=%s)(HOST=%s)(PORT=%s))(CONNECT_DATA=(SID=%s)))", protocol, @@ -75,36 +102,20 @@ public JsonNode toJdbcConfig(final JsonNode config) { configBuilder.put("password", config.get("password").asText()); } - if (!additionalParameters.isEmpty()) { - final String connectionParams = String.join(";", additionalParameters); - configBuilder.put("connection_properties", connectionParams); - } - return Jsons.jsonNode(configBuilder.build()); } - private Protocol obtainConnectionProtocol(final JsonNode encryption, - final List additionalParameters) { - final String encryptionMethod = encryption.get("encryption_method").asText(); + protected Protocol obtainConnectionProtocol(final JsonNode config) { + if (!config.has(ENCRYPTION_KEY)) { + return Protocol.TCP; + } + final JsonNode encryption = config.get(ENCRYPTION_KEY); + final String encryptionMethod = encryption.get(ENCRYPTION_METHOD_KEY).asText(); switch (encryptionMethod) { - case "unencrypted" -> { - return Protocol.TCP; - } - case "client_nne" -> { - final String algorithm = encryption.get("encryption_algorithm").asText(); - additionalParameters.add("oracle.net.encryption_client=REQUIRED"); - additionalParameters.add("oracle.net.encryption_types_client=( " + algorithm + " )"); + case "unencrypted", "client_nne" -> { return Protocol.TCP; } case "encrypted_verify_certificate" -> { - try { - convertAndImportCertificate(encryption.get("ssl_certificate").asText()); - } catch (final IOException | InterruptedException e) { - throw new RuntimeException("Failed to import certificate into Java Keystore"); - } - additionalParameters.add("javax.net.ssl.trustStore=" + KEY_STORE_FILE_PATH); - additionalParameters.add("javax.net.ssl.trustStoreType=JKS"); - additionalParameters.add("javax.net.ssl.trustStorePassword=" + KEY_STORE_PASS); return Protocol.TCPS; } } @@ -112,6 +123,14 @@ private Protocol obtainConnectionProtocol(final JsonNode encryption, "Failed to obtain connection protocol from config " + encryption.asText()); } + private static void tryConvertAndImportCertificate(final String certificate) { + try { + convertAndImportCertificate(certificate); + } catch (final IOException | InterruptedException e) { + throw new RuntimeException("Failed to import certificate into Java Keystore"); + } + } + private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json index 3acaa8b981ae1..9422b0958de98 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json @@ -2,8 +2,11 @@ "documentationUrl": "https://docs.airbyte.io/integrations/destinations/oracle", "supportsIncremental": true, "supportsNormalization": false, - "supportsDBT": true, - "supported_destination_sync_modes": ["overwrite", "append"], + "supportsDBT": false, + "supported_destination_sync_modes": [ + "overwrite", + "append" + ], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Oracle Destination Spec", diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java index 278b955220de4..6554e982a506d 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java @@ -15,6 +15,7 @@ import io.airbyte.db.jdbc.JdbcDatabase; import java.sql.SQLException; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.junit.Test; @@ -37,18 +38,22 @@ public void testEncryption() throws SQLException { config.get("port").asText(), config.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", - "oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " - + algorithm + " )"); + getAdditionalProperties(algorithm)); final String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; - final List collect = database.query(network_service_banner).collect(Collectors.toList()); + final List collect = database.query(network_service_banner).toList(); assertThat(collect.get(2).get("NETWORK_SERVICE_BANNER").asText(), equals("Oracle Advanced Security: " + algorithm + " encryption")); } + private Map getAdditionalProperties(final String algorithm) { + return ImmutableMap. + of("oracle.net.encryption_client", "REQUIRED", + "oracle.net.encryption_types_client", String.format("( %s )", algorithm)); + } + @Test public void testCheckProtocol() throws SQLException { final JsonNode clone = Jsons.clone(getConfig()); @@ -67,9 +72,7 @@ public void testCheckProtocol() throws SQLException { clone.get("port").asText(), clone.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", - "oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " - + algorithm + " )"); + getAdditionalProperties(algorithm)); final String network_service_banner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.query(network_service_banner).collect(Collectors.toList()); diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java index 1646e2f1dc0e8..e2e9054a7f4be 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java @@ -175,7 +175,7 @@ protected void tearDown(final TestDestinationEnv testEnv) throws Exception { @Override protected boolean supportsDBT() { - return true; + return false; } @Override diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java index 8e57e31ef7ffc..3201356782b27 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java @@ -75,7 +75,7 @@ protected boolean implementsNamespaces() { @Override protected boolean supportsDBT() { - return true; + return false; } @Override @@ -111,8 +111,8 @@ private List retrieveRecordsFromTable(final String tableName, final St throws SQLException { final List result = getDatabase(config) .query(ctx -> ctx.fetch( - String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, - OracleDestination.COLUMN_NAME_EMITTED_AT)) + String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, + OracleDestination.COLUMN_NAME_EMITTED_AT)) .stream() .collect(Collectors.toList())); return result diff --git a/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java b/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java new file mode 100644 index 0000000000000..be6c72a0e98fd --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java @@ -0,0 +1,110 @@ +package io.airbyte.integrations.destination.oracle; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.map.MoreMaps; +import io.airbyte.integrations.destination.oracle.OracleDestination.Protocol; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; + +public class OracleDestinationTest { + + private OracleDestination destination; + + private JsonNode createConfig() { + return createConfig(new HashMap<>()); + } + + private JsonNode createConfig(final Map additionalConfigs) { + return Jsons.jsonNode(MoreMaps.merge(baseParameters(), additionalConfigs)); + } + + private Map baseParameters() { + return ImmutableMap.builder() + .put("host", "localhost") + .put("port", "1773") + .put("database", "db") + .put("username", "username") + .put("password", "verysecure") + .build(); + } + + @BeforeEach + void setUp() { + destination = new OracleDestination(); + } + + @Test + void testNoEncryption() { + final Map properties = destination.getDefaultConnectionProperties(createConfig()); + assertNull(properties.get(OracleDestination.ENCRYPTION_KEY)); + assertNull(properties.get("javax.net.ssl.trustStorePassword")); + + final Protocol protocol = destination.obtainConnectionProtocol(createConfig()); + assertEquals(Protocol.TCP, protocol); + } + + @Test + void testUnencrypted() { + final Map encryptionNode = ImmutableMap.of(OracleDestination.ENCRYPTION_METHOD_KEY, "unencrypted"); + final JsonNode inputConfig = createConfig(ImmutableMap.of(OracleDestination.ENCRYPTION_KEY, encryptionNode)); + final Map properties = destination.getDefaultConnectionProperties(inputConfig); + assertNull(properties.get(OracleDestination.ENCRYPTION_KEY)); + assertNull(properties.get("javax.net.ssl.trustStorePassword")); + + final Protocol protocol = destination.obtainConnectionProtocol(inputConfig); + assertEquals(Protocol.TCP, protocol); + } + + @Test + void testClientNne() { + final String algorithm = "AES256"; + final Map encryptionNode = ImmutableMap.of( + OracleDestination.ENCRYPTION_METHOD_KEY, "client_nne", + "encryption_algorithm", algorithm); + final JsonNode inputConfig = createConfig(ImmutableMap.of(OracleDestination.ENCRYPTION_KEY, encryptionNode)); + final Map properties = destination.getDefaultConnectionProperties(inputConfig); + assertEquals(properties.get("oracle.net.encryption_client"), "REQUIRED"); + assertEquals(properties.get("oracle.net.encryption_types_client"), String.format("( %s )", algorithm)); + assertNull(properties.get("javax.net.ssl.trustStorePassword")); + + final Protocol protocol = destination.obtainConnectionProtocol(inputConfig); + assertEquals(Protocol.TCP, protocol); + } + + @Test + void testEncryptedVerifyCertificate() { + final Map encryptionNode = ImmutableMap.of( + OracleDestination.ENCRYPTION_METHOD_KEY, "encrypted_verify_certificate", "ssl_certificate", "certificate"); + final JsonNode inputConfig = createConfig(ImmutableMap.of(OracleDestination.ENCRYPTION_KEY, encryptionNode)); + final Map properties = destination.getDefaultConnectionProperties(inputConfig); + assertEquals(properties.get("javax.net.ssl.trustStore"), OracleDestination.KEY_STORE_FILE_PATH); + assertEquals(properties.get("javax.net.ssl.trustStoreType"), "JKS"); + assertNotNull(properties.get("javax.net.ssl.trustStorePassword")); + + final Protocol protocol = destination.obtainConnectionProtocol(inputConfig); + assertEquals(Protocol.TCPS, protocol); + } + + @Test + void testInvalidEncryptionMethod() { + final Map encryptionNode = ImmutableMap.of( + OracleDestination.ENCRYPTION_METHOD_KEY, "invalid_encryption_method"); + final JsonNode inputConfig = createConfig(ImmutableMap.of(OracleDestination.ENCRYPTION_KEY, encryptionNode)); + assertThrows(RuntimeException.class, () -> + destination.getDefaultConnectionProperties(inputConfig) + ); + assertThrows(RuntimeException.class, () -> + destination.obtainConnectionProtocol(inputConfig) + ); + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile index d7a514ace0319..1c32dea0e209b 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-postgres/Dockerfile b/airbyte-integrations/connectors/destination-postgres/Dockerfile index a58a4cfb3e532..4bc95fceba408 100644 --- a/airbyte-integrations/connectors/destination-postgres/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.14 +LABEL io.airbyte.version=0.3.15 LABEL io.airbyte.name=airbyte/destination-postgres diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java index 9803a677649f0..73e4c81be33a9 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java @@ -11,8 +11,9 @@ import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.ssh.SshWrappedDestination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,6 +26,10 @@ public class PostgresDestination extends AbstractJdbcDestination implements Dest public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); + static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( + "ssl", "true", + "sslmode", "require"); + public static Destination sshWrappedDestination() { return new SshWrappedDestination(new PostgresDestination(), HOST_KEY, PORT_KEY); } @@ -33,29 +38,28 @@ public PostgresDestination() { super(DRIVER_CLASS, new PostgresSQLNameTransformer(), new PostgresSqlOperations()); } + @Override + protected Map getDefaultConnectionProperties(final JsonNode config) { + if (useSsl(config)) { + return SSL_JDBC_PARAMETERS; + } else { + // No need for any parameters if the connection doesn't use SSL + return Collections.emptyMap(); + } + } + @Override public JsonNode toJdbcConfig(final JsonNode config) { final String schema = Optional.ofNullable(config.get("schema")).map(JsonNode::asText).orElse("public"); - final List additionalParameters = new ArrayList<>(); - - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:postgresql://%s:%s/%s?", + final String jdbcUrl = String.format("jdbc:postgresql://%s:%s/%s?", config.get("host").asText(), config.get("port").asText(), - config.get("database").asText())); - - if (!config.has("ssl") || config.get("ssl").asBoolean()) { - additionalParameters.add("ssl=true"); - additionalParameters.add("sslmode=require"); - } - - if (!additionalParameters.isEmpty()) { - additionalParameters.forEach(x -> jdbcUrl.append(x).append("&")); - } + config.get("database").asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) - .put("jdbc_url", jdbcUrl.toString()) + .put("jdbc_url", jdbcUrl) .put("schema", schema); if (config.has("password")) { @@ -64,6 +68,10 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } + private boolean useSsl(final JsonNode config) { + return !config.has("ssl") || config.get("ssl").asBoolean(); + } + public static void main(final String[] args) throws Exception { final Destination destination = PostgresDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", PostgresDestination.class); diff --git a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java index b2d236304efa2..f66387241dca3 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.map.MoreMaps; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.AirbyteMessageConsumer; @@ -23,7 +24,9 @@ import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.time.Instant; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.junit.jupiter.api.AfterAll; @@ -47,6 +50,17 @@ public class PostgresDestinationTest { private JsonNode config; + private static final Map CONFIG_WITH_SSL = ImmutableMap.of( + "host", "localhost", + "port", "1337", + "username", "user", + "database", "db"); + + private static final Map CONFIG_NO_SSL = MoreMaps.merge( + CONFIG_WITH_SSL, + ImmutableMap.of( + "ssl", "false")); + @BeforeAll static void init() { PSQL_DB = new PostgreSQLContainer<>("postgres:13-alpine"); @@ -63,6 +77,20 @@ static void cleanUp() { PSQL_DB.close(); } + @Test + void testDefaultParamsNoSSL() { + final Map defaultProperties = new PostgresDestination().getDefaultConnectionProperties( + Jsons.jsonNode(CONFIG_NO_SSL)); + assertEquals(new HashMap<>(), defaultProperties); + } + + @Test + void testDefaultParamsWithSSL() { + final Map defaultProperties = new PostgresDestination().getDefaultConnectionProperties( + Jsons.jsonNode(CONFIG_WITH_SSL)); + assertEquals(PostgresDestination.SSL_JDBC_PARAMETERS, defaultProperties); + } + // This test is a bit redundant with PostgresIntegrationTest. It makes it easy to run the // destination in the same process as the test allowing us to put breakpoint in, which is handy for // debugging (especially since we use postgres as a guinea pig for most features). diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index d909bb1d72e68..d6d8d7ad36597 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.26 +LABEL io.airbyte.version=0.3.27 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java index 8311463a35e73..576a9246bb962 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java @@ -11,8 +11,7 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import java.util.ArrayList; -import java.util.List; +import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,6 +22,10 @@ public class RedshiftInsertDestination extends AbstractJdbcDestination implement public static final String DRIVER_CLASS = "com.amazon.redshift.jdbc.Driver"; + public static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( + "ssl", "true", + "sslfactory", "com.amazon.redshift.ssl.NonValidatingFactory"); + public RedshiftInsertDestination() { super(DRIVER_CLASS, new RedshiftSQLNameTransformer(), new RedshiftSqlOperations()); } @@ -37,21 +40,19 @@ public JdbcDatabase getDatabase(final JsonNode config) { return getJdbcDatabase(config); } - private static void addSsl(final List additionalProperties) { - additionalProperties.add("ssl=true"); - additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); + @Override + protected Map getDefaultConnectionProperties(final JsonNode config) { + return SSL_JDBC_PARAMETERS; } public static JdbcDatabase getJdbcDatabase(final JsonNode config) { - final List additionalProperties = new ArrayList<>(); final var jdbcConfig = RedshiftInsertDestination.getJdbcConfig(config); - addSsl(additionalProperties); return Databases.createJdbcDatabase( jdbcConfig.get("username").asText(), jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, jdbcConfig.get("jdbc_url").asText(), RedshiftInsertDestination.DRIVER_CLASS, - String.join(";", additionalProperties)); + SSL_JDBC_PARAMETERS); } public static JsonNode getJdbcConfig(final JsonNode redshiftConfig) { diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java index a6fdd5f877000..413ef0c261ee3 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java @@ -21,8 +21,8 @@ import java.util.stream.Collectors; /** - * Integration test testing {@link RedshiftCopyS3Destination}. The default Redshift integration test - * credentials contain S3 credentials - this automatically causes COPY to be selected. + * Integration test testing {@link RedshiftCopyS3Destination}. The default Redshift integration test credentials contain S3 credentials - this + * automatically causes COPY to be selected. */ public class RedshiftCopyDestinationAcceptanceTest extends DestinationAcceptanceTest { @@ -141,7 +141,8 @@ protected Database getDatabase() { baseConfig.get("port").asText(), baseConfig.get("database").asText()), "com.amazon.redshift.jdbc.Driver", null, - "ssl=true;sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); + RedshiftInsertDestination.SSL_JDBC_PARAMETERS + ); } @Override diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index 5912f54ed3ba0..6473177947fb5 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -18,8 +18,8 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -ENV APPLICATION_VERSION 0.4.16 +ENV APPLICATION_VERSION 0.4.17 ENV ENABLE_SENTRY true -LABEL io.airbyte.version=0.4.16 +LABEL io.airbyte.version=0.4.17 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java index 58758eeb3781d..3a346fb6909bd 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java @@ -9,6 +9,8 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; +import java.util.Collections; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,6 +28,11 @@ protected JdbcDatabase getDatabase(final JsonNode config) { return SnowflakeDatabase.getDatabase(config); } + @Override + protected Map getDefaultConnectionProperties(final JsonNode config) { + return Collections.emptyMap(); + } + // this is a no op since we override getDatabase. @Override public JsonNode toJdbcConfig(final JsonNode config) { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index 5a9a2db88b17e..33897c854f362 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -14,6 +14,8 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.Collections; +import java.util.Map; import java.util.UUID; import java.util.function.Consumer; import org.slf4j.Logger; @@ -54,7 +56,7 @@ private static void attemptSQLCreateAndDropStages(final String outputSchema, // verify we have permissions to create/drop stage final String outputTableName = namingResolver.getIdentifier("_airbyte_connection_test_" + UUID.randomUUID().toString().replaceAll("-", "")); - final String stageName = namingResolver.getStageName(outputSchema, outputTableName);; + final String stageName = namingResolver.getStageName(outputSchema, outputTableName); sqlOperations.createStageIfNotExists(database, stageName); sqlOperations.dropStageIfExists(database, stageName); } @@ -64,6 +66,11 @@ protected JdbcDatabase getDatabase(final JsonNode config) { return SnowflakeDatabase.getDatabase(config); } + @Override + protected Map getDefaultConnectionProperties(final JsonNode config) { + return Collections.emptyMap(); + } + // this is a no op since we override getDatabase. @Override public JsonNode toJdbcConfig(final JsonNode config) { diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index 60d11c7184499..9c38b060785b0 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -30,6 +30,7 @@ import io.airbyte.db.SqlDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcStreamingQueryConfiguration; +import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.dto.JdbcPrivilegeDto; import io.airbyte.integrations.source.relationaldb.AbstractRelationalDbSource; @@ -55,9 +56,8 @@ import org.slf4j.LoggerFactory; /** - * This class contains helper functions and boilerplate for implementing a source connector for a - * relational DB source which can be accessed via JDBC driver. If you are implementing a connector - * for a relational DB which has a JDBC driver, make an effort to use this class. + * This class contains helper functions and boilerplate for implementing a source connector for a relational DB source which can be accessed via JDBC + * driver. If you are implementing a connector for a relational DB which has a JDBC driver, make an effort to use this class. */ public abstract class AbstractJdbcSource extends AbstractRelationalDbSource implements Source { @@ -114,10 +114,10 @@ protected List>> discoverInternal(final JdbcData final Set internalSchemas = new HashSet<>(getExcludedInternalNameSpaces()); final Set tablesWithSelectGrantPrivilege = getPrivilegesTableForCurrentUser(database, schema); return database.bufferedResultSetQuery( - // retrieve column metadata from the database - conn -> conn.getMetaData().getColumns(getCatalog(database), schema, null, null), - // store essential column metadata to a Json object from the result set about each column - this::getColumnMetadata) + // retrieve column metadata from the database + conn -> conn.getMetaData().getColumns(getCatalog(database), schema, null, null), + // store essential column metadata to a Json object from the result set about each column + this::getColumnMetadata) .stream() .filter(excludeNotAccessibleTables(internalSchemas, tablesWithSelectGrantPrivilege)) // group by schema and table name to handle the case where a table with the same name exists in @@ -139,7 +139,8 @@ protected List>> discoverInternal(final JdbcData f.get(INTERNAL_COLUMN_TYPE_NAME).asText(), f.get(INTERNAL_COLUMN_SIZE).asInt(), jsonType); - return new CommonField(f.get(INTERNAL_COLUMN_NAME).asText(), datatype) {}; + return new CommonField(f.get(INTERNAL_COLUMN_NAME).asText(), datatype) { + }; }) .collect(Collectors.toList())) .build()) @@ -155,14 +156,14 @@ protected Predicate excludeNotAccessibleTables(final Set inter return tablesWithSelectGrantPrivilege.stream() .anyMatch(e -> e.getSchemaName().equals(jsonNode.get(INTERNAL_SCHEMA_NAME).asText())) && tablesWithSelectGrantPrivilege.stream() - .anyMatch(e -> e.getTableName().equals(jsonNode.get(INTERNAL_TABLE_NAME).asText())) + .anyMatch(e -> e.getTableName().equals(jsonNode.get(INTERNAL_TABLE_NAME).asText())) && !internalSchemas.contains(jsonNode.get(INTERNAL_SCHEMA_NAME).asText()); }; } // needs to override isNotInternalSchema for connectors that override // getPrivilegesTableForCurrentUser() - protected boolean isNotInternalSchema(JsonNode jsonNode, Set internalSchemas) { + protected boolean isNotInternalSchema(final JsonNode jsonNode, final Set internalSchemas) { return !internalSchemas.contains(jsonNode.get(INTERNAL_SCHEMA_NAME).asText()); } @@ -185,8 +186,7 @@ private JsonNode getColumnMetadata(final ResultSet resultSet) throws SQLExceptio } /** - * @param field Essential column information returned from - * {@link AbstractJdbcSource#getColumnMetadata}. + * @param field Essential column information returned from {@link AbstractJdbcSource#getColumnMetadata}. */ public Datatype getFieldType(final JsonNode field) { return sourceOperations.getFieldType(field); @@ -293,7 +293,7 @@ public JdbcDatabase createDatabase(final JsonNode config) throws SQLException { jdbcConfig.get("jdbc_url").asText(), driverClass, jdbcStreamingQueryConfiguration, - jdbcConfig.has("connection_properties") ? jdbcConfig.get("connection_properties").asText() : null, + JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties"), sourceOperations); quoteString = (quoteString == null ? database.getMetaData().getIdentifierQuoteString() : quoteString); diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java index 02ab8faef80bc..8511ad9d05206 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -24,6 +24,7 @@ import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcSourceOperations; +import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.relationaldb.models.DbState; @@ -196,7 +197,7 @@ public void setup() throws Exception { jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, jdbcConfig.get("jdbc_url").asText(), getDriverClass(), - jdbcConfig.has("connection_properties") ? jdbcConfig.get("connection_properties").asText() : null); + JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties")); if (supportsSchemas()) { createSchemas(); diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index ee9b6c937115f..7fd35a2f89080 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -77,9 +77,10 @@ Therefore, Airbyte ClickHouse destination will create tables and schemas using t ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.1 | 2021-12-21 | [\#8982](https://github.com/airbytehq/airbyte/pull/8982) | Set isSchemaRequired to false | -| 0.1.0 | 2021-11-04 | [\#7620](https://github.com/airbytehq/airbyte/pull/7620) | Add ClickHouse destination | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :--- |:---------------------------------------------| +| 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.1 | 2021-12-21 | [\#8982](https://github.com/airbytehq/airbyte/pull/8982) | Set isSchemaRequired to false | +| 0.1.0 | 2021-11-04 | [\#7620](https://github.com/airbytehq/airbyte/pull/7620) | Add ClickHouse destination | diff --git a/docs/integrations/destinations/mariadb-columnstore.md b/docs/integrations/destinations/mariadb-columnstore.md index af57ac6d9dbdf..e2a870033019a 100644 --- a/docs/integrations/destinations/mariadb-columnstore.md +++ b/docs/integrations/destinations/mariadb-columnstore.md @@ -74,10 +74,11 @@ Using this feature requires additional configuration, when creating the destinat ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------| :--- |:---------------------------------------------------------|:------------------------------------------| +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------| +| 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.2 | 2021-12-30 | [\#8809](https://github.com/airbytehq/airbyte/pull/8809) | Update connector fields title/description | -| 0.1.1 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key. | -| 0.1.0 | 2021-11-15 | [\#7961](https://github.com/airbytehq/airbyte/pull/7961) | Added MariaDB ColumnStore destination. | +| 0.1.2 | 2021-12-30 | [\#8809](https://github.com/airbytehq/airbyte/pull/8809) | Update connector fields title/description | +| 0.1.1 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key. | +| 0.1.0 | 2021-11-15 | [\#7961](https://github.com/airbytehq/airbyte/pull/7961) | Added MariaDB ColumnStore destination. | diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md index 53d91ff375894..8346c156fb1de 100644 --- a/docs/integrations/destinations/mssql.md +++ b/docs/integrations/destinations/mssql.md @@ -117,26 +117,29 @@ Using this feature requires additional configuration, when creating the source. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--- |:---------------------------------------------------------| :--- | -| 0.1.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.13 | 2021-12-28 | [\#9158](https://github.com/airbytehq/airbyte/pull/9158) | Update connector fields title/description | -| 0.1.12 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| Version | Date | Pull Request | Subject | +|:--------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| +| 0.1.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.13 | 2021-12-28 | [\#9158](https://github.com/airbytehq/airbyte/pull/9158) | Update connector fields title/description | +| 0.1.12 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | | 0.1.11 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | -| 0.1.10 | 2021-10-11 | [\#6877](https://github.com/airbytehq/airbyte/pull/6877) | Add `normalization` capability, add `append+deduplication` sync mode | -| 0.1.9 | 2021-09-29 | [\#5970](https://github.com/airbytehq/airbyte/pull/5970) | Add support & test cases for MSSQL Destination via SSH tunnels | -| 0.1.8 | 2021-08-07 | [\#5272](https://github.com/airbytehq/airbyte/pull/5272) | Add batch method to insert records | -| 0.1.7 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | -| 0.1.6 | 2021-06-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | -| 0.1.5 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | declare object types correctly in spec | -| 0.1.4 | 2021-06-17 | [\#3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix doc/params in specification file | -| 0.1.3 | 2021-05-28 | [\#3728](https://github.com/airbytehq/airbyte/pull/3973) | Change dockerfile entrypoint | -| 0.1.2 | 2021-05-13 | [\#3367](https://github.com/airbytehq/airbyte/pull/3671) | Fix handle symbols unicode | -| 0.1.1 | 2021-05-11 | [\#3566](https://github.com/airbytehq/airbyte/pull/3195) | MS SQL Server Destination Release! | +| 0.1.10 | 2021-10-11 | [\#6877](https://github.com/airbytehq/airbyte/pull/6877) | Add `normalization` capability, add `append+deduplication` sync mode | +| 0.1.9 | 2021-09-29 | [\#5970](https://github.com/airbytehq/airbyte/pull/5970) | Add support & test cases for MSSQL Destination via SSH tunnels | +| 0.1.8 | 2021-08-07 | [\#5272](https://github.com/airbytehq/airbyte/pull/5272) | Add batch method to insert records | +| 0.1.7 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | +| 0.1.6 | 2021-06-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | +| 0.1.5 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | declare object types correctly in spec | +| 0.1.4 | 2021-06-17 | [\#3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix doc/params in specification file | +| 0.1.3 | 2021-05-28 | [\#3728](https://github.com/airbytehq/airbyte/pull/3973) | Change dockerfile entrypoint | +| 0.1.2 | 2021-05-13 | [\#3367](https://github.com/airbytehq/airbyte/pull/3671) | Fix handle symbols unicode | +| 0.1.1 | 2021-05-11 | [\#3566](https://github.com/airbytehq/airbyte/pull/3195) | MS SQL Server Destination Release! | ### Changelog (Strict Encrypt) + | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.1.5 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.1.4 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.3 | 2021-12-28 | [\#9158](https://github.com/airbytehq/airbyte/pull/9158) | Update connector fields title/description | | 0.1.2 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | diff --git a/docs/integrations/destinations/mysql.md b/docs/integrations/destinations/mysql.md index 35486908b5077..26ab85bbf7f2c 100644 --- a/docs/integrations/destinations/mysql.md +++ b/docs/integrations/destinations/mysql.md @@ -103,31 +103,32 @@ Using this feature requires additional configuration, when creating the destinat ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------| :--- | :--- | :--- | -| 0.1.17 | 2022-02-16 | [10362](https://github.com/airbytehq/airbyte/pull/10362) | Add jdbc_url_params support for optional JDBC parameters | -| 0.1.16 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.15 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| Version | Date | Pull Request | Subject | +|:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------| +| 0.1.18 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.17 | 2022-02-16 | [10362](https://github.com/airbytehq/airbyte/pull/10362) | Add jdbc_url_params support for optional JDBC parameters | +| 0.1.16 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.15 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | | 0.1.14 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | -| 0.1.13 | 2021-09-28 | [\#6506](https://github.com/airbytehq/airbyte/pull/6506) | Added support for MySQL destination via TLS/SSL | -| 0.1.12 | 2021-09-24 | [\#6317](https://github.com/airbytehq/airbyte/pull/6317) | Added option to connect to DB via SSH | -| 0.1.11 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | -| 0.1.10 | 2021-07-28 | [\#5026](https://github.com/airbytehq/airbyte/pull/5026) | Add sanitized json fields in raw tables to handle quotes in column names | -| 0.1.7 | 2021-07-09 | [\#4651](https://github.com/airbytehq/airbyte/pull/4651) | Switch normalization flag on so users can use normalization. | -| 0.1.6 | 2021-07-03 | [\#4531](https://github.com/airbytehq/airbyte/pull/4531) | Added normalization for MySQL. | -| 0.1.5 | 2021-07-03 | [\#3973](https://github.com/airbytehq/airbyte/pull/3973) | Added `AIRBYTE_ENTRYPOINT` for kubernetes support. | -| 0.1.4 | 2021-07-03 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Switched to get states from destination instead of source. | -| 0.1.3 | 2021-07-03 | [\#3387](https://github.com/airbytehq/airbyte/pull/3387) | Fixed a bug for message length checking. | -| 0.1.2 | 2021-07-03 | [\#3327](https://github.com/airbytehq/airbyte/pull/3327) | Fixed LSEP unicode characters. | -| 0.1.1 | 2021-07-03 | [\#3289](https://github.com/airbytehq/airbyte/pull/3289) | Added support for outputting messages. | -| 0.1.0 | 2021-05-06 | [\#3242](https://github.com/airbytehq/airbyte/pull/3242) | Added MySQL destination. | - +| 0.1.13 | 2021-09-28 | [\#6506](https://github.com/airbytehq/airbyte/pull/6506) | Added support for MySQL destination via TLS/SSL | +| 0.1.12 | 2021-09-24 | [\#6317](https://github.com/airbytehq/airbyte/pull/6317) | Added option to connect to DB via SSH | +| 0.1.11 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | +| 0.1.10 | 2021-07-28 | [\#5026](https://github.com/airbytehq/airbyte/pull/5026) | Add sanitized json fields in raw tables to handle quotes in column names | +| 0.1.7 | 2021-07-09 | [\#4651](https://github.com/airbytehq/airbyte/pull/4651) | Switch normalization flag on so users can use normalization. | +| 0.1.6 | 2021-07-03 | [\#4531](https://github.com/airbytehq/airbyte/pull/4531) | Added normalization for MySQL. | +| 0.1.5 | 2021-07-03 | [\#3973](https://github.com/airbytehq/airbyte/pull/3973) | Added `AIRBYTE_ENTRYPOINT` for kubernetes support. | +| 0.1.4 | 2021-07-03 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Switched to get states from destination instead of source. | +| 0.1.3 | 2021-07-03 | [\#3387](https://github.com/airbytehq/airbyte/pull/3387) | Fixed a bug for message length checking. | +| 0.1.2 | 2021-07-03 | [\#3327](https://github.com/airbytehq/airbyte/pull/3327) | Fixed LSEP unicode characters. | +| 0.1.1 | 2021-07-03 | [\#3289](https://github.com/airbytehq/airbyte/pull/3289) | Added support for outputting messages. | +| 0.1.0 | 2021-05-06 | [\#3242](https://github.com/airbytehq/airbyte/pull/3242) | Added MySQL destination. | ## CHANGELOG destination-mysql-strict-encrypt | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------| :--- | -| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.2 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | | 0.1.1 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | | 0.1.0 | 06.10.2021 | [\#6763](https://github.com/airbytehq/airbyte/pull/6763) | Added destination-mysql-strict-encrypt that supports SSL connections only. | diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md index 2d16d3c393ae3..c2d244980c6f4 100644 --- a/docs/integrations/destinations/oracle.md +++ b/docs/integrations/destinations/oracle.md @@ -70,66 +70,47 @@ When using an SSH tunnel, you are configuring Airbyte to connect to an intermedi Using this feature requires additional configuration, when creating the source. We will talk through what each piece of configuration means. 1. Configure all fields for the source as you normally would, except `SSH Tunnel Method`. -2. `SSH Tunnel Method` defaults to `No Tunnel` \(meaning a direct connection\). If you want to use - an SSH Tunnel choose `SSH Key Authentication` or `Password Authentication`. - 1. Choose `Key Authentication` if you will be using an RSA private key as your secret for - establishing the SSH Tunnel \(see below for more information on generating this key\). - 2. Choose `Password Authentication` if you will be using a password as your secret for - establishing the SSH Tunnel. -3. `SSH Tunnel Jump Server Host` refers to the intermediate \(bastion\) server that Airbyte will - connect to. This should be a hostname or an IP Address. -4. `SSH Connection Port` is the port on the bastion server with which to make the SSH connection. - The default port for SSH connections is `22`, so unless you have explicitly changed something, go - with the default. -5. `SSH Login Username` is the username that Airbyte should use when connection to the bastion - server. This is NOT the Oracle username. -6. If you are using `Password Authentication`, then `SSH Login Username` should be set to the - password of the User from the previous step. If you are using `SSH Key Authentication` leave this - blank. Again, this is not the Oracle password, but the password for the OS-user that Airbyte is - using to perform commands on the bastion. -7. If you are using `SSH Key Authentication`, then `SSH Private Key` should be set to the RSA - Private Key that you are using to create the SSH connection. This should be the full contents of - the key file starting with `-----BEGIN RSA PRIVATE KEY-----` and ending - with `-----END RSA PRIVATE KEY-----`. +2. `SSH Tunnel Method` defaults to `No Tunnel` \(meaning a direct connection\). If you want to use an SSH Tunnel choose `SSH Key Authentication` or `Password Authentication`. + 1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel \(see below for more information on generating this key\). + 2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel. +3. `SSH Tunnel Jump Server Host` refers to the intermediate \(bastion\) server that Airbyte will connect to. This should be a hostname or an IP Address. +4. `SSH Connection Port` is the port on the bastion server with which to make the SSH connection. The default port for SSH connections is `22`, so unless you have explicitly changed something, go with the default. +5. `SSH Login Username` is the username that Airbyte should use when connection to the bastion server. This is NOT the Oracle username. +6. If you are using `Password Authentication`, then `SSH Login Username` should be set to the password of the User from the previous step. If you are using `SSH Key Authentication` leave this blank. Again, this is not the Oracle password, but the password for the OS-user that Airbyte is using to perform commands on the bastion. +7. If you are using `SSH Key Authentication`, then `SSH Private Key` should be set to the RSA Private Key that you are using to create the SSH connection. This should be the full contents of the key file starting with `-----BEGIN RSA PRIVATE KEY-----` and ending with `-----END RSA PRIVATE KEY-----`. ## Encryption Options Airbite has the ability to connect to the Oracle source with 3 network connectivity options: -1. `Unencrypted` the connection will be made using the TCP protocol. In this case, all data over the - network will be transmitted in unencrypted form. -2. `Native network encryption` gives you the ability to encrypt database connections, without the - configuration overhead of TCP / IP and SSL / TLS and without the need to open and listen on - different ports. In this case, the *SQLNET.ENCRYPTION_CLIENT* - option will always be set as *REQUIRED* by default: The client or server will only accept - encrypted traffic, but the user has the opportunity to choose an `Encryption algorithm` according - to the security policies he needs. -3. `TLS Encrypted` (verify certificate) - if this option is selected, data transfer will be - transfered using the TLS protocol, taking into account the handshake procedure and certificate - verification. To use this option, insert the content of the certificate issued by the server into - the `SSL PEM file` field +1. `Unencrypted` the connection will be made using the TCP protocol. In this case, all data over the network will be transmitted in unencrypted form. +2. `Native network encryption` gives you the ability to encrypt database connections, without the configuration overhead of TCP / IP and SSL / TLS and without the need to open and listen on different ports. In this case, the *SQLNET.ENCRYPTION_CLIENT* + option will always be set as *REQUIRED* by default: The client or server will only accept encrypted traffic, but the user has the opportunity to choose an `Encryption algorithm` according to the security policies he needs. +3. `TLS Encrypted` (verify certificate) - if this option is selected, data transfer will be transfered using the TLS protocol, taking into account the handshake procedure and certificate verification. To use this option, insert the content of the certificate issued by the server into the `SSL PEM file` field ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- |:---------------------------------------------------------| :--- | -| 0.1.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.13 | 2021-12-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description | -| 0.1.12 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | -| 0.1.10 | 2021-10-08 | [\#6893](https://github.com/airbytehq/airbyte/pull/6893) | 🎉 Destination Oracle: implemented connection encryption | -| 0.1.9 | 2021-10-06 | [\#6611](https://github.com/airbytehq/airbyte/pull/6611) | 🐛 Destination Oracle: maxStringLength should be 128 | -| 0.1.8 | 2021-09-28 | [\#6370](https://github.com/airbytehq/airbyte/pull/6370) | Add SSH Support for Oracle Destination | -| 0.1.7 | 2021-08-30 | [\#5746](https://github.com/airbytehq/airbyte/pull/5746) | Use default column name for raw tables | -| 0.1.6 | 2021-08-23 | [\#5542](https://github.com/airbytehq/airbyte/pull/5542) | Remove support for Oracle 11g to allow normalization | -| 0.1.5 | 2021-08-10 | [\#5307](https://github.com/airbytehq/airbyte/pull/5307) | 🐛 Destination Oracle: Fix destination check for users without dba role | -| 0.1.4 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | -| 0.1.3 | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | -| 0.1.2 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | Require `sid` instead of `database` in connector specification | - +| Version | Date | Pull Request | Subject | +|:--------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| +| 0.1.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling and remove DBT support | +| 0.1.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.13 | 2021-12-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description | +| 0.1.12 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | +| 0.1.10 | 2021-10-08 | [\#6893](https://github.com/airbytehq/airbyte/pull/6893) | 🎉 Destination Oracle: implemented connection encryption | +| 0.1.9 | 2021-10-06 | [\#6611](https://github.com/airbytehq/airbyte/pull/6611) | 🐛 Destination Oracle: maxStringLength should be 128 | +| 0.1.8 | 2021-09-28 | [\#6370](https://github.com/airbytehq/airbyte/pull/6370) | Add SSH Support for Oracle Destination | +| 0.1.7 | 2021-08-30 | [\#5746](https://github.com/airbytehq/airbyte/pull/5746) | Use default column name for raw tables | +| 0.1.6 | 2021-08-23 | [\#5542](https://github.com/airbytehq/airbyte/pull/5542) | Remove support for Oracle 11g to allow normalization | +| 0.1.5 | 2021-08-10 | [\#5307](https://github.com/airbytehq/airbyte/pull/5307) | 🐛 Destination Oracle: Fix destination check for users without dba role | +| 0.1.4 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | +| 0.1.3 | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | +| 0.1.2 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | Require `sid` instead of `database` in connector specification | ### Changelog (Strict Encrypt) -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:--------------------------------------------------------| :--- | -| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.2 | 2021-01-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description | -| 0.1.1 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:--------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------| +| 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling and remove DBT support | +| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.2 | 2021-01-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description | +| 0.1.1 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md index 6ec3b4d743da1..3c10867e65d8e 100644 --- a/docs/integrations/destinations/postgres.md +++ b/docs/integrations/destinations/postgres.md @@ -82,11 +82,12 @@ Therefore, Airbyte Postgres destination will create tables and schemas using the ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--- | :--- | :--- | -| 0.3.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.3.13 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| Version | Date | Pull Request | Subject | +|:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------| +| 0.3.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.3.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.3.13 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | | 0.3.12 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | -| 0.3.11 | 2021-09-07 | [\#5743](https://github.com/airbytehq/airbyte/pull/5743) | Add SSH Tunnel support | -| 0.3.10 | 2021-08-11 | [\#5336](https://github.com/airbytehq/airbyte/pull/5336) | 🐛 Destination Postgres: fix \u0000\(NULL\) value processing | +| 0.3.11 | 2021-09-07 | [\#5743](https://github.com/airbytehq/airbyte/pull/5743) | Add SSH Tunnel support | +| 0.3.10 | 2021-08-11 | [\#5336](https://github.com/airbytehq/airbyte/pull/5336) | 🐛 Destination Postgres: fix \u0000\(NULL\) value processing | diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 7885ac484ec68..17d5c6fb3ed6d 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -123,13 +123,14 @@ All Redshift connections are encrypted using SSL ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :-------- | :----- | :------ | -| 0.3.25 | 2022-02-14 | [#9920](https://github.com/airbytehq/airbyte/pull/9920) | Updated the size of staging files for S3 staging. Also, added closure of S3 writers to staging files when data has been written to an staging file. | -| 0.3.24 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.3.23 | 2021-12-16 | [\#8855](https://github.com/airbytehq/airbyte/pull/8855) | Add `purgeStagingData` option to enable/disable deleting the staging data | -| 0.3.22 | 2021-12-15 | [#8607](https://github.com/airbytehq/airbyte/pull/8607) | Accept a path for the staging data | -| 0.3.21 | 2021-12-10 | [#8562](https://github.com/airbytehq/airbyte/pull/8562) | Moving classes around for better dependency management | -| 0.3.20 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | +|:--------| :-------- | :----- | :------ | +| 0.3.27 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.3.25 | 2022-02-14 | [#9920](https://github.com/airbytehq/airbyte/pull/9920) | Updated the size of staging files for S3 staging. Also, added closure of S3 writers to staging files when data has been written to an staging file. | +| 0.3.24 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.3.23 | 2021-12-16 | [\#8855](https://github.com/airbytehq/airbyte/pull/8855) | Add `purgeStagingData` option to enable/disable deleting the staging data | +| 0.3.22 | 2021-12-15 | [#8607](https://github.com/airbytehq/airbyte/pull/8607) | Accept a path for the staging data | +| 0.3.21 | 2021-12-10 | [#8562](https://github.com/airbytehq/airbyte/pull/8562) | Moving classes around for better dependency management | +| 0.3.20 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | | 0.3.19 | 2021-10-21 | [7234](https://github.com/airbytehq/airbyte/pull/7234) | Allow SSL traffic only | | 0.3.17 | 2021-10-12 | [6965](https://github.com/airbytehq/airbyte/pull/6965) | Added SSL Support | | 0.3.16 | 2021-10-11 | [6949](https://github.com/airbytehq/airbyte/pull/6949) | Each stream was split into files of 10,000 records each for copying using S3 or GCS | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index aa39e657f2b7f..8f0b165cfce7f 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -224,6 +224,7 @@ Finally, you need to add read/write permissions to your bucket with that email. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----- | :------ | +| 0.4.17 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.4.14 | 2022-02-17 | [\#10394](https://github.com/airbytehq/airbyte/pull/10394) | Reduce memory footprint. | | 0.4.13 | 2022-02-16 | [\#10212](https://github.com/airbytehq/airbyte/pull/10212) | Execute COPY command in parallel for S3 and GCS staging | | 0.4.12 | 2022-02-15 | [\#10342](https://github.com/airbytehq/airbyte/pull/10342) | Use connection pool, and fix connection leak. |