From 16b0f10fef2a417743c408c8978f76aa9f41efa8 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 13:11:11 -0800 Subject: [PATCH 01/60] refactoring to allow testing --- .../main/java/io/airbyte/db/Databases.java | 150 +++++++++--------- .../MariadbColumnstoreDestination.java | 6 +- .../destination/mysql/MySQLDestination.java | 30 +--- .../mysql/MySQLDestinationTest.java | 26 ++- .../source/jdbc/AbstractJdbcSource.java | 48 +++--- build.gradle | 42 ++--- 6 files changed, 156 insertions(+), 146 deletions(-) diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java index e25bd240e2a05..6b36b2c6a8d63 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java @@ -4,6 +4,8 @@ package io.airbyte.db; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Maps; import io.airbyte.commons.lang.Exceptions; import io.airbyte.db.bigquery.BigQueryDatabase; import io.airbyte.db.jdbc.DefaultJdbcDatabase; @@ -14,8 +16,8 @@ import io.airbyte.db.jdbc.StreamingJdbcDatabase; import io.airbyte.db.mongodb.MongoDatabase; import java.io.IOException; +import java.util.HashMap; import java.util.Map; -import java.util.Optional; import java.util.function.Function; import lombok.val; import org.apache.commons.dbcp2.BasicDataSource; @@ -33,15 +35,15 @@ public static Database createPostgresDatabase(final String username, final Strin } public static Database createPostgresDatabaseWithRetry(final String username, - final String password, - final String jdbcConnectionString, - final Function isDbReady) { + final String password, + final String jdbcConnectionString, + final Function isDbReady) { Database database = null; while (database == null) { try { val infinity = Integer.MAX_VALUE; database = createPostgresDatabaseWithRetryTimeout(username, password, jdbcConnectionString, isDbReady, infinity); - } catch (IOException e) { + } catch (final IOException e) { // This should theoretically never happen since we set the timeout to be a very high number. } } @@ -51,10 +53,10 @@ public static Database createPostgresDatabaseWithRetry(final String username, } public static Database createPostgresDatabaseWithRetryTimeout(final String username, - final String password, - final String jdbcConnectionString, - final Function isDbReady, - final long timeoutMs) + final String password, + final String jdbcConnectionString, + final Function isDbReady, + final long timeoutMs) throws IOException { Database database = null; if (jdbcConnectionString == null || jdbcConnectionString.trim().equals("")) { @@ -117,111 +119,91 @@ public static Database createMariaDbDatabase(final String username, final String } public static Database createDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final SQLDialect dialect) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final SQLDialect dialect) { final BasicDataSource connectionPool = createBasicDataSource(username, password, jdbcConnectionString, driverClassName); return new Database(connectionPool, dialect); } public static Database createDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final SQLDialect dialect, - final String connectionProperties) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final SQLDialect dialect, + final Map connectionProperties) { final BasicDataSource connectionPool = - createBasicDataSource(username, password, jdbcConnectionString, driverClassName, Optional.ofNullable(connectionProperties)); + createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); return new Database(connectionPool, dialect); } public static JdbcDatabase createJdbcDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName) { + final String password, + final String jdbcConnectionString, + final String driverClassName) { return createJdbcDatabase(username, password, jdbcConnectionString, driverClassName, JdbcUtils.getDefaultSourceOperations()); } public static JdbcDatabase createJdbcDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final JdbcSourceOperations sourceOperations) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final JdbcSourceOperations sourceOperations) { final BasicDataSource connectionPool = createBasicDataSource(username, password, jdbcConnectionString, driverClassName); return new DefaultJdbcDatabase(connectionPool, sourceOperations); } public static JdbcDatabase createJdbcDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final String connectionProperties) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final Map connectionProperties) { return createJdbcDatabase(username, password, jdbcConnectionString, driverClassName, connectionProperties, JdbcUtils.getDefaultSourceOperations()); } public static JdbcDatabase createJdbcDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final String connectionProperties, - final JdbcCompatibleSourceOperations sourceOperations) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final Map connectionProperties, + final JdbcCompatibleSourceOperations sourceOperations) { final BasicDataSource connectionPool = - createBasicDataSource(username, password, jdbcConnectionString, driverClassName, Optional.ofNullable(connectionProperties)); + createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); return new DefaultJdbcDatabase(connectionPool, sourceOperations); } public static JdbcDatabase createStreamingJdbcDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final JdbcStreamingQueryConfiguration jdbcStreamingQuery, - final String connectionProperties, - final JdbcCompatibleSourceOperations sourceOperations) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final JdbcStreamingQueryConfiguration jdbcStreamingQuery, + final Map connectionProperties, + final JdbcCompatibleSourceOperations sourceOperations) { final BasicDataSource connectionPool = - createBasicDataSource(username, password, jdbcConnectionString, driverClassName, Optional.ofNullable(connectionProperties)); + createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); return new StreamingJdbcDatabase(connectionPool, sourceOperations, jdbcStreamingQuery); } private static BasicDataSource createBasicDataSource(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName) { + final String password, + final String jdbcConnectionString, + final String driverClassName) { return createBasicDataSource(username, password, jdbcConnectionString, driverClassName, - Optional.empty()); - } - - /** - * Prefer to use the method that takes in the connection properties as a map. - */ - @Deprecated - private static BasicDataSource createBasicDataSource(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final Optional connectionProperties) { - final BasicDataSource connectionPool = new BasicDataSource(); - connectionPool.setDriverClassName(driverClassName); - connectionPool.setUsername(username); - connectionPool.setPassword(password); - connectionPool.setInitialSize(0); - connectionPool.setMaxTotal(5); - connectionPool.setUrl(jdbcConnectionString); - connectionProperties.ifPresent(connectionPool::setConnectionProperties); - return connectionPool; + Maps.newHashMap()); } public static BasicDataSource createBasicDataSource(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final Map connectionProperties) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final Map connectionProperties) { final BasicDataSource connectionPool = new BasicDataSource(); connectionPool.setDriverClassName(driverClassName); connectionPool.setUsername(username); @@ -241,4 +223,30 @@ public static MongoDatabase createMongoDatabase(final String connectionString, f return new MongoDatabase(connectionString, databaseName); } + public static Map parseJdbcParameters(final JsonNode config, final String jdbcParametersKey) { + if (config.has(jdbcParametersKey)) { + return parseJdbcParameters(config.get(jdbcParametersKey).asText()); + } else { + return Maps.newHashMap(); + } + } + + public static Map parseJdbcParameters(final String jdbcPropertiesString) { + final Map parameters = new HashMap<>(); + if (!jdbcPropertiesString.isBlank()) { + final String[] keyValuePairs = jdbcPropertiesString.split("&"); + for (final String kv : keyValuePairs) { + final String[] split = kv.split("="); + if (split.length == 2) { + parameters.put(split[0], split[1]); + } else { + throw new IllegalArgumentException( + "jdbc_url_params must be formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). Got " + + jdbcPropertiesString); + } + } + } + return parameters; + } + } diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java index 94568fcaf874a..8f224779e5568 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java @@ -36,7 +36,7 @@ public MariadbColumnstoreDestination() { } @Override - public AirbyteConnectionStatus check(JsonNode config) { + public AirbyteConnectionStatus check(final JsonNode config) { try (final JdbcDatabase database = getDatabase(config)) { final MariadbColumnstoreSqlOperations mariadbColumnstoreSqlOperations = (MariadbColumnstoreSqlOperations) getSqlOperations(); final String outputSchema = getNamingResolver().getIdentifier(config.get("database").asText()); @@ -74,7 +74,7 @@ protected JdbcDatabase getDatabase(final JsonNode config) { jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, jdbcConfig.get("jdbc_url").asText(), getDriverClass(), - "allowLoadLocalInfile=true"); + Databases.parseJdbcParameters("allowLoadLocalInfile=true")); } @Override @@ -95,7 +95,7 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } - public static void main(String[] args) throws Exception { + public static void main(final String[] args) throws Exception { final Destination destination = MariadbColumnstoreDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", MariadbColumnstoreDestination.class); new IntegrationRunner(destination).run(args); diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java index ec2016a41dfcb..ce9d6cd81ead3 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java @@ -19,7 +19,6 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -42,7 +41,7 @@ public class MySQLDestination extends AbstractJdbcDestination implements Destina public static final String USERNAME_KEY = "username"; public static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; - + static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( "useSSL", "true", "requireSSL", "true", @@ -91,12 +90,14 @@ public MySQLDestination() { protected JdbcDatabase getDatabase(final JsonNode config) { final JsonNode jdbcConfig = toJdbcConfig(config); + final Map connectionProperties = ImmutableMap.of("allowLoadLocalInfile", "true"); + return Databases.createJdbcDatabase( jdbcConfig.get(USERNAME_KEY).asText(), jdbcConfig.has(PASSWORD_KEY) ? jdbcConfig.get(PASSWORD_KEY).asText() : null, jdbcConfig.get(JDBC_URL_KEY).asText(), getDriverClass(), - "allowLoadLocalInfile=true"); + connectionProperties); } @Override @@ -129,7 +130,7 @@ public JsonNode toJdbcConfig(final JsonNode config) { } private List getAdditionalParameters(final JsonNode config) { - final Map customParameters = getCustomJdbcParameters(config); + final Map customParameters = Databases.parseJdbcParameters(config, JDBC_URL_PARAMS_KEY); if (useSSL(config)) { return convertToJdbcStrings(customParameters, MoreMaps.merge(DEFAULT_JDBC_PARAMETERS, SSL_JDBC_PARAMETERS)); @@ -156,27 +157,6 @@ private void assertCustomParametersDontOverwriteDefaultParameters(final Map getCustomJdbcParameters(final JsonNode config) { - final Map parameters = new HashMap<>(); - if (config.has(JDBC_URL_PARAMS_KEY)) { - final String jdbcParams = config.get(JDBC_URL_PARAMS_KEY).asText(); - if (!jdbcParams.isBlank()) { - final String[] keyValuePairs = jdbcParams.split("&"); - for (final String kv : keyValuePairs) { - final String[] split = kv.split("="); - if (split.length == 2) { - parameters.put(split[0], split[1]); - } else { - throw new IllegalArgumentException( - "jdbc_url_params must be formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). Got " - + jdbcParams); - } - } - } - } - return parameters; - } - private boolean useSSL(final JsonNode config) { return !config.has(SSL_KEY) || config.get(SSL_KEY).asBoolean(); } diff --git a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java index aad36914cf401..93f33764130ba 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java @@ -3,20 +3,26 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.spy; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.times; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; +import io.airbyte.db.Databases; +import java.sql.SQLException; import java.util.Map; import java.util.Map.Entry; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; public class MySQLDestinationTest { private MySQLDestination getDestination() { - final MySQLDestination result = spy(MySQLDestination.class); + final MySQLDestination result = new MySQLDestination(); return result; } @@ -133,4 +139,20 @@ void testInvalidExtraParam() { getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam)); }); } + + @Test + void test() throws SQLException { + final ImmutableMap connectionProperties = ImmutableMap.of("allowLoadLocalInfile", "true"); + verifyJdbcDatabaseIsCreatedWithConnectionProperties(() -> getDestination().getDatabase(buildConfigNoExtraJdbcParametersWithoutSsl()), + connectionProperties); + } + + void verifyJdbcDatabaseIsCreatedWithConnectionProperties(final Runnable runnable, final Map connectionProperties) { + try (final MockedStatic databases = Mockito.mockStatic(Databases.class)) { + runnable.run(); + databases.verify(() -> Databases.createJdbcDatabase( + anyString(), nullable(String.class), anyString(), anyString(), Mockito.eq(connectionProperties)), + times(1)); + } + } } diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index f105185c4608a..4e64f544d82e5 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -55,9 +55,8 @@ import org.slf4j.LoggerFactory; /** - * This class contains helper functions and boilerplate for implementing a source connector for a - * relational DB source which can be accessed via JDBC driver. If you are implementing a connector - * for a relational DB which has a JDBC driver, make an effort to use this class. + * This class contains helper functions and boilerplate for implementing a source connector for a relational DB source which can be accessed via JDBC + * driver. If you are implementing a connector for a relational DB which has a JDBC driver, make an effort to use this class. */ public abstract class AbstractJdbcSource extends AbstractRelationalDbSource implements Source { @@ -69,9 +68,11 @@ public abstract class AbstractJdbcSource extends AbstractRelationalDbS protected String quoteString; + public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params"; + public AbstractJdbcSource(final String driverClass, - final JdbcStreamingQueryConfiguration jdbcStreamingQueryConfiguration, - final JdbcCompatibleSourceOperations sourceOperations) { + final JdbcStreamingQueryConfiguration jdbcStreamingQueryConfiguration, + final JdbcCompatibleSourceOperations sourceOperations) { this.driverClass = driverClass; this.jdbcStreamingQueryConfiguration = jdbcStreamingQueryConfiguration; this.sourceOperations = sourceOperations; @@ -114,10 +115,10 @@ protected List>> discoverInternal(final JdbcData final Set internalSchemas = new HashSet<>(getExcludedInternalNameSpaces()); final Set tablesWithSelectGrantPrivilege = getPrivilegesTableForCurrentUser(database, schema); return database.bufferedResultSetQuery( - // retrieve column metadata from the database - conn -> conn.getMetaData().getColumns(getCatalog(database), schema, null, null), - // store essential column metadata to a Json object from the result set about each column - this::getColumnMetadata) + // retrieve column metadata from the database + conn -> conn.getMetaData().getColumns(getCatalog(database), schema, null, null), + // store essential column metadata to a Json object from the result set about each column + this::getColumnMetadata) .stream() .filter(excludeNotAccessibleTables(internalSchemas, tablesWithSelectGrantPrivilege)) // group by schema and table name to handle the case where a table with the same name exists in @@ -139,7 +140,8 @@ protected List>> discoverInternal(final JdbcData f.get(INTERNAL_COLUMN_TYPE_NAME).asText(), f.get(INTERNAL_COLUMN_SIZE).asInt(), jsonType); - return new CommonField(f.get(INTERNAL_COLUMN_NAME).asText(), datatype) {}; + return new CommonField(f.get(INTERNAL_COLUMN_NAME).asText(), datatype) { + }; }) .collect(Collectors.toList())) .build()) @@ -147,7 +149,7 @@ protected List>> discoverInternal(final JdbcData } protected Predicate excludeNotAccessibleTables(final Set internalSchemas, - final Set tablesWithSelectGrantPrivilege) { + final Set tablesWithSelectGrantPrivilege) { return jsonNode -> { if (tablesWithSelectGrantPrivilege.isEmpty()) { return isNotInternalSchema(jsonNode, internalSchemas); @@ -155,13 +157,13 @@ protected Predicate excludeNotAccessibleTables(final Set inter return tablesWithSelectGrantPrivilege.stream() .anyMatch(e -> e.getSchemaName().equals(jsonNode.get(INTERNAL_SCHEMA_NAME).asText())) && tablesWithSelectGrantPrivilege.stream() - .anyMatch(e -> e.getTableName().equals(jsonNode.get(INTERNAL_TABLE_NAME).asText())) + .anyMatch(e -> e.getTableName().equals(jsonNode.get(INTERNAL_TABLE_NAME).asText())) && !internalSchemas.contains(jsonNode.get(INTERNAL_SCHEMA_NAME).asText()); }; } // needs to override isNotInternalSchema for connectors that override getPrivilegesTableForCurrentUser() - protected boolean isNotInternalSchema(JsonNode jsonNode, Set internalSchemas) { + protected boolean isNotInternalSchema(final JsonNode jsonNode, final Set internalSchemas) { return !internalSchemas.contains(jsonNode.get(INTERNAL_SCHEMA_NAME).asText()); } @@ -184,8 +186,7 @@ private JsonNode getColumnMetadata(final ResultSet resultSet) throws SQLExceptio } /** - * @param field Essential column information returned from - * {@link AbstractJdbcSource#getColumnMetadata}. + * @param field Essential column information returned from {@link AbstractJdbcSource#getColumnMetadata}. */ public Datatype getFieldType(final JsonNode field) { return sourceOperations.getFieldType(field); @@ -204,7 +205,7 @@ public JsonSchemaType getType(final Datatype columnType) { @Override protected Map> discoverPrimaryKeys(final JdbcDatabase database, - final List>> tableInfos) { + final List>> tableInfos) { LOGGER.info("Discover primary keys for tables: " + tableInfos.stream().map(TableInfo::getName).collect( Collectors.toSet())); try { @@ -251,12 +252,12 @@ protected String getQuoteString() { @Override public AutoCloseableIterator queryTableIncremental(final JdbcDatabase database, - final List columnNames, - final String schemaName, - final String tableName, - final String cursorField, - final Datatype cursorFieldType, - final String cursor) { + final List columnNames, + final String schemaName, + final String tableName, + final String cursorField, + final Datatype cursorFieldType, + final String cursor) { LOGGER.info("Queueing query for table: {}", tableName); return AutoCloseableIterators.lazyIterator(() -> { try { @@ -292,12 +293,11 @@ public JdbcDatabase createDatabase(final JsonNode config) throws SQLException { jdbcConfig.get("jdbc_url").asText(), driverClass, jdbcStreamingQueryConfiguration, - jdbcConfig.has("connection_properties") ? jdbcConfig.get("connection_properties").asText() : null, + Databases.parseJdbcParameters(config.get("connection_properties").asText()), sourceOperations); quoteString = (quoteString == null ? database.getMetaData().getIdentifierQuoteString() : quoteString); return database; } - } diff --git a/build.gradle b/build.gradle index 5f5a6960a468f..3b83e1002f85f 100644 --- a/build.gradle +++ b/build.gradle @@ -71,25 +71,25 @@ def createJavaLicenseWith = { license -> // monorepo setup and it doesn't actually exclude directories reliably. This code makes the behavior predictable. def createSpotlessTarget = { pattern -> def excludes = [ - '.gradle', - 'node_modules', - '.eggs', - '.mypy_cache', - '.venv', - '*.egg-info', - 'build', - 'dbt-project-template', - 'dbt-project-template-mssql', - 'dbt-project-template-mysql', - 'dbt-project-template-oracle', - 'dbt-project-template-clickhouse', - 'dbt-project-template-snowflake', - 'dbt_test_config', - 'normalization_test_output', - 'tools', - 'secrets', - 'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. - 'resources/seed/*_specs.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file. + '.gradle', + 'node_modules', + '.eggs', + '.mypy_cache', + '.venv', + '*.egg-info', + 'build', + 'dbt-project-template', + 'dbt-project-template-mssql', + 'dbt-project-template-mysql', + 'dbt-project-template-oracle', + 'dbt-project-template-clickhouse', + 'dbt-project-template-snowflake', + 'dbt_test_config', + 'normalization_test_output', + 'tools', + 'secrets', + 'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. + 'resources/seed/*_specs.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file. ] if (System.getenv().containsKey("SUB_BUILD")) { @@ -129,7 +129,7 @@ check.dependsOn 'spotlessApply' @SuppressWarnings('GroovyAssignabilityCheck') def Task getDockerBuildTask(String artifactName, String projectDir) { - return task ("buildDockerImage-$artifactName"(type: DockerBuildImage) { + return task("buildDockerImage-$artifactName"(type: DockerBuildImage) { def buildTag = System.getenv('VERSION') ?: 'dev' def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64' def alpineImage = System.getenv('ALPINE_IMAGE') ?: 'alpine:3.4' @@ -304,7 +304,7 @@ subprojects { testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2' testImplementation 'org.mockito:mockito-junit-jupiter:4.0.0' testImplementation 'org.assertj:assertj-core:3.21.0' - + testImplementation "org.mockito:mockito-inline:4.1.0" } tasks.withType(Tar) { From 79348de3d9616bf8e7c1bc2de69d31b6be2b2749 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 14:27:48 -0800 Subject: [PATCH 02/60] MySQLDestination uses connection property map instead of url arguments --- .../main/java/io/airbyte/db/Databases.java | 29 ------ .../java/io/airbyte/db/jdbc/JdbcUtils.java | 1 - .../jdbc/AbstractJdbcDestination.java | 48 ++++++++-- .../MariadbColumnstoreDestination.java | 3 +- .../destination/mysql/MySQLDestination.java | 94 ++++++++----------- .../mysql/MySQLDestinationTest.java | 64 ++++++++----- .../source/jdbc/AbstractJdbcSource.java | 3 +- 7 files changed, 122 insertions(+), 120 deletions(-) diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java index 6b36b2c6a8d63..b095a429dfd87 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java @@ -4,7 +4,6 @@ package io.airbyte.db; -import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Maps; import io.airbyte.commons.lang.Exceptions; import io.airbyte.db.bigquery.BigQueryDatabase; @@ -16,7 +15,6 @@ import io.airbyte.db.jdbc.StreamingJdbcDatabase; import io.airbyte.db.mongodb.MongoDatabase; import java.io.IOException; -import java.util.HashMap; import java.util.Map; import java.util.function.Function; import lombok.val; @@ -222,31 +220,4 @@ public static BigQueryDatabase createBigQueryDatabase(final String projectId, fi public static MongoDatabase createMongoDatabase(final String connectionString, final String databaseName) { return new MongoDatabase(connectionString, databaseName); } - - public static Map parseJdbcParameters(final JsonNode config, final String jdbcParametersKey) { - if (config.has(jdbcParametersKey)) { - return parseJdbcParameters(config.get(jdbcParametersKey).asText()); - } else { - return Maps.newHashMap(); - } - } - - public static Map parseJdbcParameters(final String jdbcPropertiesString) { - final Map parameters = new HashMap<>(); - if (!jdbcPropertiesString.isBlank()) { - final String[] keyValuePairs = jdbcPropertiesString.split("&"); - for (final String kv : keyValuePairs) { - final String[] split = kv.split("="); - if (split.length == 2) { - parameters.put(split[0], split[1]); - } else { - throw new IllegalArgumentException( - "jdbc_url_params must be formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). Got " - + jdbcPropertiesString); - } - } - } - return parameters; - } - } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index a7d681857aae6..02d13fec795f6 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -23,5 +23,4 @@ public static JSONFormat getDefaultJSONFormat() { public static String getFullyQualifiedTableName(final String schemaName, final String tableName) { return schemaName != null ? schemaName + "." + tableName : tableName; } - } diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java index 4a58e8299a8fc..5442d42c92395 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.destination.jdbc; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Maps; import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; @@ -17,6 +18,8 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; import java.util.function.Consumer; import org.slf4j.Logger; @@ -43,8 +46,8 @@ protected SqlOperations getSqlOperations() { } public AbstractJdbcDestination(final String driverClass, - final NamingConventionTransformer namingResolver, - final SqlOperations sqlOperations) { + final NamingConventionTransformer namingResolver, + final SqlOperations sqlOperations) { this.driverClass = driverClass; this.namingResolver = namingResolver; this.sqlOperations = sqlOperations; @@ -67,9 +70,9 @@ public AirbyteConnectionStatus check(final JsonNode config) { } public static void attemptSQLCreateAndDropTableOperations(final String outputSchema, - final JdbcDatabase database, - final NamingConventionTransformer namingResolver, - final SqlOperations sqlOps) + final JdbcDatabase database, + final NamingConventionTransformer namingResolver, + final SqlOperations sqlOps) throws Exception { // attempt to get metadata from the database as a cheap way of seeing if we can connect. database.bufferedResultSetQuery(conn -> conn.getMetaData().getCatalogs(), JdbcUtils.getDefaultSourceOperations()::rowToJson); @@ -89,16 +92,45 @@ protected JdbcDatabase getDatabase(final JsonNode config) { jdbcConfig.get("username").asText(), jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, jdbcConfig.get("jdbc_url").asText(), - driverClass); + driverClass, + getConnectionProperties(config)); } + protected abstract Map getConnectionProperties(final JsonNode config); + public abstract JsonNode toJdbcConfig(JsonNode config); @Override public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) { + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) { return JdbcBufferedConsumerFactory.create(outputRecordCollector, getDatabase(config), sqlOperations, namingResolver, config, catalog); } + + public static Map parseJdbcParameters(final JsonNode config, final String jdbcParametersKey) { + if (config.has(jdbcParametersKey)) { + return parseJdbcParameters(config.get(jdbcParametersKey).asText()); + } else { + return Maps.newHashMap(); + } + } + + public static Map parseJdbcParameters(final String jdbcPropertiesString) { + final Map parameters = new HashMap<>(); + if (!jdbcPropertiesString.isBlank()) { + final String[] keyValuePairs = jdbcPropertiesString.split("&"); + for (final String kv : keyValuePairs) { + final String[] split = kv.split("="); + if (split.length == 2) { + parameters.put(split[0], split[1]); + } else { + throw new IllegalArgumentException( + "jdbc_url_params must be formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). Got " + + jdbcPropertiesString); + } + } + } + return parameters; + } } diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java index 8f224779e5568..3881a2f027e3f 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java @@ -9,6 +9,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.ssh.SshWrappedDestination; @@ -74,7 +75,7 @@ protected JdbcDatabase getDatabase(final JsonNode config) { jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, jdbcConfig.get("jdbc_url").asText(), getDriverClass(), - Databases.parseJdbcParameters("allowLoadLocalInfile=true")); + JdbcUtils.parseJdbcParameters("allowLoadLocalInfile=true")); } @Override diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java index ce9d6cd81ead3..7e1093810d06a 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java @@ -6,10 +6,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Streams; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; @@ -18,12 +16,9 @@ import io.airbyte.integrations.destination.mysql.MySQLSqlOperations.VersionCompatibility; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +43,13 @@ public class MySQLDestination extends AbstractJdbcDestination implements Destina "verifyServerCertificate", "false" ); static final Map DEFAULT_JDBC_PARAMETERS = ImmutableMap.of( - "zeroDateTimeBehavior", "convertToNull" + // zero dates by default cannot be parsed into java date objects (they will throw an error) + // in addition, users don't always have agency in fixing them e.g: maybe they don't own the database + // and can't + // remove zero date values. + // since zero dates are placeholders, we convert them to null by default + "zeroDateTimeBehavior", "convertToNull", + "allowLoadLocalInfile", "true" ); public static Destination sshWrappedDestination() { @@ -86,66 +87,30 @@ public MySQLDestination() { super(DRIVER_CLASS, new MySQLNameTransformer(), new MySQLSqlOperations()); } + //Override for testing purposes... @Override protected JdbcDatabase getDatabase(final JsonNode config) { - final JsonNode jdbcConfig = toJdbcConfig(config); - - final Map connectionProperties = ImmutableMap.of("allowLoadLocalInfile", "true"); - - return Databases.createJdbcDatabase( - jdbcConfig.get(USERNAME_KEY).asText(), - jdbcConfig.has(PASSWORD_KEY) ? jdbcConfig.get(PASSWORD_KEY).asText() : null, - jdbcConfig.get(JDBC_URL_KEY).asText(), - getDriverClass(), - connectionProperties); + return super.getDatabase(config); } @Override - public JsonNode toJdbcConfig(final JsonNode config) { - final List additionalParameters = getAdditionalParameters(config); - - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:mysql://%s:%s/%s", - config.get(HOST_KEY).asText(), - config.get(PORT_KEY).asText(), - config.get(DATABASE_KEY).asText())); - // zero dates by default cannot be parsed into java date objects (they will throw an error) - // in addition, users don't always have agency in fixing them e.g: maybe they don't own the database - // and can't - // remove zero date values. - // since zero dates are placeholders, we convert them to null by default - if (!additionalParameters.isEmpty()) { - jdbcUrl.append("?"); - jdbcUrl.append(String.join("&", additionalParameters)); - } - - final ImmutableMap.Builder configBuilder = ImmutableMap.builder() - .put(USERNAME_KEY, config.get(USERNAME_KEY).asText()) - .put(JDBC_URL_KEY, jdbcUrl.toString()); - - if (config.has(PASSWORD_KEY)) { - configBuilder.put(PASSWORD_KEY, config.get(PASSWORD_KEY).asText()); - } - - return Jsons.jsonNode(configBuilder.build()); + protected Map getConnectionProperties(final JsonNode config) { + final Map customProperties = parseJdbcParameters(config, JDBC_URL_PARAMS_KEY); + final Map defaultProperties = getDefaultConnectionProperties(config); + assertCustomParametersDontOverwriteDefaultParameters(customProperties, defaultProperties); + return MoreMaps.merge(customProperties, defaultProperties); } - private List getAdditionalParameters(final JsonNode config) { - final Map customParameters = Databases.parseJdbcParameters(config, JDBC_URL_PARAMS_KEY); - + private Map getDefaultConnectionProperties(final JsonNode config) { if (useSSL(config)) { - return convertToJdbcStrings(customParameters, MoreMaps.merge(DEFAULT_JDBC_PARAMETERS, SSL_JDBC_PARAMETERS)); + return MoreMaps.merge(DEFAULT_JDBC_PARAMETERS, SSL_JDBC_PARAMETERS); } else { - return convertToJdbcStrings(customParameters, DEFAULT_JDBC_PARAMETERS); + return MoreMaps.merge(DEFAULT_JDBC_PARAMETERS); } } - private List convertToJdbcStrings(final Map customParameters, final Map defaultParametersMap) { - assertCustomParametersDontOverwriteDefaultParameters(customParameters, defaultParametersMap); - return Streams.concat(Stream.of(customParameters, defaultParametersMap)) - .map(Map::entrySet) - .flatMap(Collection::stream) - .map(entry -> formatParameter(entry.getKey(), entry.getValue())) - .collect(Collectors.toList()); + private boolean useSSL(final JsonNode config) { + return !config.has(SSL_KEY) || config.get(SSL_KEY).asBoolean(); } private void assertCustomParametersDontOverwriteDefaultParameters(final Map customParameters, @@ -157,8 +122,25 @@ private void assertCustomParametersDontOverwriteDefaultParameters(final Map configBuilder = ImmutableMap.builder() + .put(USERNAME_KEY, config.get(USERNAME_KEY).asText()) + .put(JDBC_URL_KEY, jdbcUrl.toString()); + + if (config.has(PASSWORD_KEY)) { + configBuilder.put(PASSWORD_KEY, config.get(PASSWORD_KEY).asText()); + } + if (config.has(JDBC_URL_PARAMS_KEY)) { + configBuilder.put(JDBC_URL_PARAMS_KEY, config.get(JDBC_URL_PARAMS_KEY)); + } + + return Jsons.jsonNode(configBuilder.build()); } static String formatParameter(final String key, final String value) { diff --git a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java index 93f33764130ba..ea12c3864f013 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java @@ -12,7 +12,6 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; import io.airbyte.db.Databases; -import java.sql.SQLException; import java.util.Map; import java.util.Map.Entry; import org.junit.jupiter.api.Test; @@ -21,6 +20,16 @@ public class MySQLDestinationTest { + private static final Map DEFAULT_PARAMETERS_WITH_SSL = MoreMaps.merge( + MySQLDestination.DEFAULT_JDBC_PARAMETERS, + MySQLDestination.SSL_JDBC_PARAMETERS + ); + private static final Map CUSTOM_PROPERTIES = ImmutableMap.of( + "key1", "value1", + "key2", "value2", + "key3", "value3" + ); + private MySQLDestination getDestination() { final MySQLDestination result = new MySQLDestination(); return result; @@ -74,14 +83,16 @@ private JsonNode buildConfigNoExtraJdbcParametersWithoutSsl() { void testNoExtraParams() { final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigNoJdbcParameters()); final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals("jdbc:mysql://localhost:1337/db?verifyServerCertificate=false&zeroDateTimeBehavior=convertToNull&requireSSL=true&useSSL=true", url); + assertEquals("jdbc:mysql://localhost:1337/db", url); + verifyJdbcDatabaseIsCreatedWithConnectionProperties(buildConfigNoJdbcParameters(), DEFAULT_PARAMETERS_WITH_SSL); } @Test void testEmptyExtraParams() { final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters("")); final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals("jdbc:mysql://localhost:1337/db?verifyServerCertificate=false&zeroDateTimeBehavior=convertToNull&requireSSL=true&useSSL=true", url); + assertEquals("jdbc:mysql://localhost:1337/db", url); + verifyJdbcDatabaseIsCreatedWithConnectionProperties(buildConfigNoJdbcParameters(), DEFAULT_PARAMETERS_WITH_SSL); } @Test @@ -89,9 +100,10 @@ void testExtraParams() { final String extraParam = "key1=value1&key2=value2&key3=value3"; final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam)); final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals( - "jdbc:mysql://localhost:1337/db?key1=value1&key2=value2&key3=value3&verifyServerCertificate=false&zeroDateTimeBehavior=convertToNull&requireSSL=true&useSSL=true", - url); + assertEquals("jdbc:mysql://localhost:1337/db", url); + + verifyJdbcDatabaseIsCreatedWithConnectionProperties(buildConfigWithExtraJdbcParameters(extraParam), + MoreMaps.merge(CUSTOM_PROPERTIES, DEFAULT_PARAMETERS_WITH_SSL)); } @Test @@ -104,11 +116,17 @@ void testExtraParamsWithDefaultParameter() { // Do not throw an exception if the values are equal assertDoesNotThrow(() -> - getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(identicalParameter)).get("jdbc_url").asText() + getDestination().getDatabase(buildConfigWithExtraJdbcParameters(identicalParameter)) ); + final Map connectionProperties = MoreMaps.merge( + ImmutableMap.of(entry.getKey(), entry.getValue()), + MySQLDestination.DEFAULT_JDBC_PARAMETERS + ); + verifyJdbcDatabaseIsCreatedWithConnectionProperties(buildConfigWithExtraJdbcParametersWithNoSsl(identicalParameter), connectionProperties); + // Throw an exception if the values are different assertThrows(IllegalArgumentException.class, () -> - getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(overridingParameter)) + getDestination().getDatabase(buildConfigWithExtraJdbcParameters(overridingParameter)) ); } } @@ -118,38 +136,36 @@ void testExtraParameterNoSsl() { final String extraParam = "key1=value1&key2=value2&key3=value3"; final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigWithExtraJdbcParametersWithNoSsl(extraParam)); final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals( - "jdbc:mysql://localhost:1337/db?key1=value1&key2=value2&key3=value3&zeroDateTimeBehavior=convertToNull", - url); + assertEquals("jdbc:mysql://localhost:1337/db", url); + + final Map connectionProperties = ImmutableMap.of( + "key1", "value1", + "key2", "value2", + "key3", "value3"); + verifyJdbcDatabaseIsCreatedWithConnectionProperties(buildConfigWithExtraJdbcParametersWithNoSsl(extraParam), + MoreMaps.merge(connectionProperties, MySQLDestination.DEFAULT_JDBC_PARAMETERS)); } @Test void testNoExtraParameterNoSsl() { final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigNoExtraJdbcParametersWithoutSsl()); final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals( - "jdbc:mysql://localhost:1337/db?zeroDateTimeBehavior=convertToNull", - url); + assertEquals("jdbc:mysql://localhost:1337/db", url); + + verifyJdbcDatabaseIsCreatedWithConnectionProperties(buildConfigNoExtraJdbcParametersWithoutSsl(), MySQLDestination.DEFAULT_JDBC_PARAMETERS); } @Test void testInvalidExtraParam() { final String extraParam = "key1=value1&sdf&"; assertThrows(IllegalArgumentException.class, () -> { - getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam)); + getDestination().getDatabase(buildConfigWithExtraJdbcParameters(extraParam)); }); } - @Test - void test() throws SQLException { - final ImmutableMap connectionProperties = ImmutableMap.of("allowLoadLocalInfile", "true"); - verifyJdbcDatabaseIsCreatedWithConnectionProperties(() -> getDestination().getDatabase(buildConfigNoExtraJdbcParametersWithoutSsl()), - connectionProperties); - } - - void verifyJdbcDatabaseIsCreatedWithConnectionProperties(final Runnable runnable, final Map connectionProperties) { + void verifyJdbcDatabaseIsCreatedWithConnectionProperties(final JsonNode jsonNode, final Map connectionProperties) { try (final MockedStatic databases = Mockito.mockStatic(Databases.class)) { - runnable.run(); + getDestination().getDatabase(jsonNode); databases.verify(() -> Databases.createJdbcDatabase( anyString(), nullable(String.class), anyString(), anyString(), Mockito.eq(connectionProperties)), times(1)); diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index 4e64f544d82e5..613ef63d21dba 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -30,6 +30,7 @@ import io.airbyte.db.SqlDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcStreamingQueryConfiguration; +import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.dto.JdbcPrivilegeDto; import io.airbyte.integrations.source.relationaldb.AbstractRelationalDbSource; @@ -293,7 +294,7 @@ public JdbcDatabase createDatabase(final JsonNode config) throws SQLException { jdbcConfig.get("jdbc_url").asText(), driverClass, jdbcStreamingQueryConfiguration, - Databases.parseJdbcParameters(config.get("connection_properties").asText()), + JdbcUtils.parseJdbcParameters(config.get("connection_properties").asText()), sourceOperations); quoteString = (quoteString == null ? database.getMetaData().getIdentifierQuoteString() : quoteString); From 4330d375958874d5eb429d09944cd2de7269c9cf Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 14:41:25 -0800 Subject: [PATCH 03/60] Update jdbc destinations --- .../clickhouse/ClickhouseDestination.java | 9 ++++++++- .../jdbc/AbstractJdbcDestination.java | 4 ---- .../MariadbColumnstoreDestination.java | 18 +++++++---------- .../destination/oracle/OracleDestination.java | 9 ++++++++- .../postgres/PostgresDestination.java | 7 +++++++ .../redshift/RedshiftInsertDestination.java | 19 ++++++++++-------- .../snowflake/SnowflakeInsertDestination.java | 7 +++++++ .../SnowflakeInternalStagingDestination.java | 20 +++++++++++++------ 8 files changed, 62 insertions(+), 31 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index 95c0b767b1436..bdd47de55382d 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -16,7 +16,9 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,7 +85,12 @@ public AirbyteConnectionStatus check(final JsonNode config) { } } - public static void main(String[] args) throws Exception { + @Override + protected Map getConnectionProperties(final JsonNode config) { + return new HashMap<>(); + } + + public static void main(final String[] args) throws Exception { final Destination destination = ClickhouseDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", ClickhouseDestination.class); new IntegrationRunner(destination).run(args); diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java index 5442d42c92395..b588053ec46b8 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -33,10 +33,6 @@ public abstract class AbstractJdbcDestination extends BaseConnector implements D private final NamingConventionTransformer namingResolver; private final SqlOperations sqlOperations; - protected String getDriverClass() { - return driverClass; - } - protected NamingConventionTransformer getNamingResolver() { return namingResolver; } diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java index 3881a2f027e3f..e3eaed60b5770 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java @@ -7,9 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.ssh.SshWrappedDestination; @@ -18,6 +16,7 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; import java.util.List; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,6 +27,10 @@ public class MariadbColumnstoreDestination extends AbstractJdbcDestination imple public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); + static final Map DEFAULT_JDBC_PARAMETERS = ImmutableMap.of( + "allowLoadLocalInfile", "true" + ); + public static Destination sshWrappedDestination() { return new SshWrappedDestination(new MariadbColumnstoreDestination(), HOST_KEY, PORT_KEY); } @@ -67,15 +70,8 @@ public AirbyteConnectionStatus check(final JsonNode config) { } @Override - protected JdbcDatabase getDatabase(final JsonNode config) { - final JsonNode jdbcConfig = toJdbcConfig(config); - - return Databases.createJdbcDatabase( - jdbcConfig.get("username").asText(), - jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, - jdbcConfig.get("jdbc_url").asText(), - getDriverClass(), - JdbcUtils.parseJdbcParameters("allowLoadLocalInfile=true")); + protected Map getConnectionProperties(final JsonNode config) { + return DEFAULT_JDBC_PARAMETERS; } @Override diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index 6b7ea9eeb4e1a..415c8aec2ff0b 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -15,7 +15,9 @@ import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; @@ -53,6 +55,11 @@ public static Destination sshWrappedDestination() { return new SshWrappedDestination(new OracleDestination(), List.of("host"), List.of("port")); } + @Override + protected Map getConnectionProperties(final JsonNode config) { + return new HashMap<>(); + } + @Override public JsonNode toJdbcConfig(final JsonNode config) { final List additionalParameters = new ArrayList<>(); @@ -84,7 +91,7 @@ public JsonNode toJdbcConfig(final JsonNode config) { } private Protocol obtainConnectionProtocol(final JsonNode encryption, - final List additionalParameters) { + final List additionalParameters) { final String encryptionMethod = encryption.get("encryption_method").asText(); switch (encryptionMethod) { case "unencrypted" -> { diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java index 9803a677649f0..589079044c0f0 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java @@ -12,7 +12,9 @@ import io.airbyte.integrations.base.ssh.SshWrappedDestination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +35,11 @@ public PostgresDestination() { super(DRIVER_CLASS, new PostgresSQLNameTransformer(), new PostgresSqlOperations()); } + @Override + protected Map getConnectionProperties(final JsonNode config) { + return new HashMap<>(); + } + @Override public JsonNode toJdbcConfig(final JsonNode config) { final String schema = Optional.ofNullable(config.get("schema")).map(JsonNode::asText).orElse("public"); diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java index 8311463a35e73..25f1f76431ad6 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java @@ -11,8 +11,8 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,6 +23,11 @@ public class RedshiftInsertDestination extends AbstractJdbcDestination implement public static final String DRIVER_CLASS = "com.amazon.redshift.jdbc.Driver"; + private static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( + "ssl", "true", + "sslfactory", "com.amazon.redshift.ssl.NonValidatingFactory" + ); + public RedshiftInsertDestination() { super(DRIVER_CLASS, new RedshiftSQLNameTransformer(), new RedshiftSqlOperations()); } @@ -37,21 +42,19 @@ public JdbcDatabase getDatabase(final JsonNode config) { return getJdbcDatabase(config); } - private static void addSsl(final List additionalProperties) { - additionalProperties.add("ssl=true"); - additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); + @Override + protected Map getConnectionProperties(final JsonNode config) { + return new HashMap<>(); } public static JdbcDatabase getJdbcDatabase(final JsonNode config) { - final List additionalProperties = new ArrayList<>(); final var jdbcConfig = RedshiftInsertDestination.getJdbcConfig(config); - addSsl(additionalProperties); return Databases.createJdbcDatabase( jdbcConfig.get("username").asText(), jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, jdbcConfig.get("jdbc_url").asText(), RedshiftInsertDestination.DRIVER_CLASS, - String.join(";", additionalProperties)); + SSL_JDBC_PARAMETERS); } public static JsonNode getJdbcConfig(final JsonNode redshiftConfig) { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java index 58758eeb3781d..daf2b81b97502 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java @@ -9,6 +9,8 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; +import java.util.HashMap; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,6 +28,11 @@ protected JdbcDatabase getDatabase(final JsonNode config) { return SnowflakeDatabase.getDatabase(config); } + @Override + protected Map getConnectionProperties(final JsonNode config) { + return new HashMap<>(); + } + // this is a no op since we override getDatabase. @Override public JsonNode toJdbcConfig(final JsonNode config) { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index 5a9a2db88b17e..8a3841c013356 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -14,6 +14,8 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; import java.util.function.Consumer; import org.slf4j.Logger; @@ -47,14 +49,15 @@ public AirbyteConnectionStatus check(final JsonNode config) { } private static void attemptSQLCreateAndDropStages(final String outputSchema, - final JdbcDatabase database, - final SnowflakeSQLNameTransformer namingResolver, - final SnowflakeStagingSqlOperations sqlOperations) + final JdbcDatabase database, + final SnowflakeSQLNameTransformer namingResolver, + final SnowflakeStagingSqlOperations sqlOperations) throws Exception { // verify we have permissions to create/drop stage final String outputTableName = namingResolver.getIdentifier("_airbyte_connection_test_" + UUID.randomUUID().toString().replaceAll("-", "")); - final String stageName = namingResolver.getStageName(outputSchema, outputTableName);; + final String stageName = namingResolver.getStageName(outputSchema, outputTableName); + ; sqlOperations.createStageIfNotExists(database, stageName); sqlOperations.dropStageIfExists(database, stageName); } @@ -64,6 +67,11 @@ protected JdbcDatabase getDatabase(final JsonNode config) { return SnowflakeDatabase.getDatabase(config); } + @Override + protected Map getConnectionProperties(final JsonNode config) { + return new HashMap<>(); + } + // this is a no op since we override getDatabase. @Override public JsonNode toJdbcConfig(final JsonNode config) { @@ -72,8 +80,8 @@ public JsonNode toJdbcConfig(final JsonNode config) { @Override public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) { + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) { return new SnowflakeInternalStagingConsumerFactory().create(outputRecordCollector, getDatabase(config), new SnowflakeStagingSqlOperations(), new SnowflakeSQLNameTransformer(), config, catalog); } From 92fbf815758a9b6b48aeee55aa602f05ceda50d6 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 14:52:10 -0800 Subject: [PATCH 04/60] A little more generic --- .../clickhouse/ClickhouseDestination.java | 2 +- .../jdbc/AbstractJdbcDestination.java | 47 +++++++++++++------ .../MariadbColumnstoreDestination.java | 2 +- .../destination/mysql/MySQLDestination.java | 26 ++-------- .../mysql/MySQLDestinationTest.java | 8 +++- .../destination/oracle/OracleDestination.java | 2 +- .../postgres/PostgresDestination.java | 2 +- .../redshift/RedshiftInsertDestination.java | 2 +- .../snowflake/SnowflakeInsertDestination.java | 2 +- .../SnowflakeInternalStagingDestination.java | 2 +- 10 files changed, 49 insertions(+), 46 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index bdd47de55382d..260187abacaf8 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -86,7 +86,7 @@ public AirbyteConnectionStatus check(final JsonNode config) { } @Override - protected Map getConnectionProperties(final JsonNode config) { + protected Map getDefaultConnectionProperties(final JsonNode config) { return new HashMap<>(); } diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java index b588053ec46b8..7b0ed57c9f6d8 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Maps; +import io.airbyte.commons.map.MoreMaps; import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; @@ -20,6 +21,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.UUID; import java.util.function.Consumer; import org.slf4j.Logger; @@ -29,6 +31,8 @@ public abstract class AbstractJdbcDestination extends BaseConnector implements D private static final Logger LOGGER = LoggerFactory.getLogger(AbstractJdbcDestination.class); + public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params"; + private final String driverClass; private final NamingConventionTransformer namingResolver; private final SqlOperations sqlOperations; @@ -92,27 +96,22 @@ protected JdbcDatabase getDatabase(final JsonNode config) { getConnectionProperties(config)); } - protected abstract Map getConnectionProperties(final JsonNode config); - - public abstract JsonNode toJdbcConfig(JsonNode config); - - @Override - public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) { - return JdbcBufferedConsumerFactory.create(outputRecordCollector, getDatabase(config), sqlOperations, namingResolver, config, catalog); + protected Map getConnectionProperties(final JsonNode config) { + final Map customProperties = parseJdbcParameters(config); + final Map defaultProperties = getDefaultConnectionProperties(config); + assertCustomParametersDontOverwriteDefaultParameters(customProperties, defaultProperties); + return MoreMaps.merge(customProperties, defaultProperties); } - - public static Map parseJdbcParameters(final JsonNode config, final String jdbcParametersKey) { - if (config.has(jdbcParametersKey)) { - return parseJdbcParameters(config.get(jdbcParametersKey).asText()); + private Map parseJdbcParameters(final JsonNode config) { + if (config.has(JDBC_URL_PARAMS_KEY)) { + return parseJdbcParameters(config.get(JDBC_URL_PARAMS_KEY).asText()); } else { return Maps.newHashMap(); } } - public static Map parseJdbcParameters(final String jdbcPropertiesString) { + private Map parseJdbcParameters(final String jdbcPropertiesString) { final Map parameters = new HashMap<>(); if (!jdbcPropertiesString.isBlank()) { final String[] keyValuePairs = jdbcPropertiesString.split("&"); @@ -129,4 +128,24 @@ public static Map parseJdbcParameters(final String jdbcPropertie } return parameters; } + + private void assertCustomParametersDontOverwriteDefaultParameters(final Map customParameters, + final Map defaultParameters) { + for (final String key : defaultParameters.keySet()) { + if (customParameters.containsKey(key) && !Objects.equals(customParameters.get(key), defaultParameters.get(key))) { + throw new IllegalArgumentException("Cannot overwrite default JDBC parameter " + key); + } + } + } + + protected abstract Map getDefaultConnectionProperties(final JsonNode config); + + public abstract JsonNode toJdbcConfig(JsonNode config); + + @Override + public AirbyteMessageConsumer getConsumer(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) { + return JdbcBufferedConsumerFactory.create(outputRecordCollector, getDatabase(config), sqlOperations, namingResolver, config, catalog); + } } diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java index e3eaed60b5770..441eb2147c5c8 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java @@ -70,7 +70,7 @@ public AirbyteConnectionStatus check(final JsonNode config) { } @Override - protected Map getConnectionProperties(final JsonNode config) { + protected Map getDefaultConnectionProperties(final JsonNode config) { return DEFAULT_JDBC_PARAMETERS; } diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java index 7e1093810d06a..6ac9d437dd362 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java @@ -18,7 +18,6 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; import java.util.List; import java.util.Map; -import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -93,19 +92,13 @@ protected JdbcDatabase getDatabase(final JsonNode config) { return super.getDatabase(config); } - @Override - protected Map getConnectionProperties(final JsonNode config) { - final Map customProperties = parseJdbcParameters(config, JDBC_URL_PARAMS_KEY); - final Map defaultProperties = getDefaultConnectionProperties(config); - assertCustomParametersDontOverwriteDefaultParameters(customProperties, defaultProperties); - return MoreMaps.merge(customProperties, defaultProperties); - } - private Map getDefaultConnectionProperties(final JsonNode config) { + @Override + protected Map getDefaultConnectionProperties(final JsonNode config) { if (useSSL(config)) { return MoreMaps.merge(DEFAULT_JDBC_PARAMETERS, SSL_JDBC_PARAMETERS); } else { - return MoreMaps.merge(DEFAULT_JDBC_PARAMETERS); + return DEFAULT_JDBC_PARAMETERS; } } @@ -113,15 +106,6 @@ private boolean useSSL(final JsonNode config) { return !config.has(SSL_KEY) || config.get(SSL_KEY).asBoolean(); } - private void assertCustomParametersDontOverwriteDefaultParameters(final Map customParameters, - final Map defaultParameters) { - for (final String key : defaultParameters.keySet()) { - if (customParameters.containsKey(key) && !Objects.equals(customParameters.get(key), defaultParameters.get(key))) { - throw new IllegalArgumentException("Cannot overwrite default JDBC parameter " + key); - } - } - } - @Override public JsonNode toJdbcConfig(final JsonNode config) { final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:mysql://%s:%s/%s", @@ -143,10 +127,6 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } - static String formatParameter(final String key, final String value) { - return String.format("%s=%s", key, value); - } - public static void main(final String[] args) throws Exception { final Destination destination = MySQLDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", MySQLDestination.class); diff --git a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java index ea12c3864f013..17755e768cfca 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java @@ -111,8 +111,8 @@ void testExtraParamsWithDefaultParameter() { final Map allDefaultParameters = MoreMaps.merge(MySQLDestination.SSL_JDBC_PARAMETERS, MySQLDestination.DEFAULT_JDBC_PARAMETERS); for (final Entry entry : allDefaultParameters.entrySet()) { - final String identicalParameter = MySQLDestination.formatParameter(entry.getKey(), entry.getValue()); - final String overridingParameter = MySQLDestination.formatParameter(entry.getKey(), "DIFFERENT_VALUE"); + final String identicalParameter = formatParameter(entry.getKey(), entry.getValue()); + final String overridingParameter = formatParameter(entry.getKey(), "DIFFERENT_VALUE"); // Do not throw an exception if the values are equal assertDoesNotThrow(() -> @@ -171,4 +171,8 @@ void verifyJdbcDatabaseIsCreatedWithConnectionProperties(final JsonNode jsonNode times(1)); } } + + String formatParameter(final String key, final String value) { + return String.format("%s=%s", key, value); + } } diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index 415c8aec2ff0b..142a85348608a 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -56,7 +56,7 @@ public static Destination sshWrappedDestination() { } @Override - protected Map getConnectionProperties(final JsonNode config) { + protected Map getDefaultConnectionProperties(final JsonNode config) { return new HashMap<>(); } diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java index 589079044c0f0..709f7d68744b6 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java @@ -36,7 +36,7 @@ public PostgresDestination() { } @Override - protected Map getConnectionProperties(final JsonNode config) { + protected Map getDefaultConnectionProperties(final JsonNode config) { return new HashMap<>(); } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java index 25f1f76431ad6..e0b550f405016 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java @@ -43,7 +43,7 @@ public JdbcDatabase getDatabase(final JsonNode config) { } @Override - protected Map getConnectionProperties(final JsonNode config) { + protected Map getDefaultConnectionProperties(final JsonNode config) { return new HashMap<>(); } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java index daf2b81b97502..4bcf4ec5142e2 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java @@ -29,7 +29,7 @@ protected JdbcDatabase getDatabase(final JsonNode config) { } @Override - protected Map getConnectionProperties(final JsonNode config) { + protected Map getDefaultConnectionProperties(final JsonNode config) { return new HashMap<>(); } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index 8a3841c013356..1e419cb163bd1 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -68,7 +68,7 @@ protected JdbcDatabase getDatabase(final JsonNode config) { } @Override - protected Map getConnectionProperties(final JsonNode config) { + protected Map getDefaultConnectionProperties(final JsonNode config) { return new HashMap<>(); } From 34698263f10832bf3b54f9f55743b9ed98521199 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 14:53:53 -0800 Subject: [PATCH 05/60] reset to master --- airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index 02d13fec795f6..a7d681857aae6 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -23,4 +23,5 @@ public static JSONFormat getDefaultJSONFormat() { public static String getFullyQualifiedTableName(final String schemaName, final String tableName) { return schemaName != null ? schemaName + "." + tableName : tableName; } + } From 66ea4f707dcd03f1078982ee08ee5c3723328e55 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 14:55:07 -0800 Subject: [PATCH 06/60] reset to master --- build.gradle | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/build.gradle b/build.gradle index 3b83e1002f85f..79af04f23bc77 100644 --- a/build.gradle +++ b/build.gradle @@ -71,25 +71,25 @@ def createJavaLicenseWith = { license -> // monorepo setup and it doesn't actually exclude directories reliably. This code makes the behavior predictable. def createSpotlessTarget = { pattern -> def excludes = [ - '.gradle', - 'node_modules', - '.eggs', - '.mypy_cache', - '.venv', - '*.egg-info', - 'build', - 'dbt-project-template', - 'dbt-project-template-mssql', - 'dbt-project-template-mysql', - 'dbt-project-template-oracle', - 'dbt-project-template-clickhouse', - 'dbt-project-template-snowflake', - 'dbt_test_config', - 'normalization_test_output', - 'tools', - 'secrets', - 'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. - 'resources/seed/*_specs.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file. + '.gradle', + 'node_modules', + '.eggs', + '.mypy_cache', + '.venv', + '*.egg-info', + 'build', + 'dbt-project-template', + 'dbt-project-template-mssql', + 'dbt-project-template-mysql', + 'dbt-project-template-oracle', + 'dbt-project-template-clickhouse', + 'dbt-project-template-snowflake', + 'dbt_test_config', + 'normalization_test_output', + 'tools', + 'secrets', + 'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. + 'resources/seed/*_specs.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file. ] if (System.getenv().containsKey("SUB_BUILD")) { @@ -129,7 +129,7 @@ check.dependsOn 'spotlessApply' @SuppressWarnings('GroovyAssignabilityCheck') def Task getDockerBuildTask(String artifactName, String projectDir) { - return task("buildDockerImage-$artifactName"(type: DockerBuildImage) { + return task ("buildDockerImage-$artifactName"(type: DockerBuildImage) { def buildTag = System.getenv('VERSION') ?: 'dev' def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64' def alpineImage = System.getenv('ALPINE_IMAGE') ?: 'alpine:3.4' From b9231bbd92ce4e79fb2c10e13bb2fe62bea72ec9 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 14:59:11 -0800 Subject: [PATCH 07/60] move to jdbcutils --- .../java/io/airbyte/db/jdbc/JdbcUtils.java | 30 +++++++++++++++++++ .../jdbc/AbstractJdbcDestination.java | 30 +------------------ .../source/jdbc/AbstractJdbcSource.java | 5 ++-- 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index a7d681857aae6..05caf59336b02 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -4,6 +4,10 @@ package io.airbyte.db.jdbc; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Maps; +import java.util.HashMap; +import java.util.Map; import org.jooq.JSONFormat; public class JdbcUtils { @@ -24,4 +28,30 @@ public static String getFullyQualifiedTableName(final String schemaName, final S return schemaName != null ? schemaName + "." + tableName : tableName; } + public static Map parseJdbcParameters(final JsonNode config, final String jdbcUrlParamsKey) { + if (config.has(jdbcUrlParamsKey)) { + return parseJdbcParameters(config.get(jdbcUrlParamsKey).asText()); + } else { + return Maps.newHashMap(); + } + } + + public static Map parseJdbcParameters(final String jdbcPropertiesString) { + final Map parameters = new HashMap<>(); + if (!jdbcPropertiesString.isBlank()) { + final String[] keyValuePairs = jdbcPropertiesString.split("&"); + for (final String kv : keyValuePairs) { + final String[] split = kv.split("="); + if (split.length == 2) { + parameters.put(split[0], split[1]); + } else { + throw new IllegalArgumentException( + "jdbc_url_params must be formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). Got " + + jdbcPropertiesString); + } + } + } + return parameters; + } + } diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java index 7b0ed57c9f6d8..d62abcac6903a 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -5,7 +5,6 @@ package io.airbyte.integrations.destination.jdbc; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Maps; import io.airbyte.commons.map.MoreMaps; import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; @@ -19,7 +18,6 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.UUID; @@ -97,38 +95,12 @@ protected JdbcDatabase getDatabase(final JsonNode config) { } protected Map getConnectionProperties(final JsonNode config) { - final Map customProperties = parseJdbcParameters(config); + final Map customProperties = JdbcUtils.parseJdbcParameters(config, JDBC_URL_PARAMS_KEY); final Map defaultProperties = getDefaultConnectionProperties(config); assertCustomParametersDontOverwriteDefaultParameters(customProperties, defaultProperties); return MoreMaps.merge(customProperties, defaultProperties); } - private Map parseJdbcParameters(final JsonNode config) { - if (config.has(JDBC_URL_PARAMS_KEY)) { - return parseJdbcParameters(config.get(JDBC_URL_PARAMS_KEY).asText()); - } else { - return Maps.newHashMap(); - } - } - - private Map parseJdbcParameters(final String jdbcPropertiesString) { - final Map parameters = new HashMap<>(); - if (!jdbcPropertiesString.isBlank()) { - final String[] keyValuePairs = jdbcPropertiesString.split("&"); - for (final String kv : keyValuePairs) { - final String[] split = kv.split("="); - if (split.length == 2) { - parameters.put(split[0], split[1]); - } else { - throw new IllegalArgumentException( - "jdbc_url_params must be formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). Got " - + jdbcPropertiesString); - } - } - } - return parameters; - } - private void assertCustomParametersDontOverwriteDefaultParameters(final Map customParameters, final Map defaultParameters) { for (final String key : defaultParameters.keySet()) { diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index 613ef63d21dba..4953f6c5e6b63 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -69,8 +69,6 @@ public abstract class AbstractJdbcSource extends AbstractRelationalDbS protected String quoteString; - public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params"; - public AbstractJdbcSource(final String driverClass, final JdbcStreamingQueryConfiguration jdbcStreamingQueryConfiguration, final JdbcCompatibleSourceOperations sourceOperations) { @@ -294,11 +292,12 @@ public JdbcDatabase createDatabase(final JsonNode config) throws SQLException { jdbcConfig.get("jdbc_url").asText(), driverClass, jdbcStreamingQueryConfiguration, - JdbcUtils.parseJdbcParameters(config.get("connection_properties").asText()), + JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties"), sourceOperations); quoteString = (quoteString == null ? database.getMetaData().getIdentifierQuoteString() : quoteString); return database; } + } From ec5631168275bb82f323780c720bc62cf94c366e Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 15:03:35 -0800 Subject: [PATCH 08/60] Align when multiline --- .../main/java/io/airbyte/db/Databases.java | 90 +++++++++---------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java index b095a429dfd87..2a628cc01abf3 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java @@ -33,9 +33,9 @@ public static Database createPostgresDatabase(final String username, final Strin } public static Database createPostgresDatabaseWithRetry(final String username, - final String password, - final String jdbcConnectionString, - final Function isDbReady) { + final String password, + final String jdbcConnectionString, + final Function isDbReady) { Database database = null; while (database == null) { try { @@ -51,10 +51,10 @@ public static Database createPostgresDatabaseWithRetry(final String username, } public static Database createPostgresDatabaseWithRetryTimeout(final String username, - final String password, - final String jdbcConnectionString, - final Function isDbReady, - final long timeoutMs) + final String password, + final String jdbcConnectionString, + final Function isDbReady, + final long timeoutMs) throws IOException { Database database = null; if (jdbcConnectionString == null || jdbcConnectionString.trim().equals("")) { @@ -117,21 +117,21 @@ public static Database createMariaDbDatabase(final String username, final String } public static Database createDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final SQLDialect dialect) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final SQLDialect dialect) { final BasicDataSource connectionPool = createBasicDataSource(username, password, jdbcConnectionString, driverClassName); return new Database(connectionPool, dialect); } public static Database createDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final SQLDialect dialect, - final Map connectionProperties) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final SQLDialect dialect, + final Map connectionProperties) { final BasicDataSource connectionPool = createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); @@ -139,37 +139,37 @@ public static Database createDatabase(final String username, } public static JdbcDatabase createJdbcDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName) { + final String password, + final String jdbcConnectionString, + final String driverClassName) { return createJdbcDatabase(username, password, jdbcConnectionString, driverClassName, JdbcUtils.getDefaultSourceOperations()); } public static JdbcDatabase createJdbcDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final JdbcSourceOperations sourceOperations) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final JdbcSourceOperations sourceOperations) { final BasicDataSource connectionPool = createBasicDataSource(username, password, jdbcConnectionString, driverClassName); return new DefaultJdbcDatabase(connectionPool, sourceOperations); } public static JdbcDatabase createJdbcDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final Map connectionProperties) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final Map connectionProperties) { return createJdbcDatabase(username, password, jdbcConnectionString, driverClassName, connectionProperties, JdbcUtils.getDefaultSourceOperations()); } public static JdbcDatabase createJdbcDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final Map connectionProperties, - final JdbcCompatibleSourceOperations sourceOperations) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final Map connectionProperties, + final JdbcCompatibleSourceOperations sourceOperations) { final BasicDataSource connectionPool = createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); @@ -177,12 +177,12 @@ public static JdbcDatabase createJdbcDatabase(final String username, } public static JdbcDatabase createStreamingJdbcDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final JdbcStreamingQueryConfiguration jdbcStreamingQuery, - final Map connectionProperties, - final JdbcCompatibleSourceOperations sourceOperations) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final JdbcStreamingQueryConfiguration jdbcStreamingQuery, + final Map connectionProperties, + final JdbcCompatibleSourceOperations sourceOperations) { final BasicDataSource connectionPool = createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); @@ -190,18 +190,18 @@ public static JdbcDatabase createStreamingJdbcDatabase(final String username, } private static BasicDataSource createBasicDataSource(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName) { + final String password, + final String jdbcConnectionString, + final String driverClassName) { return createBasicDataSource(username, password, jdbcConnectionString, driverClassName, Maps.newHashMap()); } public static BasicDataSource createBasicDataSource(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final Map connectionProperties) { + final String password, + final String jdbcConnectionString, + final String driverClassName, + final Map connectionProperties) { final BasicDataSource connectionPool = new BasicDataSource(); connectionPool.setDriverClassName(driverClassName); connectionPool.setUsername(username); From 889aaf08a40eb09e71a47044e7d22a2abd3904d3 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 15:10:09 -0800 Subject: [PATCH 09/60] Align when multiline --- .../jdbc/AbstractJdbcDestination.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java index d62abcac6903a..6a81e1dcb4cac 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -44,8 +44,8 @@ protected SqlOperations getSqlOperations() { } public AbstractJdbcDestination(final String driverClass, - final NamingConventionTransformer namingResolver, - final SqlOperations sqlOperations) { + final NamingConventionTransformer namingResolver, + final SqlOperations sqlOperations) { this.driverClass = driverClass; this.namingResolver = namingResolver; this.sqlOperations = sqlOperations; @@ -68,9 +68,9 @@ public AirbyteConnectionStatus check(final JsonNode config) { } public static void attemptSQLCreateAndDropTableOperations(final String outputSchema, - final JdbcDatabase database, - final NamingConventionTransformer namingResolver, - final SqlOperations sqlOps) + final JdbcDatabase database, + final NamingConventionTransformer namingResolver, + final SqlOperations sqlOps) throws Exception { // attempt to get metadata from the database as a cheap way of seeing if we can connect. database.bufferedResultSetQuery(conn -> conn.getMetaData().getCatalogs(), JdbcUtils.getDefaultSourceOperations()::rowToJson); @@ -102,7 +102,7 @@ protected Map getConnectionProperties(final JsonNode config) { } private void assertCustomParametersDontOverwriteDefaultParameters(final Map customParameters, - final Map defaultParameters) { + final Map defaultParameters) { for (final String key : defaultParameters.keySet()) { if (customParameters.containsKey(key) && !Objects.equals(customParameters.get(key), defaultParameters.get(key))) { throw new IllegalArgumentException("Cannot overwrite default JDBC parameter " + key); @@ -116,8 +116,8 @@ private void assertCustomParametersDontOverwriteDefaultParameters(final Map outputRecordCollector) { + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) { return JdbcBufferedConsumerFactory.create(outputRecordCollector, getDatabase(config), sqlOperations, namingResolver, config, catalog); } } From 316e41940b8d357819c23f5342e156d8c7d8702a Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 15:16:18 -0800 Subject: [PATCH 10/60] Update postgres to use property map --- .../destination/mysql/MySQLDestination.java | 6 ++-- .../postgres/PostgresDestination.java | 33 ++++++++++--------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java index 6ac9d437dd362..6f15b62e851ae 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java @@ -108,14 +108,14 @@ private boolean useSSL(final JsonNode config) { @Override public JsonNode toJdbcConfig(final JsonNode config) { - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:mysql://%s:%s/%s", + final String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s", config.get(HOST_KEY).asText(), config.get(PORT_KEY).asText(), - config.get(DATABASE_KEY).asText())); + config.get(DATABASE_KEY).asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put(USERNAME_KEY, config.get(USERNAME_KEY).asText()) - .put(JDBC_URL_KEY, jdbcUrl.toString()); + .put(JDBC_URL_KEY, jdbcUrl); if (config.has(PASSWORD_KEY)) { configBuilder.put(PASSWORD_KEY, config.get(PASSWORD_KEY).asText()); diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java index 709f7d68744b6..b2c571d5975f4 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java @@ -11,7 +11,6 @@ import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.ssh.SshWrappedDestination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,6 +26,11 @@ public class PostgresDestination extends AbstractJdbcDestination implements Dest public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); + static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( + "ssl", "true", + "sslmode", "require" + ); + public static Destination sshWrappedDestination() { return new SshWrappedDestination(new PostgresDestination(), HOST_KEY, PORT_KEY); } @@ -37,32 +41,25 @@ public PostgresDestination() { @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - return new HashMap<>(); + if (useSsl(config)) { + return SSL_JDBC_PARAMETERS; + } else { + return new HashMap<>(); + } } @Override public JsonNode toJdbcConfig(final JsonNode config) { final String schema = Optional.ofNullable(config.get("schema")).map(JsonNode::asText).orElse("public"); - final List additionalParameters = new ArrayList<>(); - - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:postgresql://%s:%s/%s?", + final String jdbcUrl = String.format("jdbc:postgresql://%s:%s/%s?", config.get("host").asText(), config.get("port").asText(), - config.get("database").asText())); - - if (!config.has("ssl") || config.get("ssl").asBoolean()) { - additionalParameters.add("ssl=true"); - additionalParameters.add("sslmode=require"); - } - - if (!additionalParameters.isEmpty()) { - additionalParameters.forEach(x -> jdbcUrl.append(x).append("&")); - } + config.get("database").asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) - .put("jdbc_url", jdbcUrl.toString()) + .put("jdbc_url", jdbcUrl) .put("schema", schema); if (config.has("password")) { @@ -71,6 +68,10 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } + private boolean useSsl(final JsonNode config) { + return !config.has("ssl") || config.get("ssl").asBoolean(); + } + public static void main(final String[] args) throws Exception { final Destination destination = PostgresDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", PostgresDestination.class); From fd034af4d7fdf8b8c0f8a5f5f884a0392176a0a8 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 16:06:51 -0800 Subject: [PATCH 11/60] Move tests to AbstractJdbcDestinationTest --- .../jdbc/AbstractJdbcDestinationTest.java | 135 ++++++++++++++++++ .../mysql/MySQLDestinationTest.java | 129 +++-------------- 2 files changed, 156 insertions(+), 108 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java diff --git a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java new file mode 100644 index 0000000000000..f51dae130563c --- /dev/null +++ b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java @@ -0,0 +1,135 @@ +package io.airbyte.integrations.destination.jdbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.StandardNameTransformer; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +public class AbstractJdbcDestinationTest { + + private JsonNode buildConfigNoJdbcParameters() { + final JsonNode config = Jsons.jsonNode(ImmutableMap.of( + "host", "localhost", + "port", 1337, + "username", "user", + "database", "db" + )); + return config; + } + + private JsonNode buildConfigWithExtraJdbcParameters(final String extraParam) { + final JsonNode config = Jsons.jsonNode(ImmutableMap.of( + "host", "localhost", + "port", 1337, + "username", "user", + "database", "db", + "jdbc_url_params", extraParam + )); + return config; + } + + @Test + void testNoExtraParamsNoDefault() { + final Map connectionProperties = new TestJdbcDestination().getConnectionProperties(buildConfigNoJdbcParameters()); + + final Map expectedProperties = ImmutableMap.of(); + assertEquals(expectedProperties, connectionProperties); + } + + @Test + void testNoExtraParamsWithDefault() { + final Map defaultProperties = ImmutableMap.of("A_PARAMETER", "A_VALUE"); + + final Map connectionProperties = new TestJdbcDestination(defaultProperties).getConnectionProperties( + buildConfigNoJdbcParameters()); + + assertEquals(defaultProperties, connectionProperties); + } + + @Test + void testExtraParamNoDefault() { + final String extraParam = "key1=value1&key2=value2&key3=value3"; + final Map connectionProperties = new TestJdbcDestination().getConnectionProperties( + buildConfigWithExtraJdbcParameters(extraParam)); + final Map expectedProperties = ImmutableMap.of( + "key1", "value1", + "key2", "value2", + "key3", "value3"); + assertEquals(expectedProperties, connectionProperties); + } + + @Test + void testExtraParamWithDefault() { + final Map defaultProperties = ImmutableMap.of("A_PARAMETER", "A_VALUE"); + final String extraParam = "key1=value1&key2=value2&key3=value3"; + final Map connectionProperties = new TestJdbcDestination(defaultProperties).getConnectionProperties( + buildConfigWithExtraJdbcParameters(extraParam)); + final Map expectedProperties = ImmutableMap.of( + "A_PARAMETER", "A_VALUE", + "key1", "value1", + "key2", "value2", + "key3", "value3"); + assertEquals(expectedProperties, connectionProperties); + } + + @Test + void testExtraParameterEqualToDefault() { + final Map defaultProperties = ImmutableMap.of("key1", "value1"); + final String extraParam = "key1=value1&key2=value2&key3=value3"; + final Map connectionProperties = new TestJdbcDestination(defaultProperties).getConnectionProperties( + buildConfigWithExtraJdbcParameters(extraParam)); + final Map expectedProperties = ImmutableMap.of( + "key1", "value1", + "key2", "value2", + "key3", "value3"); + assertEquals(expectedProperties, connectionProperties); + } + + @Test + void testExtraParameterDiffersFromDefault() { + final Map defaultProperties = ImmutableMap.of("key1", "value0"); + final String extraParam = "key1=value1&key2=value2&key3=value3"; + + assertThrows(IllegalArgumentException.class, () -> + new TestJdbcDestination(defaultProperties).getConnectionProperties( + buildConfigWithExtraJdbcParameters(extraParam)) + ); + } + + @Test + void testInvalidExtraParam() { + final String extraParam = "key1=value1&sdf&"; + assertThrows(IllegalArgumentException.class, + () -> new TestJdbcDestination().getConnectionProperties(buildConfigWithExtraJdbcParameters(extraParam))); + } + + static class TestJdbcDestination extends AbstractJdbcDestination { + + private final Map defaultProperties; + + public TestJdbcDestination() { + this(new HashMap<>()); + } + + public TestJdbcDestination(final Map defaultProperties) { + super("", new StandardNameTransformer(), new TestJdbcSqlOperations()); + this.defaultProperties = defaultProperties; + } + + @Override + protected Map getDefaultConnectionProperties(final JsonNode config) { + return defaultProperties; + } + + @Override + public JsonNode toJdbcConfig(final JsonNode config) { + return config; + } + } +} diff --git a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java index 17755e768cfca..9a1f1121df3ce 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java @@ -1,22 +1,14 @@ package io.airbyte.integrations.destination.mysql; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Mockito.times; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.Databases; import java.util.Map; -import java.util.Map.Entry; +import org.junit.Before; import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; -import org.mockito.Mockito; public class MySQLDestinationTest { @@ -24,16 +16,9 @@ public class MySQLDestinationTest { MySQLDestination.DEFAULT_JDBC_PARAMETERS, MySQLDestination.SSL_JDBC_PARAMETERS ); - private static final Map CUSTOM_PROPERTIES = ImmutableMap.of( - "key1", "value1", - "key2", "value2", - "key3", "value3" - ); - private MySQLDestination getDestination() { - final MySQLDestination result = new MySQLDestination(); - return result; - } + public static final String JDBC_URL = "jdbc:mysql://localhost:1337/db"; + private MySQLDestination destination; private JsonNode buildConfigNoJdbcParameters() { final JsonNode config = Jsons.jsonNode(ImmutableMap.of( @@ -56,18 +41,6 @@ private JsonNode buildConfigWithExtraJdbcParameters(final String extraParam) { return config; } - private JsonNode buildConfigWithExtraJdbcParametersWithNoSsl(final String extraParam) { - final JsonNode config = Jsons.jsonNode(ImmutableMap.of( - "host", "localhost", - "port", 1337, - "username", "user", - "database", "db", - "ssl", false, - "jdbc_url_params", extraParam - )); - return config; - } - private JsonNode buildConfigNoExtraJdbcParametersWithoutSsl() { final JsonNode config = Jsons.jsonNode(ImmutableMap.of( "host", "localhost", @@ -79,100 +52,40 @@ private JsonNode buildConfigNoExtraJdbcParametersWithoutSsl() { return config; } + @Before + public void setUp() { + destination = new MySQLDestination(); + } + @Test void testNoExtraParams() { - final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigNoJdbcParameters()); - final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals("jdbc:mysql://localhost:1337/db", url); - verifyJdbcDatabaseIsCreatedWithConnectionProperties(buildConfigNoJdbcParameters(), DEFAULT_PARAMETERS_WITH_SSL); + final JsonNode config = buildConfigNoJdbcParameters(); + final JsonNode jdbcConfig = destination.toJdbcConfig(config); + assertEquals(JDBC_URL, jdbcConfig.get("jdbc_url").asText()); } @Test void testEmptyExtraParams() { - final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters("")); - final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals("jdbc:mysql://localhost:1337/db", url); - verifyJdbcDatabaseIsCreatedWithConnectionProperties(buildConfigNoJdbcParameters(), DEFAULT_PARAMETERS_WITH_SSL); + final JsonNode jdbcConfig = destination.toJdbcConfig(buildConfigWithExtraJdbcParameters("")); + assertEquals(JDBC_URL, jdbcConfig.get("jdbc_url").asText()); } @Test void testExtraParams() { final String extraParam = "key1=value1&key2=value2&key3=value3"; - final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam)); - final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals("jdbc:mysql://localhost:1337/db", url); - - verifyJdbcDatabaseIsCreatedWithConnectionProperties(buildConfigWithExtraJdbcParameters(extraParam), - MoreMaps.merge(CUSTOM_PROPERTIES, DEFAULT_PARAMETERS_WITH_SSL)); - } - - @Test - void testExtraParamsWithDefaultParameter() { - final Map allDefaultParameters = MoreMaps.merge(MySQLDestination.SSL_JDBC_PARAMETERS, - MySQLDestination.DEFAULT_JDBC_PARAMETERS); - for (final Entry entry : allDefaultParameters.entrySet()) { - final String identicalParameter = formatParameter(entry.getKey(), entry.getValue()); - final String overridingParameter = formatParameter(entry.getKey(), "DIFFERENT_VALUE"); - - // Do not throw an exception if the values are equal - assertDoesNotThrow(() -> - getDestination().getDatabase(buildConfigWithExtraJdbcParameters(identicalParameter)) - ); - final Map connectionProperties = MoreMaps.merge( - ImmutableMap.of(entry.getKey(), entry.getValue()), - MySQLDestination.DEFAULT_JDBC_PARAMETERS - ); - verifyJdbcDatabaseIsCreatedWithConnectionProperties(buildConfigWithExtraJdbcParametersWithNoSsl(identicalParameter), connectionProperties); - - // Throw an exception if the values are different - assertThrows(IllegalArgumentException.class, () -> - getDestination().getDatabase(buildConfigWithExtraJdbcParameters(overridingParameter)) - ); - } - } - - @Test - void testExtraParameterNoSsl() { - final String extraParam = "key1=value1&key2=value2&key3=value3"; - final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigWithExtraJdbcParametersWithNoSsl(extraParam)); - final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals("jdbc:mysql://localhost:1337/db", url); - - final Map connectionProperties = ImmutableMap.of( - "key1", "value1", - "key2", "value2", - "key3", "value3"); - verifyJdbcDatabaseIsCreatedWithConnectionProperties(buildConfigWithExtraJdbcParametersWithNoSsl(extraParam), - MoreMaps.merge(connectionProperties, MySQLDestination.DEFAULT_JDBC_PARAMETERS)); + final JsonNode jdbcConfig = destination.toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam)); + assertEquals(JDBC_URL, jdbcConfig.get("jdbc_url").asText()); } @Test - void testNoExtraParameterNoSsl() { - final JsonNode jdbcConfig = getDestination().toJdbcConfig(buildConfigNoExtraJdbcParametersWithoutSsl()); - final String url = jdbcConfig.get("jdbc_url").asText(); - assertEquals("jdbc:mysql://localhost:1337/db", url); - - verifyJdbcDatabaseIsCreatedWithConnectionProperties(buildConfigNoExtraJdbcParametersWithoutSsl(), MySQLDestination.DEFAULT_JDBC_PARAMETERS); + void testDefaultParamsNoSSL() { + final Map defaultProperties = destination.getDefaultConnectionProperties(buildConfigNoExtraJdbcParametersWithoutSsl()); + assertEquals(MySQLDestination.DEFAULT_JDBC_PARAMETERS, defaultProperties); } @Test - void testInvalidExtraParam() { - final String extraParam = "key1=value1&sdf&"; - assertThrows(IllegalArgumentException.class, () -> { - getDestination().getDatabase(buildConfigWithExtraJdbcParameters(extraParam)); - }); - } - - void verifyJdbcDatabaseIsCreatedWithConnectionProperties(final JsonNode jsonNode, final Map connectionProperties) { - try (final MockedStatic databases = Mockito.mockStatic(Databases.class)) { - getDestination().getDatabase(jsonNode); - databases.verify(() -> Databases.createJdbcDatabase( - anyString(), nullable(String.class), anyString(), anyString(), Mockito.eq(connectionProperties)), - times(1)); - } - } - - String formatParameter(final String key, final String value) { - return String.format("%s=%s", key, value); + void testDefaultParamsWithSSL() { + final Map defaultProperties = destination.getDefaultConnectionProperties(buildConfigNoJdbcParameters()); + assertEquals(DEFAULT_PARAMETERS_WITH_SSL, defaultProperties); } } From 5e1bcfc59684287c48f1c3f07d6f1f28b63813cb Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 16:07:27 -0800 Subject: [PATCH 12/60] clean --- .../destination/jdbc/AbstractJdbcDestinationTest.java | 6 ++---- .../destination/mysql/MySQLDestinationTest.java | 9 +++------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java index f51dae130563c..a8567b3940d44 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java @@ -14,24 +14,22 @@ public class AbstractJdbcDestinationTest { private JsonNode buildConfigNoJdbcParameters() { - final JsonNode config = Jsons.jsonNode(ImmutableMap.of( + return Jsons.jsonNode(ImmutableMap.of( "host", "localhost", "port", 1337, "username", "user", "database", "db" )); - return config; } private JsonNode buildConfigWithExtraJdbcParameters(final String extraParam) { - final JsonNode config = Jsons.jsonNode(ImmutableMap.of( + return Jsons.jsonNode(ImmutableMap.of( "host", "localhost", "port", 1337, "username", "user", "database", "db", "jdbc_url_params", extraParam )); - return config; } @Test diff --git a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java index 9a1f1121df3ce..4ca7a9049e7f1 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java @@ -21,35 +21,32 @@ public class MySQLDestinationTest { private MySQLDestination destination; private JsonNode buildConfigNoJdbcParameters() { - final JsonNode config = Jsons.jsonNode(ImmutableMap.of( + return Jsons.jsonNode(ImmutableMap.of( "host", "localhost", "port", 1337, "username", "user", "database", "db" )); - return config; } private JsonNode buildConfigWithExtraJdbcParameters(final String extraParam) { - final JsonNode config = Jsons.jsonNode(ImmutableMap.of( + return Jsons.jsonNode(ImmutableMap.of( "host", "localhost", "port", 1337, "username", "user", "database", "db", "jdbc_url_params", extraParam )); - return config; } private JsonNode buildConfigNoExtraJdbcParametersWithoutSsl() { - final JsonNode config = Jsons.jsonNode(ImmutableMap.of( + return Jsons.jsonNode(ImmutableMap.of( "host", "localhost", "port", 1337, "username", "user", "database", "db", "ssl", false )); - return config; } @Before From 5dd0591dc96a1806e8f21e1d51d8d56c768da5fc Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 16:11:42 -0800 Subject: [PATCH 13/60] Align when multiline --- .../integrations/destination/oracle/OracleDestination.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index 142a85348608a..a98fd39c357c3 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -91,7 +91,7 @@ public JsonNode toJdbcConfig(final JsonNode config) { } private Protocol obtainConnectionProtocol(final JsonNode encryption, - final List additionalParameters) { + final List additionalParameters) { final String encryptionMethod = encryption.get("encryption_method").asText(); switch (encryptionMethod) { case "unencrypted" -> { From b69363da4eb169196c65231a8648c01dfae9d958 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 16:15:58 -0800 Subject: [PATCH 14/60] return property map --- .../destination/redshift/RedshiftInsertDestination.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java index e0b550f405016..cae137feb00ce 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java @@ -11,7 +11,6 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import java.util.HashMap; import java.util.Map; import java.util.Optional; import org.slf4j.Logger; @@ -44,7 +43,7 @@ public JdbcDatabase getDatabase(final JsonNode config) { @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - return new HashMap<>(); + return SSL_JDBC_PARAMETERS; } public static JdbcDatabase getJdbcDatabase(final JsonNode config) { From 6780ebdb781a6e6de4c17489aa72782d063989dd Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 17 Feb 2022 16:22:54 -0800 Subject: [PATCH 15/60] Add postgres tests --- .../postgres/PostgresDestinationTest.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java index b2d236304efa2..5d13862539273 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.map.MoreMaps; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.AirbyteMessageConsumer; @@ -23,7 +24,9 @@ import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.time.Instant; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.junit.jupiter.api.AfterAll; @@ -47,6 +50,18 @@ public class PostgresDestinationTest { private JsonNode config; + private static final Map CONFIG_WITH_SSL = ImmutableMap.of( + "host", "localhost", + "port", "1337", + "username", "user", + "database", "db" + ); + private static final Map CONFIG_NO_SSL = MoreMaps.merge( + CONFIG_WITH_SSL, + ImmutableMap.of( + "ssl", "false" + )); + @BeforeAll static void init() { PSQL_DB = new PostgreSQLContainer<>("postgres:13-alpine"); @@ -63,6 +78,22 @@ static void cleanUp() { PSQL_DB.close(); } + @Test + void testDefaultParamsNoSSL() { + final Map defaultProperties = new PostgresDestination().getDefaultConnectionProperties( + Jsons.jsonNode(CONFIG_NO_SSL) + ); + assertEquals(new HashMap<>(), defaultProperties); + } + + @Test + void testDefaultParamsWithSSL() { + final Map defaultProperties = new PostgresDestination().getDefaultConnectionProperties( + Jsons.jsonNode(CONFIG_WITH_SSL) + ); + assertEquals(PostgresDestination.SSL_JDBC_PARAMETERS, defaultProperties); + } + // This test is a bit redundant with PostgresIntegrationTest. It makes it easy to run the // destination in the same process as the test allowing us to put breakpoint in, which is handy for // debugging (especially since we use postgres as a guinea pig for most features). From 7e9a36f0afa73a8c4f8645755ab1db8570fc1cc9 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 18 Feb 2022 08:21:34 -0800 Subject: [PATCH 16/60] update clickhouse --- .../clickhouse/ClickhouseDestination.java | 34 ++++++++++--------- .../clickhouse/ClickhouseDestinationTest.java | 32 +++++++++++++++++ .../postgres/PostgresDestinationTest.java | 1 + 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index 260187abacaf8..96c69fed9b82b 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -15,7 +15,6 @@ import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -33,6 +32,12 @@ public class ClickhouseDestination extends AbstractJdbcDestination implements De private static final String PASSWORD = "password"; + static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( + "ssl", "true", + "sslmode", "none" + ); + + public static Destination sshWrappedDestination() { return new SshWrappedDestination(new ClickhouseDestination(), HOST_KEY, PORT_KEY); } @@ -43,25 +48,14 @@ public ClickhouseDestination() { @Override public JsonNode toJdbcConfig(final JsonNode config) { - final List additionalParameters = new ArrayList<>(); - - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:clickhouse://%s:%s/%s?", + final String jdbcUrl = String.format("jdbc:clickhouse://%s:%s/%s?", config.get("host").asText(), config.get("port").asText(), - config.get("database").asText())); - - if (!config.has("ssl") || config.get("ssl").asBoolean()) { - additionalParameters.add("ssl=true"); - additionalParameters.add("sslmode=none"); - } - - if (!additionalParameters.isEmpty()) { - additionalParameters.forEach(x -> jdbcUrl.append(x).append("&")); - } + config.get("database").asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) - .put("jdbc_url", jdbcUrl.toString()); + .put("jdbc_url", jdbcUrl); if (config.has(PASSWORD)) { configBuilder.put(PASSWORD, config.get(PASSWORD).asText()); @@ -70,6 +64,10 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } + private boolean useSsl(final JsonNode config) { + return !config.has("ssl") || config.get("ssl").asBoolean(); + } + @Override public AirbyteConnectionStatus check(final JsonNode config) { try (final JdbcDatabase database = getDatabase(config)) { @@ -87,7 +85,11 @@ public AirbyteConnectionStatus check(final JsonNode config) { @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - return new HashMap<>(); + if (useSsl(config)) { + return SSL_JDBC_PARAMETERS; + } else { + return new HashMap<>(); + } } public static void main(final String[] args) throws Exception { diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java index 36fcb20ac7b40..db97f952944ac 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.map.MoreMaps; import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; @@ -25,7 +26,9 @@ import io.airbyte.protocol.models.JsonSchemaType; import java.time.Instant; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.junit.jupiter.api.AfterAll; @@ -44,6 +47,19 @@ public class ClickhouseDestinationTest { private static ConfiguredAirbyteCatalog catalog; private static JsonNode config; + private static final Map CONFIG_WITH_SSL = ImmutableMap.of( + "host", "localhost", + "port", "1337", + "username", "user", + "database", "db" + ); + + private static final Map CONFIG_NO_SSL = MoreMaps.merge( + CONFIG_WITH_SSL, + ImmutableMap.of( + "ssl", "false" + )); + @BeforeAll static void init() { db = new ClickHouseContainer("yandex/clickhouse-server"); @@ -76,6 +92,22 @@ static void cleanUp() { db.close(); } + @Test + void testDefaultParamsNoSSL() { + final Map defaultProperties = new ClickhouseDestination().getDefaultConnectionProperties( + Jsons.jsonNode(CONFIG_NO_SSL) + ); + assertEquals(new HashMap<>(), defaultProperties); + } + + @Test + void testDefaultParamsWithSSL() { + final Map defaultProperties = new ClickhouseDestination().getDefaultConnectionProperties( + Jsons.jsonNode(CONFIG_WITH_SSL) + ); + assertEquals(ClickhouseDestination.SSL_JDBC_PARAMETERS, defaultProperties); + } + @Test void sanityTest() throws Exception { final Destination dest = new ClickhouseDestination(); diff --git a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java index 5d13862539273..01f7fd96caa61 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java @@ -56,6 +56,7 @@ public class PostgresDestinationTest { "username", "user", "database", "db" ); + private static final Map CONFIG_NO_SSL = MoreMaps.merge( CONFIG_WITH_SSL, ImmutableMap.of( From 6b19e90e20218b3d69bada4ac1441996a3824c4c Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 18 Feb 2022 08:32:07 -0800 Subject: [PATCH 17/60] reformat --- .../source/jdbc/AbstractJdbcSource.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index 4953f6c5e6b63..586fefecfb1c8 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -70,8 +70,8 @@ public abstract class AbstractJdbcSource extends AbstractRelationalDbS protected String quoteString; public AbstractJdbcSource(final String driverClass, - final JdbcStreamingQueryConfiguration jdbcStreamingQueryConfiguration, - final JdbcCompatibleSourceOperations sourceOperations) { + final JdbcStreamingQueryConfiguration jdbcStreamingQueryConfiguration, + final JdbcCompatibleSourceOperations sourceOperations) { this.driverClass = driverClass; this.jdbcStreamingQueryConfiguration = jdbcStreamingQueryConfiguration; this.sourceOperations = sourceOperations; @@ -148,7 +148,7 @@ protected List>> discoverInternal(final JdbcData } protected Predicate excludeNotAccessibleTables(final Set internalSchemas, - final Set tablesWithSelectGrantPrivilege) { + final Set tablesWithSelectGrantPrivilege) { return jsonNode -> { if (tablesWithSelectGrantPrivilege.isEmpty()) { return isNotInternalSchema(jsonNode, internalSchemas); @@ -204,7 +204,7 @@ public JsonSchemaType getType(final Datatype columnType) { @Override protected Map> discoverPrimaryKeys(final JdbcDatabase database, - final List>> tableInfos) { + final List>> tableInfos) { LOGGER.info("Discover primary keys for tables: " + tableInfos.stream().map(TableInfo::getName).collect( Collectors.toSet())); try { @@ -251,12 +251,12 @@ protected String getQuoteString() { @Override public AutoCloseableIterator queryTableIncremental(final JdbcDatabase database, - final List columnNames, - final String schemaName, - final String tableName, - final String cursorField, - final Datatype cursorFieldType, - final String cursor) { + final List columnNames, + final String schemaName, + final String tableName, + final String cursorField, + final Datatype cursorFieldType, + final String cursor) { LOGGER.info("Queueing query for table: {}", tableName); return AutoCloseableIterators.lazyIterator(() -> { try { From 886734265f49e046f86e9ffa44eb9182a87a2852 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 18 Feb 2022 08:32:39 -0800 Subject: [PATCH 18/60] reset --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 79af04f23bc77..5f5a6960a468f 100644 --- a/build.gradle +++ b/build.gradle @@ -304,7 +304,7 @@ subprojects { testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2' testImplementation 'org.mockito:mockito-junit-jupiter:4.0.0' testImplementation 'org.assertj:assertj-core:3.21.0' - testImplementation "org.mockito:mockito-inline:4.1.0" + } tasks.withType(Tar) { From bea8d41a347dd5e9a2e79b432725751e8847f533 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 18 Feb 2022 09:37:20 -0800 Subject: [PATCH 19/60] reformat --- .../MariadbColumnstoreDestination.java | 6 +++--- .../snowflake/SnowflakeInternalStagingDestination.java | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java index 441eb2147c5c8..e98a5dfbdd397 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java @@ -76,14 +76,14 @@ protected Map getDefaultConnectionProperties(final JsonNode conf @Override public JsonNode toJdbcConfig(final JsonNode config) { - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:mariadb://%s:%s/%s", + final String jdbcUrl = String.format("jdbc:mariadb://%s:%s/%s", config.get("host").asText(), config.get("port").asText(), - config.get("database").asText())); + config.get("database").asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) - .put("jdbc_url", jdbcUrl.toString()); + .put("jdbc_url", jdbcUrl); if (config.has("password")) { configBuilder.put("password", config.get("password").asText()); diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index 1e419cb163bd1..09c4ff776f773 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -49,9 +49,9 @@ public AirbyteConnectionStatus check(final JsonNode config) { } private static void attemptSQLCreateAndDropStages(final String outputSchema, - final JdbcDatabase database, - final SnowflakeSQLNameTransformer namingResolver, - final SnowflakeStagingSqlOperations sqlOperations) + final JdbcDatabase database, + final SnowflakeSQLNameTransformer namingResolver, + final SnowflakeStagingSqlOperations sqlOperations) throws Exception { // verify we have permissions to create/drop stage @@ -80,8 +80,8 @@ public JsonNode toJdbcConfig(final JsonNode config) { @Override public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) { + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) { return new SnowflakeInternalStagingConsumerFactory().create(outputRecordCollector, getDatabase(config), new SnowflakeStagingSqlOperations(), new SnowflakeSQLNameTransformer(), config, catalog); } From e090e7a2c5effcfecf4003c1221e395ceabcca7d Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 18 Feb 2022 09:59:47 -0800 Subject: [PATCH 20/60] fix test --- .../destination/mysql/MySQLDestinationTest.java | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java index 4ca7a9049e7f1..cef908044219e 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java @@ -7,7 +7,6 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; import java.util.Map; -import org.junit.Before; import org.junit.jupiter.api.Test; public class MySQLDestinationTest { @@ -18,7 +17,6 @@ public class MySQLDestinationTest { ); public static final String JDBC_URL = "jdbc:mysql://localhost:1337/db"; - private MySQLDestination destination; private JsonNode buildConfigNoJdbcParameters() { return Jsons.jsonNode(ImmutableMap.of( @@ -49,40 +47,35 @@ private JsonNode buildConfigNoExtraJdbcParametersWithoutSsl() { )); } - @Before - public void setUp() { - destination = new MySQLDestination(); - } - @Test void testNoExtraParams() { final JsonNode config = buildConfigNoJdbcParameters(); - final JsonNode jdbcConfig = destination.toJdbcConfig(config); + final JsonNode jdbcConfig = new MySQLDestination().toJdbcConfig(config); assertEquals(JDBC_URL, jdbcConfig.get("jdbc_url").asText()); } @Test void testEmptyExtraParams() { - final JsonNode jdbcConfig = destination.toJdbcConfig(buildConfigWithExtraJdbcParameters("")); + final JsonNode jdbcConfig = new MySQLDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters("")); assertEquals(JDBC_URL, jdbcConfig.get("jdbc_url").asText()); } @Test void testExtraParams() { final String extraParam = "key1=value1&key2=value2&key3=value3"; - final JsonNode jdbcConfig = destination.toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam)); + final JsonNode jdbcConfig = new MySQLDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam)); assertEquals(JDBC_URL, jdbcConfig.get("jdbc_url").asText()); } @Test void testDefaultParamsNoSSL() { - final Map defaultProperties = destination.getDefaultConnectionProperties(buildConfigNoExtraJdbcParametersWithoutSsl()); + final Map defaultProperties = new MySQLDestination().getDefaultConnectionProperties(buildConfigNoExtraJdbcParametersWithoutSsl()); assertEquals(MySQLDestination.DEFAULT_JDBC_PARAMETERS, defaultProperties); } @Test void testDefaultParamsWithSSL() { - final Map defaultProperties = destination.getDefaultConnectionProperties(buildConfigNoJdbcParameters()); + final Map defaultProperties = new MySQLDestination().getDefaultConnectionProperties(buildConfigNoJdbcParameters()); assertEquals(DEFAULT_PARAMETERS_WITH_SSL, defaultProperties); } } From 585aed691d0aa9079fd91c85c1513d2d880888b8 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Tue, 22 Feb 2022 08:49:35 -0800 Subject: [PATCH 21/60] reformat --- .../src/main/java/io/airbyte/db/Databases.java | 1 + .../clickhouse/ClickhouseDestination.java | 4 +--- .../clickhouse/ClickhouseDestinationTest.java | 12 ++++-------- .../jdbc/AbstractJdbcDestination.java | 1 + .../jdbc/AbstractJdbcDestinationTest.java | 18 ++++++++++-------- .../MariadbColumnstoreDestination.java | 3 +-- .../destination/mysql/MySQLDestination.java | 6 ++---- .../mysql/MySQLDestinationTest.java | 3 +-- .../postgres/PostgresDestination.java | 3 +-- .../postgres/PostgresDestinationTest.java | 14 +++++--------- .../redshift/RedshiftInsertDestination.java | 3 +-- .../SnowflakeInternalStagingDestination.java | 1 - 12 files changed, 28 insertions(+), 41 deletions(-) diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java index 2a628cc01abf3..3fb65f85f73b5 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java @@ -220,4 +220,5 @@ public static BigQueryDatabase createBigQueryDatabase(final String projectId, fi public static MongoDatabase createMongoDatabase(final String connectionString, final String databaseName) { return new MongoDatabase(connectionString, databaseName); } + } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index 96c69fed9b82b..d781aefab93a0 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -34,9 +34,7 @@ public class ClickhouseDestination extends AbstractJdbcDestination implements De static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( "ssl", "true", - "sslmode", "none" - ); - + "sslmode", "none"); public static Destination sshWrappedDestination() { return new SshWrappedDestination(new ClickhouseDestination(), HOST_KEY, PORT_KEY); diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java index db97f952944ac..75e5d96637422 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java @@ -51,14 +51,12 @@ public class ClickhouseDestinationTest { "host", "localhost", "port", "1337", "username", "user", - "database", "db" - ); + "database", "db"); private static final Map CONFIG_NO_SSL = MoreMaps.merge( CONFIG_WITH_SSL, ImmutableMap.of( - "ssl", "false" - )); + "ssl", "false")); @BeforeAll static void init() { @@ -95,16 +93,14 @@ static void cleanUp() { @Test void testDefaultParamsNoSSL() { final Map defaultProperties = new ClickhouseDestination().getDefaultConnectionProperties( - Jsons.jsonNode(CONFIG_NO_SSL) - ); + Jsons.jsonNode(CONFIG_NO_SSL)); assertEquals(new HashMap<>(), defaultProperties); } @Test void testDefaultParamsWithSSL() { final Map defaultProperties = new ClickhouseDestination().getDefaultConnectionProperties( - Jsons.jsonNode(CONFIG_WITH_SSL) - ); + Jsons.jsonNode(CONFIG_WITH_SSL)); assertEquals(ClickhouseDestination.SSL_JDBC_PARAMETERS, defaultProperties); } diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java index 6a81e1dcb4cac..d8afb121b1526 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -120,4 +120,5 @@ public AirbyteMessageConsumer getConsumer(final JsonNode config, final Consumer outputRecordCollector) { return JdbcBufferedConsumerFactory.create(outputRecordCollector, getDatabase(config), sqlOperations, namingResolver, config, catalog); } + } diff --git a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java index a8567b3940d44..a62128c41fc8c 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.jdbc; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -18,8 +22,7 @@ private JsonNode buildConfigNoJdbcParameters() { "host", "localhost", "port", 1337, "username", "user", - "database", "db" - )); + "database", "db")); } private JsonNode buildConfigWithExtraJdbcParameters(final String extraParam) { @@ -28,8 +31,7 @@ private JsonNode buildConfigWithExtraJdbcParameters(final String extraParam) { "port", 1337, "username", "user", "database", "db", - "jdbc_url_params", extraParam - )); + "jdbc_url_params", extraParam)); } @Test @@ -94,10 +96,8 @@ void testExtraParameterDiffersFromDefault() { final Map defaultProperties = ImmutableMap.of("key1", "value0"); final String extraParam = "key1=value1&key2=value2&key3=value3"; - assertThrows(IllegalArgumentException.class, () -> - new TestJdbcDestination(defaultProperties).getConnectionProperties( - buildConfigWithExtraJdbcParameters(extraParam)) - ); + assertThrows(IllegalArgumentException.class, () -> new TestJdbcDestination(defaultProperties).getConnectionProperties( + buildConfigWithExtraJdbcParameters(extraParam))); } @Test @@ -129,5 +129,7 @@ protected Map getDefaultConnectionProperties(final JsonNode conf public JsonNode toJdbcConfig(final JsonNode config) { return config; } + } + } diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java index e98a5dfbdd397..f8d6ed49025fc 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java @@ -28,8 +28,7 @@ public class MariadbColumnstoreDestination extends AbstractJdbcDestination imple public static final List PORT_KEY = List.of("port"); static final Map DEFAULT_JDBC_PARAMETERS = ImmutableMap.of( - "allowLoadLocalInfile", "true" - ); + "allowLoadLocalInfile", "true"); public static Destination sshWrappedDestination() { return new SshWrappedDestination(new MariadbColumnstoreDestination(), HOST_KEY, PORT_KEY); diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java index 788e4577ed4f3..f3f3de7d37865 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java @@ -47,8 +47,7 @@ public class MySQLDestination extends AbstractJdbcDestination implements Destina // remove zero date values. // since zero dates are placeholders, we convert them to null by default "zeroDateTimeBehavior", "convertToNull", - "allowLoadLocalInfile", "true" - ); + "allowLoadLocalInfile", "true"); public static Destination sshWrappedDestination() { return new SshWrappedDestination(new MySQLDestination(), List.of(HOST_KEY), List.of(PORT_KEY)); @@ -85,13 +84,12 @@ public MySQLDestination() { super(DRIVER_CLASS, new MySQLNameTransformer(), new MySQLSqlOperations()); } - //Override for testing purposes... + // Override for testing purposes... @Override protected JdbcDatabase getDatabase(final JsonNode config) { return super.getDatabase(config); } - @Override protected Map getDefaultConnectionProperties(final JsonNode config) { if (useSSL(config)) { diff --git a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java index 2acce8a4a0712..31c7d6e38830d 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java @@ -17,8 +17,7 @@ public class MySQLDestinationTest { private static final Map DEFAULT_PARAMETERS_WITH_SSL = MoreMaps.merge( MySQLDestination.DEFAULT_JDBC_PARAMETERS, - MySQLDestination.SSL_JDBC_PARAMETERS - ); + MySQLDestination.SSL_JDBC_PARAMETERS); public static final String JDBC_URL = "jdbc:mysql://localhost:1337/db"; diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java index b2c571d5975f4..4e2b16c651d9c 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java @@ -28,8 +28,7 @@ public class PostgresDestination extends AbstractJdbcDestination implements Dest static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( "ssl", "true", - "sslmode", "require" - ); + "sslmode", "require"); public static Destination sshWrappedDestination() { return new SshWrappedDestination(new PostgresDestination(), HOST_KEY, PORT_KEY); diff --git a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java index 01f7fd96caa61..f66387241dca3 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java @@ -54,14 +54,12 @@ public class PostgresDestinationTest { "host", "localhost", "port", "1337", "username", "user", - "database", "db" - ); - + "database", "db"); + private static final Map CONFIG_NO_SSL = MoreMaps.merge( CONFIG_WITH_SSL, ImmutableMap.of( - "ssl", "false" - )); + "ssl", "false")); @BeforeAll static void init() { @@ -82,16 +80,14 @@ static void cleanUp() { @Test void testDefaultParamsNoSSL() { final Map defaultProperties = new PostgresDestination().getDefaultConnectionProperties( - Jsons.jsonNode(CONFIG_NO_SSL) - ); + Jsons.jsonNode(CONFIG_NO_SSL)); assertEquals(new HashMap<>(), defaultProperties); } @Test void testDefaultParamsWithSSL() { final Map defaultProperties = new PostgresDestination().getDefaultConnectionProperties( - Jsons.jsonNode(CONFIG_WITH_SSL) - ); + Jsons.jsonNode(CONFIG_WITH_SSL)); assertEquals(PostgresDestination.SSL_JDBC_PARAMETERS, defaultProperties); } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java index cae137feb00ce..0c0c26451851b 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java @@ -24,8 +24,7 @@ public class RedshiftInsertDestination extends AbstractJdbcDestination implement private static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( "ssl", "true", - "sslfactory", "com.amazon.redshift.ssl.NonValidatingFactory" - ); + "sslfactory", "com.amazon.redshift.ssl.NonValidatingFactory"); public RedshiftInsertDestination() { super(DRIVER_CLASS, new RedshiftSQLNameTransformer(), new RedshiftSqlOperations()); diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index 09c4ff776f773..3b707be8fdd17 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -57,7 +57,6 @@ private static void attemptSQLCreateAndDropStages(final String outputSchema, // verify we have permissions to create/drop stage final String outputTableName = namingResolver.getIdentifier("_airbyte_connection_test_" + UUID.randomUUID().toString().replaceAll("-", "")); final String stageName = namingResolver.getStageName(outputSchema, outputTableName); - ; sqlOperations.createStageIfNotExists(database, stageName); sqlOperations.dropStageIfExists(database, stageName); } From 4ad590dd3b1958eee2db67a59cced4ae878b7826 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Tue, 22 Feb 2022 09:10:48 -0800 Subject: [PATCH 22/60] fix bug --- .../integrations/destination/mssql/MSSQLDestination.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java index 86aeb9dc35477..3625e5d10cd55 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java +++ b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java @@ -13,7 +13,9 @@ import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import java.io.File; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +32,11 @@ public MSSQLDestination() { super(DRIVER_CLASS, new MSSQLNameTransformer(), new SqlServerOperations()); } + @Override + protected Map getDefaultConnectionProperties(final JsonNode config) { + return new HashMap<>(); + } + @Override public JsonNode toJdbcConfig(final JsonNode config) { final String schema = Optional.ofNullable(config.get("schema")).map(JsonNode::asText).orElse("public"); From c014492702f9832cf409606c4a560a8ebca0ef3f Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Tue, 22 Feb 2022 10:32:17 -0800 Subject: [PATCH 23/60] Add mssql tests --- .../destination/mssql/MSSQLDestination.java | 80 ++++---- .../mssql/MSSQLDestinationTest.java | 185 ++++++++++++++++++ 2 files changed, 220 insertions(+), 45 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java diff --git a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java index 3625e5d10cd55..ca8a95c7dcf25 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java +++ b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java @@ -12,7 +12,6 @@ import io.airbyte.integrations.base.ssh.SshWrappedDestination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import java.io.File; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,30 +33,42 @@ public MSSQLDestination() { @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - return new HashMap<>(); + final HashMap properties = new HashMap<>(); + if (config.has("ssl_method")) { + switch (config.get("ssl_method").asText()) { + case "unencrypted" -> properties.put("encrypt", "false"); + case "encrypted_trust_server_certificate" -> { + properties.put("encrypt", "true"); + properties.put("trustServerCertificate", "true"); + } + case "encrypted_verify_certificate" -> { + properties.put("encrypt", "true"); + properties.put("trustStore", getTrustStoreLocation()); + final String trustStorePassword = System.getProperty("javax.net.ssl.trustStorePassword"); + if (trustStorePassword != null && !trustStorePassword.isEmpty()) { + properties.put("trustStorePassword", config.get("trustStorePassword").asText()); + } + if (config.has("hostNameInCertificate")) { + properties.put("hostNameInCertificate", config.get("hostNameInCertificate").asText()); + } + } + } + } + + return properties; } @Override public JsonNode toJdbcConfig(final JsonNode config) { final String schema = Optional.ofNullable(config.get("schema")).map(JsonNode::asText).orElse("public"); - final List additionalParameters = new ArrayList<>(); - - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:sqlserver://%s:%s;databaseName=%s;", + final String jdbcUrl = String.format("jdbc:sqlserver://%s:%s;databaseName=%s;", config.get("host").asText(), config.get("port").asText(), - config.get("database").asText())); - - if (config.has("ssl_method")) { - readSsl(config, additionalParameters); - } - - if (!additionalParameters.isEmpty()) { - jdbcUrl.append(String.join(";", additionalParameters)); - } + config.get("database").asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() - .put("jdbc_url", jdbcUrl.toString()) + .put("jdbc_url", jdbcUrl) .put("username", config.get("username").asText()) .put("password", config.get("password").asText()) .put("schema", schema); @@ -65,37 +76,16 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } - private void readSsl(final JsonNode config, final List additionalParameters) { - switch (config.get("ssl_method").asText()) { - case "unencrypted": - additionalParameters.add("encrypt=false"); - break; - case "encrypted_trust_server_certificate": - additionalParameters.add("encrypt=true"); - additionalParameters.add("trustServerCertificate=true"); - break; - case "encrypted_verify_certificate": - additionalParameters.add("encrypt=true"); - - // trust store location code found at https://stackoverflow.com/a/56570588 - final String trustStoreLocation = Optional.ofNullable(System.getProperty("javax.net.ssl.trustStore")) - .orElseGet(() -> System.getProperty("java.home") + "/lib/security/cacerts"); - final File trustStoreFile = new File(trustStoreLocation); - if (!trustStoreFile.exists()) { - throw new RuntimeException("Unable to locate the Java TrustStore: the system property javax.net.ssl.trustStore is undefined or " - + trustStoreLocation + " does not exist."); - } - final String trustStorePassword = System.getProperty("javax.net.ssl.trustStorePassword"); - - additionalParameters.add("trustStore=" + trustStoreLocation); - if (trustStorePassword != null && !trustStorePassword.isEmpty()) { - additionalParameters.add("trustStorePassword=" + config.get("trustStorePassword").asText()); - } - if (config.has("hostNameInCertificate")) { - additionalParameters.add("hostNameInCertificate=" + config.get("hostNameInCertificate").asText()); - } - break; + private String getTrustStoreLocation() { + // trust store location code found at https://stackoverflow.com/a/56570588 + final String trustStoreLocation = Optional.ofNullable(System.getProperty("javax.net.ssl.trustStore")) + .orElseGet(() -> System.getProperty("java.home") + "/lib/security/cacerts"); + final File trustStoreFile = new File(trustStoreLocation); + if (!trustStoreFile.exists()) { + throw new RuntimeException("Unable to locate the Java TrustStore: the system property javax.net.ssl.trustStore is undefined or " + + trustStoreLocation + " does not exist."); } + return trustStoreLocation; } public static Destination sshWrappedDestination() { diff --git a/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java b/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java new file mode 100644 index 0000000000000..614884d524c9a --- /dev/null +++ b/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java @@ -0,0 +1,185 @@ +package io.airbyte.integrations.destination.mssql; + +import static java.lang.System.getProperty; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; + +public class MSSQLDestinationTest { + + private Map existingProperties; + + private JsonNode createConfig(final String sslMethod) { + return Jsons.jsonNode(ImmutableMap.builder() + .put("ssl_method", sslMethod) + .put("host", "localhost") + .put("port", "1773") + .put("database", "db") + .put("username", "username") + .put("password", "verysecure") + .build() + ); + } + + private JsonNode createConfigWithTrustStorePassword(final String trustStorePassword) { + return Jsons.jsonNode(ImmutableMap.builder() + .put("ssl_method", "encrypted_verify_certificate") + .put("trustStorePassword", trustStorePassword) + .put("host", "localhost") + .put("port", "1773") + .put("database", "db") + .put("username", "username") + .put("password", "verysecure") + .build() + ); + } + + private JsonNode createConfigWithHostnameInCertificate(final String hostnameInCertificate) { + return Jsons.jsonNode(ImmutableMap.builder() + .put("ssl_method", "encrypted_verify_certificate") + .put("hostNameInCertificate", hostnameInCertificate) + .put("host", "localhost") + .put("port", "1773") + .put("database", "db") + .put("username", "username") + .put("password", "verysecure") + .build() + ); + } + + @BeforeEach + public void setUp() { + existingProperties = new HashMap<>(); + } + + @AfterEach + public void tearDown() { + resetProperties(); + } + + @Test + public void testNoSsl() { + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = Jsons.jsonNode(ImmutableMap.of()); + final Map properties = destination.getDefaultConnectionProperties(config); + assertTrue(properties.isEmpty()); + } + + @Test + public void testUnencrypted() { + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = createConfig("unencrypted"); + final Map properties = destination.getDefaultConnectionProperties(config); + assertEquals(properties.get("encrypt"), "false"); + } + + @Test + public void testEncryptedTrustServerCertificate() { + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = createConfig("encrypted_trust_server_certificate"); + final Map properties = destination.getDefaultConnectionProperties(config); + assertEquals(properties.get("encrypt"), "true"); + assertEquals(properties.get("trustServerCertificate"), "true"); + } + + @Test + public void testEncryptedVerifyCertificate() { + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = createConfig("encrypted_verify_certificate"); + + final Map properties = destination.getDefaultConnectionProperties(config); + assertEquals(properties.get("encrypt"), "true"); + + final String trustStoreLocation = getProperty("java.home") + "/lib/security/cacerts"; + assertEquals(properties.get("trustStore"), trustStoreLocation); + assertNull(properties.get("trustStorePassword")); + assertNull(properties.get("hostNameInCertificate")); //TODO: add test with hostname in certificate + } + + @Test + public void testInvalidTrustStoreFile() { + setProperty("javax.net.ssl.trustStore", "/NOT_A_TRUST_STORE"); + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = createConfig("encrypted_verify_certificate"); + + assertThrows(RuntimeException.class, () -> + destination.getDefaultConnectionProperties(config) + ); + } + + @Test + public void testEncryptedVerifyCertificateWithEmptyTrustStorePassword() { + setProperty("javax.net.ssl.trustStorePassword", ""); + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = createConfigWithTrustStorePassword(""); + + final Map properties = destination.getDefaultConnectionProperties(config); + assertEquals(properties.get("encrypt"), "true"); + + final String trustStoreLocation = getProperty("java.home") + "/lib/security/cacerts"; + assertEquals(properties.get("trustStore"), trustStoreLocation); + assertNull(properties.get("trustStorePassword")); + assertNull(properties.get("hostNameInCertificate")); + } + + @Test + public void testEncryptedVerifyCertificateWithNonEmptyTrustStorePassword() { + final String TRUST_STORE_PASSWORD = "TRUSTSTOREPASSWORD"; + setProperty("javax.net.ssl.trustStorePassword", TRUST_STORE_PASSWORD); + final MSSQLDestination destination = new MSSQLDestination(); + final JsonNode config = createConfigWithTrustStorePassword(TRUST_STORE_PASSWORD); + + final Map properties = destination.getDefaultConnectionProperties(config); + assertEquals(properties.get("encrypt"), "true"); + + final String trustStoreLocation = getProperty("java.home") + "/lib/security/cacerts"; + assertEquals(properties.get("trustStore"), trustStoreLocation); + assertEquals(properties.get("trustStorePassword"), TRUST_STORE_PASSWORD); + assertNull(properties.get("hostNameInCertificate")); + } + + @Test + public void testEncryptedVerifyCertificateWithHostNameInCertificate() { + final MSSQLDestination destination = new MSSQLDestination(); + final String HOSTNAME_IN_CERTIFICATE = "HOSTNAME_IN_CERTIFICATE"; + final JsonNode config = createConfigWithHostnameInCertificate(HOSTNAME_IN_CERTIFICATE); + + final Map properties = destination.getDefaultConnectionProperties(config); + assertEquals(properties.get("encrypt"), "true"); + + final String trustStoreLocation = getProperty("java.home") + "/lib/security/cacerts"; + assertEquals(properties.get("trustStore"), trustStoreLocation); + assertNull(properties.get("trustStorePassword")); + + assertEquals(properties.get("hostNameInCertificate"), HOSTNAME_IN_CERTIFICATE); + } + + private void setProperty(final String key, final String value) { + existingProperties.put(key, System.getProperty(key)); + System.setProperty(key, value); + } + + private void resetProperties() { + existingProperties.forEach((k, v) -> resetProperty(k)); + } + + private void resetProperty(final String key) { + final String value = existingProperties.get(key); + if (value != null) { + System.setProperty(key, value); + } else { + System.clearProperty(key); + } + } + +} From 7255cf9dd508201a6004d40c2b3079fc6b0814e9 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Tue, 22 Feb 2022 10:37:02 -0800 Subject: [PATCH 24/60] refactor test --- .../mssql/MSSQLDestinationTest.java | 40 +++++-------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java b/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java index 614884d524c9a..88697030ffa89 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.map.MoreMaps; import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.AfterEach; @@ -20,41 +21,22 @@ public class MSSQLDestinationTest { private Map existingProperties; private JsonNode createConfig(final String sslMethod) { - return Jsons.jsonNode(ImmutableMap.builder() - .put("ssl_method", sslMethod) - .put("host", "localhost") - .put("port", "1773") - .put("database", "db") - .put("username", "username") - .put("password", "verysecure") - .build() - ); + return createConfig(sslMethod, new HashMap<>()); } - private JsonNode createConfigWithTrustStorePassword(final String trustStorePassword) { - return Jsons.jsonNode(ImmutableMap.builder() - .put("ssl_method", "encrypted_verify_certificate") - .put("trustStorePassword", trustStorePassword) - .put("host", "localhost") - .put("port", "1773") - .put("database", "db") - .put("username", "username") - .put("password", "verysecure") - .build() - ); + private JsonNode createConfig(final String sslMethod, final Map additionalConfigs) { + return Jsons.jsonNode(MoreMaps.merge(baseParameters(sslMethod), additionalConfigs)); } - private JsonNode createConfigWithHostnameInCertificate(final String hostnameInCertificate) { - return Jsons.jsonNode(ImmutableMap.builder() - .put("ssl_method", "encrypted_verify_certificate") - .put("hostNameInCertificate", hostnameInCertificate) + private Map baseParameters(final String sslMethod) { + return ImmutableMap.builder() + .put("ssl_method", sslMethod) .put("host", "localhost") .put("port", "1773") .put("database", "db") .put("username", "username") .put("password", "verysecure") - .build() - ); + .build(); } @BeforeEach @@ -121,7 +103,7 @@ public void testInvalidTrustStoreFile() { public void testEncryptedVerifyCertificateWithEmptyTrustStorePassword() { setProperty("javax.net.ssl.trustStorePassword", ""); final MSSQLDestination destination = new MSSQLDestination(); - final JsonNode config = createConfigWithTrustStorePassword(""); + final JsonNode config = createConfig("encrypted_verify_certificate", ImmutableMap.of("trustStorePassword", "")); final Map properties = destination.getDefaultConnectionProperties(config); assertEquals(properties.get("encrypt"), "true"); @@ -137,7 +119,7 @@ public void testEncryptedVerifyCertificateWithNonEmptyTrustStorePassword() { final String TRUST_STORE_PASSWORD = "TRUSTSTOREPASSWORD"; setProperty("javax.net.ssl.trustStorePassword", TRUST_STORE_PASSWORD); final MSSQLDestination destination = new MSSQLDestination(); - final JsonNode config = createConfigWithTrustStorePassword(TRUST_STORE_PASSWORD); + final JsonNode config = createConfig("encrypted_verify_certificate", ImmutableMap.of("trustStorePassword", TRUST_STORE_PASSWORD)); final Map properties = destination.getDefaultConnectionProperties(config); assertEquals(properties.get("encrypt"), "true"); @@ -152,7 +134,7 @@ public void testEncryptedVerifyCertificateWithNonEmptyTrustStorePassword() { public void testEncryptedVerifyCertificateWithHostNameInCertificate() { final MSSQLDestination destination = new MSSQLDestination(); final String HOSTNAME_IN_CERTIFICATE = "HOSTNAME_IN_CERTIFICATE"; - final JsonNode config = createConfigWithHostnameInCertificate(HOSTNAME_IN_CERTIFICATE); + final JsonNode config = createConfig("encrypted_verify_certificate", ImmutableMap.of("hostNameInCertificate", HOSTNAME_IN_CERTIFICATE)); final Map properties = destination.getDefaultConnectionProperties(config); assertEquals(properties.get("encrypt"), "true"); From c3a3dca32fe4e4782c2052f1c073ecd1ab215e07 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Tue, 22 Feb 2022 10:59:11 -0800 Subject: [PATCH 25/60] fix oracle destination test --- .../NneOracleDestinationAcceptanceTest.java | 17 ++++++++++------- .../oracle/OracleDestinationTest.java | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java index 278b955220de4..1ce723a5527fc 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java @@ -15,6 +15,7 @@ import io.airbyte.db.jdbc.JdbcDatabase; import java.sql.SQLException; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.junit.Test; @@ -37,18 +38,22 @@ public void testEncryption() throws SQLException { config.get("port").asText(), config.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", - "oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " - + algorithm + " )"); + getAdditionalProperties(algorithm)); final String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; - final List collect = database.query(network_service_banner).collect(Collectors.toList()); + final List collect = database.query(network_service_banner).toList(); assertThat(collect.get(2).get("NETWORK_SERVICE_BANNER").asText(), equals("Oracle Advanced Security: " + algorithm + " encryption")); } + private Map getAdditionalProperties(final String algorithm) { + return ImmutableMap. + of("oracle.net.encryption_client", "REQUIRED", + "oracle.net.encryption_types_client", String.format("(%s)", algorithm)); + } + @Test public void testCheckProtocol() throws SQLException { final JsonNode clone = Jsons.clone(getConfig()); @@ -67,9 +72,7 @@ public void testCheckProtocol() throws SQLException { clone.get("port").asText(), clone.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", - "oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " - + algorithm + " )"); + getAdditionalProperties(algorithm)); final String network_service_banner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.query(network_service_banner).collect(Collectors.toList()); diff --git a/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java b/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java new file mode 100644 index 0000000000000..f66236e9e8349 --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java @@ -0,0 +1,14 @@ +package io.airbyte.integrations.destination.oracle; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +public class OracleDestinationTest { + + @Test + void test() { + assertTrue(true); + } + +} From 909bf4d62fe8143c3ff3bf1782b6ed29e3ef63de Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Tue, 22 Feb 2022 12:10:14 -0800 Subject: [PATCH 26/60] oracle tests --- .../destination/oracle/OracleDestination.java | 70 ++++++------ .../NneOracleDestinationAcceptanceTest.java | 2 +- .../oracle/OracleDestinationTest.java | 102 +++++++++++++++++- 3 files changed, 139 insertions(+), 35 deletions(-) diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index a98fd39c357c3..aadcfaf3c9de4 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -14,7 +14,6 @@ import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import java.io.IOException; import java.io.PrintWriter; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,8 +37,10 @@ public class OracleDestination extends AbstractJdbcDestination implements Destin public static final String COLUMN_NAME_EMITTED_AT = "\"" + JavaBaseConstants.COLUMN_NAME_EMITTED_AT.toUpperCase() + "\""; - private static final String KEY_STORE_FILE_PATH = "clientkeystore.jks"; + protected static final String KEY_STORE_FILE_PATH = "clientkeystore.jks"; private static final String KEY_STORE_PASS = RandomStringUtils.randomAlphanumeric(8); + public static final String ENCRYPTION_KEY = "encryption"; + public static final String ENCRYPTION_METHOD_KEY = "encryption_method"; enum Protocol { TCP, @@ -57,16 +58,39 @@ public static Destination sshWrappedDestination() { @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - return new HashMap<>(); + final HashMap properties = new HashMap<>(); + if (config.has(ENCRYPTION_KEY)) { + final JsonNode encryption = config.get(ENCRYPTION_KEY); + final String encryptionMethod = encryption.get(ENCRYPTION_METHOD_KEY).asText(); + switch (encryptionMethod) { + case "unencrypted" -> { + + } + case "client_nne" -> { + final String algorithm = encryption.get("encryption_algorithm").asText(); + properties.put("oracle.net.encryption_client", "REQUIRED"); + properties.put("oracle.net.encryption_types_client", "( " + algorithm + " )"); + } + case "encrypted_verify_certificate" -> { + try { + convertAndImportCertificate(encryption.get("ssl_certificate").asText()); + } catch (final IOException | InterruptedException e) { + throw new RuntimeException("Failed to import certificate into Java Keystore"); + } + properties.put("javax.net.ssl.trustStore", KEY_STORE_FILE_PATH); + properties.put("javax.net.ssl.trustStoreType", "JKS"); + properties.put("javax.net.ssl.trustStorePassword", KEY_STORE_PASS); + } + default -> throw new RuntimeException("Failed to obtain connection protocol from config " + encryption.asText()); + } + + } + return properties; } @Override public JsonNode toJdbcConfig(final JsonNode config) { - final List additionalParameters = new ArrayList<>(); - - final Protocol protocol = config.has("encryption") - ? obtainConnectionProtocol(config.get("encryption"), additionalParameters) - : Protocol.TCP; + final Protocol protocol = obtainConnectionProtocol(config); final String connectionString = String.format( "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=%s)(HOST=%s)(PORT=%s))(CONNECT_DATA=(SID=%s)))", protocol, @@ -82,36 +106,20 @@ public JsonNode toJdbcConfig(final JsonNode config) { configBuilder.put("password", config.get("password").asText()); } - if (!additionalParameters.isEmpty()) { - final String connectionParams = String.join(";", additionalParameters); - configBuilder.put("connection_properties", connectionParams); - } - return Jsons.jsonNode(configBuilder.build()); } - private Protocol obtainConnectionProtocol(final JsonNode encryption, - final List additionalParameters) { - final String encryptionMethod = encryption.get("encryption_method").asText(); + protected Protocol obtainConnectionProtocol(final JsonNode config) { + if (!config.has(ENCRYPTION_KEY)) { + return Protocol.TCP; + } + final JsonNode encryption = config.get(ENCRYPTION_KEY); + final String encryptionMethod = encryption.get(ENCRYPTION_METHOD_KEY).asText(); switch (encryptionMethod) { - case "unencrypted" -> { - return Protocol.TCP; - } - case "client_nne" -> { - final String algorithm = encryption.get("encryption_algorithm").asText(); - additionalParameters.add("oracle.net.encryption_client=REQUIRED"); - additionalParameters.add("oracle.net.encryption_types_client=( " + algorithm + " )"); + case "unencrypted", "client_nne" -> { return Protocol.TCP; } case "encrypted_verify_certificate" -> { - try { - convertAndImportCertificate(encryption.get("ssl_certificate").asText()); - } catch (final IOException | InterruptedException e) { - throw new RuntimeException("Failed to import certificate into Java Keystore"); - } - additionalParameters.add("javax.net.ssl.trustStore=" + KEY_STORE_FILE_PATH); - additionalParameters.add("javax.net.ssl.trustStoreType=JKS"); - additionalParameters.add("javax.net.ssl.trustStorePassword=" + KEY_STORE_PASS); return Protocol.TCPS; } } diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java index 1ce723a5527fc..6554e982a506d 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java @@ -51,7 +51,7 @@ public void testEncryption() throws SQLException { private Map getAdditionalProperties(final String algorithm) { return ImmutableMap. of("oracle.net.encryption_client", "REQUIRED", - "oracle.net.encryption_types_client", String.format("(%s)", algorithm)); + "oracle.net.encryption_types_client", String.format("( %s )", algorithm)); } @Test diff --git a/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java b/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java index f66236e9e8349..be6c72a0e98fd 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java @@ -1,14 +1,110 @@ package io.airbyte.integrations.destination.oracle; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.map.MoreMaps; +import io.airbyte.integrations.destination.oracle.OracleDestination.Protocol; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; public class OracleDestinationTest { + private OracleDestination destination; + + private JsonNode createConfig() { + return createConfig(new HashMap<>()); + } + + private JsonNode createConfig(final Map additionalConfigs) { + return Jsons.jsonNode(MoreMaps.merge(baseParameters(), additionalConfigs)); + } + + private Map baseParameters() { + return ImmutableMap.builder() + .put("host", "localhost") + .put("port", "1773") + .put("database", "db") + .put("username", "username") + .put("password", "verysecure") + .build(); + } + + @BeforeEach + void setUp() { + destination = new OracleDestination(); + } + + @Test + void testNoEncryption() { + final Map properties = destination.getDefaultConnectionProperties(createConfig()); + assertNull(properties.get(OracleDestination.ENCRYPTION_KEY)); + assertNull(properties.get("javax.net.ssl.trustStorePassword")); + + final Protocol protocol = destination.obtainConnectionProtocol(createConfig()); + assertEquals(Protocol.TCP, protocol); + } + + @Test + void testUnencrypted() { + final Map encryptionNode = ImmutableMap.of(OracleDestination.ENCRYPTION_METHOD_KEY, "unencrypted"); + final JsonNode inputConfig = createConfig(ImmutableMap.of(OracleDestination.ENCRYPTION_KEY, encryptionNode)); + final Map properties = destination.getDefaultConnectionProperties(inputConfig); + assertNull(properties.get(OracleDestination.ENCRYPTION_KEY)); + assertNull(properties.get("javax.net.ssl.trustStorePassword")); + + final Protocol protocol = destination.obtainConnectionProtocol(inputConfig); + assertEquals(Protocol.TCP, protocol); + } + + @Test + void testClientNne() { + final String algorithm = "AES256"; + final Map encryptionNode = ImmutableMap.of( + OracleDestination.ENCRYPTION_METHOD_KEY, "client_nne", + "encryption_algorithm", algorithm); + final JsonNode inputConfig = createConfig(ImmutableMap.of(OracleDestination.ENCRYPTION_KEY, encryptionNode)); + final Map properties = destination.getDefaultConnectionProperties(inputConfig); + assertEquals(properties.get("oracle.net.encryption_client"), "REQUIRED"); + assertEquals(properties.get("oracle.net.encryption_types_client"), String.format("( %s )", algorithm)); + assertNull(properties.get("javax.net.ssl.trustStorePassword")); + + final Protocol protocol = destination.obtainConnectionProtocol(inputConfig); + assertEquals(Protocol.TCP, protocol); + } + + @Test + void testEncryptedVerifyCertificate() { + final Map encryptionNode = ImmutableMap.of( + OracleDestination.ENCRYPTION_METHOD_KEY, "encrypted_verify_certificate", "ssl_certificate", "certificate"); + final JsonNode inputConfig = createConfig(ImmutableMap.of(OracleDestination.ENCRYPTION_KEY, encryptionNode)); + final Map properties = destination.getDefaultConnectionProperties(inputConfig); + assertEquals(properties.get("javax.net.ssl.trustStore"), OracleDestination.KEY_STORE_FILE_PATH); + assertEquals(properties.get("javax.net.ssl.trustStoreType"), "JKS"); + assertNotNull(properties.get("javax.net.ssl.trustStorePassword")); + + final Protocol protocol = destination.obtainConnectionProtocol(inputConfig); + assertEquals(Protocol.TCPS, protocol); + } + @Test - void test() { - assertTrue(true); + void testInvalidEncryptionMethod() { + final Map encryptionNode = ImmutableMap.of( + OracleDestination.ENCRYPTION_METHOD_KEY, "invalid_encryption_method"); + final JsonNode inputConfig = createConfig(ImmutableMap.of(OracleDestination.ENCRYPTION_KEY, encryptionNode)); + assertThrows(RuntimeException.class, () -> + destination.getDefaultConnectionProperties(inputConfig) + ); + assertThrows(RuntimeException.class, () -> + destination.obtainConnectionProtocol(inputConfig) + ); } } From 4dbb749de149ca8da12c773254f9d15379f8b451 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Tue, 22 Feb 2022 14:34:00 -0800 Subject: [PATCH 27/60] fix redshift acceptance test --- .../redshift/RedshiftCopyDestinationAcceptanceTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java index a6fdd5f877000..4610da6999a65 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; @@ -21,8 +22,8 @@ import java.util.stream.Collectors; /** - * Integration test testing {@link RedshiftCopyS3Destination}. The default Redshift integration test - * credentials contain S3 credentials - this automatically causes COPY to be selected. + * Integration test testing {@link RedshiftCopyS3Destination}. The default Redshift integration test credentials contain S3 credentials - this + * automatically causes COPY to be selected. */ public class RedshiftCopyDestinationAcceptanceTest extends DestinationAcceptanceTest { @@ -141,7 +142,8 @@ protected Database getDatabase() { baseConfig.get("port").asText(), baseConfig.get("database").asText()), "com.amazon.redshift.jdbc.Driver", null, - "ssl=true;sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); + ImmutableMap.of("ssl", "true", "sslfactory", "com.amazon.redshift.ssl.NonValidatingFactory") + ); } @Override From 697821738cae042db6fcb10f47c4bae819d173ac Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Tue, 22 Feb 2022 15:10:09 -0800 Subject: [PATCH 28/60] Pass string --- .../main/java/io/airbyte/db/Databases.java | 28 +++++++++++++++++++ ...RedshiftCopyDestinationAcceptanceTest.java | 4 +-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java index 3fb65f85f73b5..effde679dd72c 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java @@ -138,6 +138,18 @@ public static Database createDatabase(final String username, return new Database(connectionPool, dialect); } + public static Database createDatabase(final String username, + final String password, + final String jdbcConnectionString, + final String driverClassName, + final SQLDialect dialect, + final String connectionProperties) { + final BasicDataSource connectionPool = + createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); + + return new Database(connectionPool, dialect); + } + public static JdbcDatabase createJdbcDatabase(final String username, final String password, final String jdbcConnectionString, @@ -213,6 +225,22 @@ public static BasicDataSource createBasicDataSource(final String username, return connectionPool; } + public static BasicDataSource createBasicDataSource(final String username, + final String password, + final String jdbcConnectionString, + final String driverClassName, + final String connectionProperties) { + final BasicDataSource connectionPool = new BasicDataSource(); + connectionPool.setDriverClassName(driverClassName); + connectionPool.setUsername(username); + connectionPool.setPassword(password); + connectionPool.setInitialSize(0); + connectionPool.setMaxTotal(5); + connectionPool.setUrl(jdbcConnectionString); + connectionPool.setConnectionProperties(connectionProperties); + return connectionPool; + } + public static BigQueryDatabase createBigQueryDatabase(final String projectId, final String jsonCreds) { return new BigQueryDatabase(projectId, jsonCreds); } diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java index 4610da6999a65..d9a1fae6a06c2 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; @@ -142,8 +141,7 @@ protected Database getDatabase() { baseConfig.get("port").asText(), baseConfig.get("database").asText()), "com.amazon.redshift.jdbc.Driver", null, - ImmutableMap.of("ssl", "true", "sslfactory", "com.amazon.redshift.ssl.NonValidatingFactory") - ); + "ssl=true;sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); } @Override From edae0a677a84c0c6704ecacee6e5b4519796132e Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Tue, 22 Feb 2022 15:28:52 -0800 Subject: [PATCH 29/60] Revert "Pass string" This reverts commit 697821738cae042db6fcb10f47c4bae819d173ac. --- .../main/java/io/airbyte/db/Databases.java | 28 ------------------- ...RedshiftCopyDestinationAcceptanceTest.java | 4 ++- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java index effde679dd72c..3fb65f85f73b5 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java @@ -138,18 +138,6 @@ public static Database createDatabase(final String username, return new Database(connectionPool, dialect); } - public static Database createDatabase(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final SQLDialect dialect, - final String connectionProperties) { - final BasicDataSource connectionPool = - createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); - - return new Database(connectionPool, dialect); - } - public static JdbcDatabase createJdbcDatabase(final String username, final String password, final String jdbcConnectionString, @@ -225,22 +213,6 @@ public static BasicDataSource createBasicDataSource(final String username, return connectionPool; } - public static BasicDataSource createBasicDataSource(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final String connectionProperties) { - final BasicDataSource connectionPool = new BasicDataSource(); - connectionPool.setDriverClassName(driverClassName); - connectionPool.setUsername(username); - connectionPool.setPassword(password); - connectionPool.setInitialSize(0); - connectionPool.setMaxTotal(5); - connectionPool.setUrl(jdbcConnectionString); - connectionPool.setConnectionProperties(connectionProperties); - return connectionPool; - } - public static BigQueryDatabase createBigQueryDatabase(final String projectId, final String jsonCreds) { return new BigQueryDatabase(projectId, jsonCreds); } diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java index d9a1fae6a06c2..4610da6999a65 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; @@ -141,7 +142,8 @@ protected Database getDatabase() { baseConfig.get("port").asText(), baseConfig.get("database").asText()), "com.amazon.redshift.jdbc.Driver", null, - "ssl=true;sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); + ImmutableMap.of("ssl", "true", "sslfactory", "com.amazon.redshift.ssl.NonValidatingFactory") + ); } @Override From ee8d75245b7eafa1a5feb0cab2419a3ecc05474e Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 23 Feb 2022 07:48:13 -0800 Subject: [PATCH 30/60] Double deserialization --- .../RedshiftCopyDestinationAcceptanceTest.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java index 4610da6999a65..f89beec797170 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java @@ -60,9 +60,19 @@ protected List retrieveRecords(final TestDestinationEnv env, final String namespace, final JsonNode streamSchema) throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + final List records = retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace); + return records .stream() - .map(j -> Jsons.deserialize(j.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) + .map(j -> { + final String text = j.get(JavaBaseConstants.COLUMN_NAME_DATA).asText(); + final JsonNode deserialized = Jsons.deserialize(text); + if (deserialized.isTextual()) { + final JsonNode d = Jsons.deserialize(deserialized.asText()); + return d; + } else { + return deserialized; + } + }) .collect(Collectors.toList()); } From 213f47acc42f59953dce42538cd3a07a8dbf3360 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 23 Feb 2022 09:10:20 -0800 Subject: [PATCH 31/60] Revert "Double deserialization" This reverts commit ee8d75245b7eafa1a5feb0cab2419a3ecc05474e. --- .../RedshiftCopyDestinationAcceptanceTest.java | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java index f89beec797170..4610da6999a65 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java @@ -60,19 +60,9 @@ protected List retrieveRecords(final TestDestinationEnv env, final String namespace, final JsonNode streamSchema) throws Exception { - final List records = retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace); - return records + return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) .stream() - .map(j -> { - final String text = j.get(JavaBaseConstants.COLUMN_NAME_DATA).asText(); - final JsonNode deserialized = Jsons.deserialize(text); - if (deserialized.isTextual()) { - final JsonNode d = Jsons.deserialize(deserialized.asText()); - return d; - } else { - return deserialized; - } - }) + .map(j -> Jsons.deserialize(j.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) .collect(Collectors.toList()); } From c8022c299431c8ff43ed4f507bb6475569a74553 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 23 Feb 2022 09:14:11 -0800 Subject: [PATCH 32/60] try updating json_operations --- .../macros/cross_db_utils/json_operations.sql | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql index 619eaf46cbfdc..647c68a346394 100644 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql @@ -43,11 +43,12 @@ {%- endmacro %} {% macro redshift__format_json_path(json_path_list) -%} + {%- set quote = '"' if redshift_super_type() else "'" -%} {%- set str_list = [] -%} {%- for json_path in json_path_list -%} - {%- if str_list.append(json_path.replace("'", "''")) -%} {%- endif -%} + {%- if str_list.append(json_path.replace(quote, quote + quote)) -%} {%- endif -%} {%- endfor -%} - {{ "'" ~ str_list|join("','") ~ "'" }} + {{ quote ~ str_list|join(quote + "," + quote) ~ quote }} {%- endmacro %} {% macro snowflake__format_json_path(json_path_list) -%} @@ -114,11 +115,14 @@ {%- endmacro %} {% macro redshift__json_extract(from_table, json_column, json_path_list, normalized_json_path) -%} - {%- if from_table|string() == '' %} + {%- if from_table|string() != '' -%} + {%- set json_column = from_table|string() + "." + json_column|string() -%} + {%- endif -%} + {%- if redshift_super_type() -%} + case when {{ json_column }}.{{ format_json_path(json_path_list) }} != '' then {{ json_column }}.{{ format_json_path(json_path_list) }} end + {%- else -%} case when json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) != '' then json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) end - {% else %} - case when json_extract_path_text({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }}, true) != '' then json_extract_path_text({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }}, true) end - {% endif -%} + {%- endif -%} {%- endmacro %} {% macro snowflake__json_extract(from_table, json_column, json_path_list, normalized_json_path) -%} @@ -168,7 +172,11 @@ {%- endmacro %} {% macro redshift__json_extract_scalar(json_column, json_path_list, normalized_json_path) -%} + {%- if redshift_super_type() -%} + case when {{ json_column }}.{{ format_json_path(json_path_list) }} != '' then {{ json_column }}.{{ format_json_path(json_path_list) }} end + {%- else -%} case when json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) != '' then json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) end + {%- endif -%} {%- endmacro %} {% macro snowflake__json_extract_scalar(json_column, json_path_list, normalized_json_path) -%} @@ -210,7 +218,11 @@ {%- endmacro %} {% macro redshift__json_extract_array(json_column, json_path_list, normalized_json_path) -%} + {%- if redshift_super_type() -%} + {{ json_column }}.{{ format_json_path(json_path_list) }} + {%- else -%} json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) + {%- endif -%} {%- endmacro %} {% macro snowflake__json_extract_array(json_column, json_path_list, normalized_json_path) -%} @@ -223,4 +235,4 @@ {% macro clickhouse__json_extract_array(json_column, json_path_list, normalized_json_path) -%} JSONExtractArrayRaw({{ json_column }}, {{ format_json_path(json_path_list) }}) -{%- endmacro %} +{%- endmacro %} \ No newline at end of file From c68fc92c1e90004c2d6675bbd8ca85ca58907480 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 23 Feb 2022 09:20:02 -0800 Subject: [PATCH 33/60] Revert "try updating json_operations" This reverts commit c8022c299431c8ff43ed4f507bb6475569a74553. --- .../macros/cross_db_utils/json_operations.sql | 26 +++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql index 647c68a346394..619eaf46cbfdc 100644 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql @@ -43,12 +43,11 @@ {%- endmacro %} {% macro redshift__format_json_path(json_path_list) -%} - {%- set quote = '"' if redshift_super_type() else "'" -%} {%- set str_list = [] -%} {%- for json_path in json_path_list -%} - {%- if str_list.append(json_path.replace(quote, quote + quote)) -%} {%- endif -%} + {%- if str_list.append(json_path.replace("'", "''")) -%} {%- endif -%} {%- endfor -%} - {{ quote ~ str_list|join(quote + "," + quote) ~ quote }} + {{ "'" ~ str_list|join("','") ~ "'" }} {%- endmacro %} {% macro snowflake__format_json_path(json_path_list) -%} @@ -115,14 +114,11 @@ {%- endmacro %} {% macro redshift__json_extract(from_table, json_column, json_path_list, normalized_json_path) -%} - {%- if from_table|string() != '' -%} - {%- set json_column = from_table|string() + "." + json_column|string() -%} - {%- endif -%} - {%- if redshift_super_type() -%} - case when {{ json_column }}.{{ format_json_path(json_path_list) }} != '' then {{ json_column }}.{{ format_json_path(json_path_list) }} end - {%- else -%} + {%- if from_table|string() == '' %} case when json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) != '' then json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) end - {%- endif -%} + {% else %} + case when json_extract_path_text({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }}, true) != '' then json_extract_path_text({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }}, true) end + {% endif -%} {%- endmacro %} {% macro snowflake__json_extract(from_table, json_column, json_path_list, normalized_json_path) -%} @@ -172,11 +168,7 @@ {%- endmacro %} {% macro redshift__json_extract_scalar(json_column, json_path_list, normalized_json_path) -%} - {%- if redshift_super_type() -%} - case when {{ json_column }}.{{ format_json_path(json_path_list) }} != '' then {{ json_column }}.{{ format_json_path(json_path_list) }} end - {%- else -%} case when json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) != '' then json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) end - {%- endif -%} {%- endmacro %} {% macro snowflake__json_extract_scalar(json_column, json_path_list, normalized_json_path) -%} @@ -218,11 +210,7 @@ {%- endmacro %} {% macro redshift__json_extract_array(json_column, json_path_list, normalized_json_path) -%} - {%- if redshift_super_type() -%} - {{ json_column }}.{{ format_json_path(json_path_list) }} - {%- else -%} json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) - {%- endif -%} {%- endmacro %} {% macro snowflake__json_extract_array(json_column, json_path_list, normalized_json_path) -%} @@ -235,4 +223,4 @@ {% macro clickhouse__json_extract_array(json_column, json_path_list, normalized_json_path) -%} JSONExtractArrayRaw({{ json_column }}, {{ format_json_path(json_path_list) }}) -{%- endmacro %} \ No newline at end of file +{%- endmacro %} From 11a6725eaa7c73b13be5e2c3ba7ed0ad25b43c3e Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 23 Feb 2022 14:30:29 -0800 Subject: [PATCH 34/60] json parse --- .../destination/redshift/RedshiftSqlOperations.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftSqlOperations.java index ac4ea2cf60a33..d1b0497b5869d 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftSqlOperations.java @@ -52,7 +52,7 @@ public void insertRecordsInternal(final JdbcDatabase database, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); - final String recordQueryComponent = "(?, ?, ?),\n"; + final String recordQueryComponent = "(?, JSON_PARSE(?), ?),\n"; SqlOperationsUtils.insertRawRecordsInSingleQuery(insertQueryComponent, recordQueryComponent, database, records); } From 8e6b34e3431a9b9c8771ef7b00e9d4d1a1b71578 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 23 Feb 2022 15:03:23 -0800 Subject: [PATCH 35/60] Revert "json parse" This reverts commit 11a6725eaa7c73b13be5e2c3ba7ed0ad25b43c3e. --- .../destination/redshift/RedshiftSqlOperations.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftSqlOperations.java index d1b0497b5869d..ac4ea2cf60a33 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftSqlOperations.java @@ -52,7 +52,7 @@ public void insertRecordsInternal(final JdbcDatabase database, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); - final String recordQueryComponent = "(?, JSON_PARSE(?), ?),\n"; + final String recordQueryComponent = "(?, ?, ?),\n"; SqlOperationsUtils.insertRawRecordsInSingleQuery(insertQueryComponent, recordQueryComponent, database, records); } From 66822454afeaa0c0c7de641b9907c904b659e535 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 23 Feb 2022 15:03:56 -0800 Subject: [PATCH 36/60] Revert "Revert "Double deserialization"" This reverts commit 213f47acc42f59953dce42538cd3a07a8dbf3360. --- .../RedshiftCopyDestinationAcceptanceTest.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java index 4610da6999a65..f89beec797170 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java @@ -60,9 +60,19 @@ protected List retrieveRecords(final TestDestinationEnv env, final String namespace, final JsonNode streamSchema) throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + final List records = retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace); + return records .stream() - .map(j -> Jsons.deserialize(j.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) + .map(j -> { + final String text = j.get(JavaBaseConstants.COLUMN_NAME_DATA).asText(); + final JsonNode deserialized = Jsons.deserialize(text); + if (deserialized.isTextual()) { + final JsonNode d = Jsons.deserialize(deserialized.asText()); + return d; + } else { + return deserialized; + } + }) .collect(Collectors.toList()); } From 58d326f50ae1b4583a62bda8317d42164a355356 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 24 Feb 2022 12:06:06 -0800 Subject: [PATCH 37/60] Revert "Revert "Revert "Double deserialization""" This reverts commit 66822454afeaa0c0c7de641b9907c904b659e535. --- .../RedshiftCopyDestinationAcceptanceTest.java | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java index f89beec797170..4610da6999a65 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java @@ -60,19 +60,9 @@ protected List retrieveRecords(final TestDestinationEnv env, final String namespace, final JsonNode streamSchema) throws Exception { - final List records = retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace); - return records + return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) .stream() - .map(j -> { - final String text = j.get(JavaBaseConstants.COLUMN_NAME_DATA).asText(); - final JsonNode deserialized = Jsons.deserialize(text); - if (deserialized.isTextual()) { - final JsonNode d = Jsons.deserialize(deserialized.asText()); - return d; - } else { - return deserialized; - } - }) + .map(j -> Jsons.deserialize(j.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) .collect(Collectors.toList()); } From 3b979fbb6f36816489be6fda36f55452521262b0 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 24 Feb 2022 13:20:10 -0800 Subject: [PATCH 38/60] move to constant --- .../destination/redshift/RedshiftCopyS3Destination.java | 6 ++++++ .../redshift/RedshiftCopyDestinationAcceptanceTest.java | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftCopyS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftCopyS3Destination.java index 8b8212bb8cf7e..74c92734ea06e 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftCopyS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftCopyS3Destination.java @@ -7,6 +7,7 @@ import static io.airbyte.integrations.destination.redshift.RedshiftInsertDestination.getJdbcDatabase; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.destination.ExtendedNameTransformer; @@ -18,6 +19,7 @@ import io.airbyte.integrations.destination.s3.S3DestinationConfig; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.Map; import java.util.function.Consumer; /** @@ -33,6 +35,10 @@ */ public class RedshiftCopyS3Destination extends CopyDestination { + public final static Map SSL_JDBC_PARAMETERS = ImmutableMap.of( + "ssl", "true", + "sslfactory", "com.amazon.redshift.ssl.NonValidatingFactory"); + @Override public AirbyteMessageConsumer getConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java index 4610da6999a65..8885a27f2973f 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; @@ -142,7 +141,7 @@ protected Database getDatabase() { baseConfig.get("port").asText(), baseConfig.get("database").asText()), "com.amazon.redshift.jdbc.Driver", null, - ImmutableMap.of("ssl", "true", "sslfactory", "com.amazon.redshift.ssl.NonValidatingFactory") + RedshiftCopyS3Destination.SSL_JDBC_PARAMETERS ); } From f263e61b31ce5b95ff683020269f2b9b0dd354a1 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 24 Feb 2022 13:22:21 -0800 Subject: [PATCH 39/60] Add comment --- .../destination/clickhouse/ClickhouseDestination.java | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index d781aefab93a0..b77459a8ee62b 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -86,6 +86,7 @@ protected Map getDefaultConnectionProperties(final JsonNode conf if (useSsl(config)) { return SSL_JDBC_PARAMETERS; } else { + // No need for any parameters if the connection doesn't use SSL return new HashMap<>(); } } From e6ad48ed169ff909f290e78e2de2f6f593344e03 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 24 Feb 2022 13:31:00 -0800 Subject: [PATCH 40/60] map can be constant --- .../destination/mysql/MySQLDestination.java | 18 +++++++----------- .../mysql/MySQLDestinationTest.java | 7 +------ 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java index f3f3de7d37865..7d4720e90c379 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java @@ -36,10 +36,6 @@ public class MySQLDestination extends AbstractJdbcDestination implements Destina public static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; - static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( - "useSSL", "true", - "requireSSL", "true", - "verifyServerCertificate", "false"); static final Map DEFAULT_JDBC_PARAMETERS = ImmutableMap.of( // zero dates by default cannot be parsed into java date objects (they will throw an error) // in addition, users don't always have agency in fixing them e.g: maybe they don't own the database @@ -49,6 +45,12 @@ public class MySQLDestination extends AbstractJdbcDestination implements Destina "zeroDateTimeBehavior", "convertToNull", "allowLoadLocalInfile", "true"); + static final Map DEFAULT_SSL_JDBC_PARAMETERS = MoreMaps.merge(ImmutableMap.of( + "useSSL", "true", + "requireSSL", "true", + "verifyServerCertificate", "false"), + DEFAULT_JDBC_PARAMETERS); + public static Destination sshWrappedDestination() { return new SshWrappedDestination(new MySQLDestination(), List.of(HOST_KEY), List.of(PORT_KEY)); } @@ -84,16 +86,10 @@ public MySQLDestination() { super(DRIVER_CLASS, new MySQLNameTransformer(), new MySQLSqlOperations()); } - // Override for testing purposes... - @Override - protected JdbcDatabase getDatabase(final JsonNode config) { - return super.getDatabase(config); - } - @Override protected Map getDefaultConnectionProperties(final JsonNode config) { if (useSSL(config)) { - return MoreMaps.merge(DEFAULT_JDBC_PARAMETERS, SSL_JDBC_PARAMETERS); + return DEFAULT_SSL_JDBC_PARAMETERS; } else { return DEFAULT_JDBC_PARAMETERS; } diff --git a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java index 31c7d6e38830d..6a98e8f3fa66e 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java @@ -9,16 +9,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.map.MoreMaps; import java.util.Map; import org.junit.jupiter.api.Test; public class MySQLDestinationTest { - private static final Map DEFAULT_PARAMETERS_WITH_SSL = MoreMaps.merge( - MySQLDestination.DEFAULT_JDBC_PARAMETERS, - MySQLDestination.SSL_JDBC_PARAMETERS); - public static final String JDBC_URL = "jdbc:mysql://localhost:1337/db"; private JsonNode buildConfigNoJdbcParameters() { @@ -76,7 +71,7 @@ void testDefaultParamsNoSSL() { @Test void testDefaultParamsWithSSL() { final Map defaultProperties = new MySQLDestination().getDefaultConnectionProperties(buildConfigNoJdbcParameters()); - assertEquals(DEFAULT_PARAMETERS_WITH_SSL, defaultProperties); + assertEquals(MySQLDestination.DEFAULT_SSL_JDBC_PARAMETERS, defaultProperties); } } From cf9019ef8cfbc73b8efbb681a9e3650f8c0fad68 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 24 Feb 2022 13:35:49 -0800 Subject: [PATCH 41/60] Add comment --- .../integrations/destination/postgres/PostgresDestination.java | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java index 4e2b16c651d9c..d592f13c852c2 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java @@ -43,6 +43,7 @@ protected Map getDefaultConnectionProperties(final JsonNode conf if (useSsl(config)) { return SSL_JDBC_PARAMETERS; } else { + // No need for any parameters if the connection doesn't use SSL return new HashMap<>(); } } From f894eb716fcfff458b92ebbdce97abc2dbb89441 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 24 Feb 2022 13:39:45 -0800 Subject: [PATCH 42/60] move map --- .../destination/redshift/RedshiftCopyS3Destination.java | 6 ------ .../destination/redshift/RedshiftInsertDestination.java | 2 +- .../redshift/RedshiftCopyDestinationAcceptanceTest.java | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftCopyS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftCopyS3Destination.java index 74c92734ea06e..8b8212bb8cf7e 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftCopyS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftCopyS3Destination.java @@ -7,7 +7,6 @@ import static io.airbyte.integrations.destination.redshift.RedshiftInsertDestination.getJdbcDatabase; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.destination.ExtendedNameTransformer; @@ -19,7 +18,6 @@ import io.airbyte.integrations.destination.s3.S3DestinationConfig; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.util.Map; import java.util.function.Consumer; /** @@ -35,10 +33,6 @@ */ public class RedshiftCopyS3Destination extends CopyDestination { - public final static Map SSL_JDBC_PARAMETERS = ImmutableMap.of( - "ssl", "true", - "sslfactory", "com.amazon.redshift.ssl.NonValidatingFactory"); - @Override public AirbyteMessageConsumer getConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java index 0c0c26451851b..576a9246bb962 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java @@ -22,7 +22,7 @@ public class RedshiftInsertDestination extends AbstractJdbcDestination implement public static final String DRIVER_CLASS = "com.amazon.redshift.jdbc.Driver"; - private static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( + public static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( "ssl", "true", "sslfactory", "com.amazon.redshift.ssl.NonValidatingFactory"); diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java index 8885a27f2973f..413ef0c261ee3 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java @@ -141,7 +141,7 @@ protected Database getDatabase() { baseConfig.get("port").asText(), baseConfig.get("database").asText()), "com.amazon.redshift.jdbc.Driver", null, - RedshiftCopyS3Destination.SSL_JDBC_PARAMETERS + RedshiftInsertDestination.SSL_JDBC_PARAMETERS ); } From 5daa1d01f3031c04dffb528efc376b3122b33bc5 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 24 Feb 2022 13:44:28 -0800 Subject: [PATCH 43/60] hide in method --- .../destination/oracle/OracleDestination.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index aadcfaf3c9de4..795882a30ad99 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -72,11 +72,7 @@ protected Map getDefaultConnectionProperties(final JsonNode conf properties.put("oracle.net.encryption_types_client", "( " + algorithm + " )"); } case "encrypted_verify_certificate" -> { - try { - convertAndImportCertificate(encryption.get("ssl_certificate").asText()); - } catch (final IOException | InterruptedException e) { - throw new RuntimeException("Failed to import certificate into Java Keystore"); - } + tryConvertAndImportCertificate(encryption.get("ssl_certificate").asText()); properties.put("javax.net.ssl.trustStore", KEY_STORE_FILE_PATH); properties.put("javax.net.ssl.trustStoreType", "JKS"); properties.put("javax.net.ssl.trustStorePassword", KEY_STORE_PASS); @@ -127,6 +123,14 @@ protected Protocol obtainConnectionProtocol(final JsonNode config) { "Failed to obtain connection protocol from config " + encryption.asText()); } + private static void tryConvertAndImportCertificate(final String certificate) { + try { + convertAndImportCertificate(certificate); + } catch (final IOException | InterruptedException e) { + throw new RuntimeException("Failed to import certificate into Java Keystore"); + } + } + private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); From 68de5fda14ed82f1e4ee64e1a8f9416a29a4e9b1 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 24 Feb 2022 13:48:38 -0800 Subject: [PATCH 44/60] no need to create new map --- .../destination/snowflake/SnowflakeInsertDestination.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java index 4bcf4ec5142e2..3a346fb6909bd 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestination.java @@ -9,7 +9,7 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import java.util.HashMap; +import java.util.Collections; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,7 +30,7 @@ protected JdbcDatabase getDatabase(final JsonNode config) { @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - return new HashMap<>(); + return Collections.emptyMap(); } // this is a no op since we override getDatabase. From d0d5a384b50e6d0ed78f1309530e945b4b51f0a9 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 24 Feb 2022 13:49:17 -0800 Subject: [PATCH 45/60] no need to create new map --- .../destination/postgres/PostgresDestination.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java index d592f13c852c2..73e4c81be33a9 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java @@ -11,7 +11,7 @@ import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.ssh.SshWrappedDestination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import java.util.HashMap; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -44,7 +44,7 @@ protected Map getDefaultConnectionProperties(final JsonNode conf return SSL_JDBC_PARAMETERS; } else { // No need for any parameters if the connection doesn't use SSL - return new HashMap<>(); + return Collections.emptyMap(); } } From 99ce3af9f48180a04982584adf171df55c810fef Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 24 Feb 2022 13:51:14 -0800 Subject: [PATCH 46/60] no need to create new map --- .../snowflake/SnowflakeInternalStagingDestination.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index 3b707be8fdd17..33897c854f362 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -14,7 +14,7 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.util.HashMap; +import java.util.Collections; import java.util.Map; import java.util.UUID; import java.util.function.Consumer; @@ -68,7 +68,7 @@ protected JdbcDatabase getDatabase(final JsonNode config) { @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - return new HashMap<>(); + return Collections.emptyMap(); } // this is a no op since we override getDatabase. From 9c2bd15c3e800871b5d22583b90dc87de4cf053c Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 24 Feb 2022 15:20:12 -0800 Subject: [PATCH 47/60] enably mysql test --- .../destination/mysql/MySQLDestinationAcceptanceTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java index a9f491059c558..6016ca02c3be0 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java @@ -189,8 +189,7 @@ protected void tearDown(final TestDestinationEnv testEnv) { public void testCustomDbtTransformations() throws Exception { // We need to create view for testing custom dbt transformations executeQuery("GRANT CREATE VIEW ON *.* TO " + db.getUsername() + "@'%';"); - // overrides test with a no-op until https://github.com/dbt-labs/jaffle_shop/pull/8 is merged - // super.testCustomDbtTransformations(); + super.testCustomDbtTransformations(); } @Test From 8b35091e51817c5b5a533908013bb107c5c1b186 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 25 Feb 2022 08:44:12 -0800 Subject: [PATCH 48/60] Update changelogs --- docs/integrations/destinations/clickhouse.md | 11 +-- .../destinations/mariadb-columnstore.md | 11 +-- docs/integrations/destinations/mssql.md | 32 ++++---- docs/integrations/destinations/mysql.md | 36 ++++----- docs/integrations/destinations/oracle.md | 79 +++++++------------ docs/integrations/destinations/postgres.md | 13 +-- docs/integrations/destinations/redshift.md | 15 ++-- docs/integrations/destinations/snowflake.md | 1 + 8 files changed, 93 insertions(+), 105 deletions(-) diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index ee9b6c937115f..7fd35a2f89080 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -77,9 +77,10 @@ Therefore, Airbyte ClickHouse destination will create tables and schemas using t ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.1 | 2021-12-21 | [\#8982](https://github.com/airbytehq/airbyte/pull/8982) | Set isSchemaRequired to false | -| 0.1.0 | 2021-11-04 | [\#7620](https://github.com/airbytehq/airbyte/pull/7620) | Add ClickHouse destination | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :--- |:---------------------------------------------| +| 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.1 | 2021-12-21 | [\#8982](https://github.com/airbytehq/airbyte/pull/8982) | Set isSchemaRequired to false | +| 0.1.0 | 2021-11-04 | [\#7620](https://github.com/airbytehq/airbyte/pull/7620) | Add ClickHouse destination | diff --git a/docs/integrations/destinations/mariadb-columnstore.md b/docs/integrations/destinations/mariadb-columnstore.md index af57ac6d9dbdf..e2a870033019a 100644 --- a/docs/integrations/destinations/mariadb-columnstore.md +++ b/docs/integrations/destinations/mariadb-columnstore.md @@ -74,10 +74,11 @@ Using this feature requires additional configuration, when creating the destinat ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------| :--- |:---------------------------------------------------------|:------------------------------------------| +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------| +| 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.2 | 2021-12-30 | [\#8809](https://github.com/airbytehq/airbyte/pull/8809) | Update connector fields title/description | -| 0.1.1 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key. | -| 0.1.0 | 2021-11-15 | [\#7961](https://github.com/airbytehq/airbyte/pull/7961) | Added MariaDB ColumnStore destination. | +| 0.1.2 | 2021-12-30 | [\#8809](https://github.com/airbytehq/airbyte/pull/8809) | Update connector fields title/description | +| 0.1.1 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key. | +| 0.1.0 | 2021-11-15 | [\#7961](https://github.com/airbytehq/airbyte/pull/7961) | Added MariaDB ColumnStore destination. | diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md index 53d91ff375894..783d7f6fbfab6 100644 --- a/docs/integrations/destinations/mssql.md +++ b/docs/integrations/destinations/mssql.md @@ -117,24 +117,26 @@ Using this feature requires additional configuration, when creating the source. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--- |:---------------------------------------------------------| :--- | -| 0.1.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.13 | 2021-12-28 | [\#9158](https://github.com/airbytehq/airbyte/pull/9158) | Update connector fields title/description | -| 0.1.12 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| Version | Date | Pull Request | Subject | +|:--------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| +| 0.1.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.13 | 2021-12-28 | [\#9158](https://github.com/airbytehq/airbyte/pull/9158) | Update connector fields title/description | +| 0.1.12 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | | 0.1.11 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | -| 0.1.10 | 2021-10-11 | [\#6877](https://github.com/airbytehq/airbyte/pull/6877) | Add `normalization` capability, add `append+deduplication` sync mode | -| 0.1.9 | 2021-09-29 | [\#5970](https://github.com/airbytehq/airbyte/pull/5970) | Add support & test cases for MSSQL Destination via SSH tunnels | -| 0.1.8 | 2021-08-07 | [\#5272](https://github.com/airbytehq/airbyte/pull/5272) | Add batch method to insert records | -| 0.1.7 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | -| 0.1.6 | 2021-06-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | -| 0.1.5 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | declare object types correctly in spec | -| 0.1.4 | 2021-06-17 | [\#3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix doc/params in specification file | -| 0.1.3 | 2021-05-28 | [\#3728](https://github.com/airbytehq/airbyte/pull/3973) | Change dockerfile entrypoint | -| 0.1.2 | 2021-05-13 | [\#3367](https://github.com/airbytehq/airbyte/pull/3671) | Fix handle symbols unicode | -| 0.1.1 | 2021-05-11 | [\#3566](https://github.com/airbytehq/airbyte/pull/3195) | MS SQL Server Destination Release! | +| 0.1.10 | 2021-10-11 | [\#6877](https://github.com/airbytehq/airbyte/pull/6877) | Add `normalization` capability, add `append+deduplication` sync mode | +| 0.1.9 | 2021-09-29 | [\#5970](https://github.com/airbytehq/airbyte/pull/5970) | Add support & test cases for MSSQL Destination via SSH tunnels | +| 0.1.8 | 2021-08-07 | [\#5272](https://github.com/airbytehq/airbyte/pull/5272) | Add batch method to insert records | +| 0.1.7 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | +| 0.1.6 | 2021-06-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | +| 0.1.5 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | declare object types correctly in spec | +| 0.1.4 | 2021-06-17 | [\#3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix doc/params in specification file | +| 0.1.3 | 2021-05-28 | [\#3728](https://github.com/airbytehq/airbyte/pull/3973) | Change dockerfile entrypoint | +| 0.1.2 | 2021-05-13 | [\#3367](https://github.com/airbytehq/airbyte/pull/3671) | Fix handle symbols unicode | +| 0.1.1 | 2021-05-11 | [\#3566](https://github.com/airbytehq/airbyte/pull/3195) | MS SQL Server Destination Release! | ### Changelog (Strict Encrypt) + | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | | 0.1.4 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | diff --git a/docs/integrations/destinations/mysql.md b/docs/integrations/destinations/mysql.md index 35486908b5077..86d0840bb9350 100644 --- a/docs/integrations/destinations/mysql.md +++ b/docs/integrations/destinations/mysql.md @@ -103,25 +103,25 @@ Using this feature requires additional configuration, when creating the destinat ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------| :--- | :--- | :--- | -| 0.1.17 | 2022-02-16 | [10362](https://github.com/airbytehq/airbyte/pull/10362) | Add jdbc_url_params support for optional JDBC parameters | -| 0.1.16 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.15 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| Version | Date | Pull Request | Subject | +|:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------| +| 0.1.18 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.17 | 2022-02-16 | [10362](https://github.com/airbytehq/airbyte/pull/10362) | Add jdbc_url_params support for optional JDBC parameters | +| 0.1.16 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.15 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | | 0.1.14 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | -| 0.1.13 | 2021-09-28 | [\#6506](https://github.com/airbytehq/airbyte/pull/6506) | Added support for MySQL destination via TLS/SSL | -| 0.1.12 | 2021-09-24 | [\#6317](https://github.com/airbytehq/airbyte/pull/6317) | Added option to connect to DB via SSH | -| 0.1.11 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | -| 0.1.10 | 2021-07-28 | [\#5026](https://github.com/airbytehq/airbyte/pull/5026) | Add sanitized json fields in raw tables to handle quotes in column names | -| 0.1.7 | 2021-07-09 | [\#4651](https://github.com/airbytehq/airbyte/pull/4651) | Switch normalization flag on so users can use normalization. | -| 0.1.6 | 2021-07-03 | [\#4531](https://github.com/airbytehq/airbyte/pull/4531) | Added normalization for MySQL. | -| 0.1.5 | 2021-07-03 | [\#3973](https://github.com/airbytehq/airbyte/pull/3973) | Added `AIRBYTE_ENTRYPOINT` for kubernetes support. | -| 0.1.4 | 2021-07-03 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Switched to get states from destination instead of source. | -| 0.1.3 | 2021-07-03 | [\#3387](https://github.com/airbytehq/airbyte/pull/3387) | Fixed a bug for message length checking. | -| 0.1.2 | 2021-07-03 | [\#3327](https://github.com/airbytehq/airbyte/pull/3327) | Fixed LSEP unicode characters. | -| 0.1.1 | 2021-07-03 | [\#3289](https://github.com/airbytehq/airbyte/pull/3289) | Added support for outputting messages. | -| 0.1.0 | 2021-05-06 | [\#3242](https://github.com/airbytehq/airbyte/pull/3242) | Added MySQL destination. | - +| 0.1.13 | 2021-09-28 | [\#6506](https://github.com/airbytehq/airbyte/pull/6506) | Added support for MySQL destination via TLS/SSL | +| 0.1.12 | 2021-09-24 | [\#6317](https://github.com/airbytehq/airbyte/pull/6317) | Added option to connect to DB via SSH | +| 0.1.11 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | +| 0.1.10 | 2021-07-28 | [\#5026](https://github.com/airbytehq/airbyte/pull/5026) | Add sanitized json fields in raw tables to handle quotes in column names | +| 0.1.7 | 2021-07-09 | [\#4651](https://github.com/airbytehq/airbyte/pull/4651) | Switch normalization flag on so users can use normalization. | +| 0.1.6 | 2021-07-03 | [\#4531](https://github.com/airbytehq/airbyte/pull/4531) | Added normalization for MySQL. | +| 0.1.5 | 2021-07-03 | [\#3973](https://github.com/airbytehq/airbyte/pull/3973) | Added `AIRBYTE_ENTRYPOINT` for kubernetes support. | +| 0.1.4 | 2021-07-03 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Switched to get states from destination instead of source. | +| 0.1.3 | 2021-07-03 | [\#3387](https://github.com/airbytehq/airbyte/pull/3387) | Fixed a bug for message length checking. | +| 0.1.2 | 2021-07-03 | [\#3327](https://github.com/airbytehq/airbyte/pull/3327) | Fixed LSEP unicode characters. | +| 0.1.1 | 2021-07-03 | [\#3289](https://github.com/airbytehq/airbyte/pull/3289) | Added support for outputting messages. | +| 0.1.0 | 2021-05-06 | [\#3242](https://github.com/airbytehq/airbyte/pull/3242) | Added MySQL destination. | ## CHANGELOG destination-mysql-strict-encrypt diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md index 2d16d3c393ae3..7c5d3c6fd953a 100644 --- a/docs/integrations/destinations/oracle.md +++ b/docs/integrations/destinations/oracle.md @@ -70,66 +70,47 @@ When using an SSH tunnel, you are configuring Airbyte to connect to an intermedi Using this feature requires additional configuration, when creating the source. We will talk through what each piece of configuration means. 1. Configure all fields for the source as you normally would, except `SSH Tunnel Method`. -2. `SSH Tunnel Method` defaults to `No Tunnel` \(meaning a direct connection\). If you want to use - an SSH Tunnel choose `SSH Key Authentication` or `Password Authentication`. - 1. Choose `Key Authentication` if you will be using an RSA private key as your secret for - establishing the SSH Tunnel \(see below for more information on generating this key\). - 2. Choose `Password Authentication` if you will be using a password as your secret for - establishing the SSH Tunnel. -3. `SSH Tunnel Jump Server Host` refers to the intermediate \(bastion\) server that Airbyte will - connect to. This should be a hostname or an IP Address. -4. `SSH Connection Port` is the port on the bastion server with which to make the SSH connection. - The default port for SSH connections is `22`, so unless you have explicitly changed something, go - with the default. -5. `SSH Login Username` is the username that Airbyte should use when connection to the bastion - server. This is NOT the Oracle username. -6. If you are using `Password Authentication`, then `SSH Login Username` should be set to the - password of the User from the previous step. If you are using `SSH Key Authentication` leave this - blank. Again, this is not the Oracle password, but the password for the OS-user that Airbyte is - using to perform commands on the bastion. -7. If you are using `SSH Key Authentication`, then `SSH Private Key` should be set to the RSA - Private Key that you are using to create the SSH connection. This should be the full contents of - the key file starting with `-----BEGIN RSA PRIVATE KEY-----` and ending - with `-----END RSA PRIVATE KEY-----`. +2. `SSH Tunnel Method` defaults to `No Tunnel` \(meaning a direct connection\). If you want to use an SSH Tunnel choose `SSH Key Authentication` or `Password Authentication`. + 1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel \(see below for more information on generating this key\). + 2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel. +3. `SSH Tunnel Jump Server Host` refers to the intermediate \(bastion\) server that Airbyte will connect to. This should be a hostname or an IP Address. +4. `SSH Connection Port` is the port on the bastion server with which to make the SSH connection. The default port for SSH connections is `22`, so unless you have explicitly changed something, go with the default. +5. `SSH Login Username` is the username that Airbyte should use when connection to the bastion server. This is NOT the Oracle username. +6. If you are using `Password Authentication`, then `SSH Login Username` should be set to the password of the User from the previous step. If you are using `SSH Key Authentication` leave this blank. Again, this is not the Oracle password, but the password for the OS-user that Airbyte is using to perform commands on the bastion. +7. If you are using `SSH Key Authentication`, then `SSH Private Key` should be set to the RSA Private Key that you are using to create the SSH connection. This should be the full contents of the key file starting with `-----BEGIN RSA PRIVATE KEY-----` and ending with `-----END RSA PRIVATE KEY-----`. ## Encryption Options Airbite has the ability to connect to the Oracle source with 3 network connectivity options: -1. `Unencrypted` the connection will be made using the TCP protocol. In this case, all data over the - network will be transmitted in unencrypted form. -2. `Native network encryption` gives you the ability to encrypt database connections, without the - configuration overhead of TCP / IP and SSL / TLS and without the need to open and listen on - different ports. In this case, the *SQLNET.ENCRYPTION_CLIENT* - option will always be set as *REQUIRED* by default: The client or server will only accept - encrypted traffic, but the user has the opportunity to choose an `Encryption algorithm` according - to the security policies he needs. -3. `TLS Encrypted` (verify certificate) - if this option is selected, data transfer will be - transfered using the TLS protocol, taking into account the handshake procedure and certificate - verification. To use this option, insert the content of the certificate issued by the server into - the `SSL PEM file` field +1. `Unencrypted` the connection will be made using the TCP protocol. In this case, all data over the network will be transmitted in unencrypted form. +2. `Native network encryption` gives you the ability to encrypt database connections, without the configuration overhead of TCP / IP and SSL / TLS and without the need to open and listen on different ports. In this case, the *SQLNET.ENCRYPTION_CLIENT* + option will always be set as *REQUIRED* by default: The client or server will only accept encrypted traffic, but the user has the opportunity to choose an `Encryption algorithm` according to the security policies he needs. +3. `TLS Encrypted` (verify certificate) - if this option is selected, data transfer will be transfered using the TLS protocol, taking into account the handshake procedure and certificate verification. To use this option, insert the content of the certificate issued by the server into the `SSL PEM file` field ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- |:---------------------------------------------------------| :--- | -| 0.1.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.13 | 2021-12-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description | -| 0.1.12 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | -| 0.1.10 | 2021-10-08 | [\#6893](https://github.com/airbytehq/airbyte/pull/6893) | 🎉 Destination Oracle: implemented connection encryption | -| 0.1.9 | 2021-10-06 | [\#6611](https://github.com/airbytehq/airbyte/pull/6611) | 🐛 Destination Oracle: maxStringLength should be 128 | -| 0.1.8 | 2021-09-28 | [\#6370](https://github.com/airbytehq/airbyte/pull/6370) | Add SSH Support for Oracle Destination | -| 0.1.7 | 2021-08-30 | [\#5746](https://github.com/airbytehq/airbyte/pull/5746) | Use default column name for raw tables | -| 0.1.6 | 2021-08-23 | [\#5542](https://github.com/airbytehq/airbyte/pull/5542) | Remove support for Oracle 11g to allow normalization | -| 0.1.5 | 2021-08-10 | [\#5307](https://github.com/airbytehq/airbyte/pull/5307) | 🐛 Destination Oracle: Fix destination check for users without dba role | -| 0.1.4 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | -| 0.1.3 | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | -| 0.1.2 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | Require `sid` instead of `database` in connector specification | - +| Version | Date | Pull Request | Subject | +|:--------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| +| 0.1.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.13 | 2021-12-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description | +| 0.1.12 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | +| 0.1.10 | 2021-10-08 | [\#6893](https://github.com/airbytehq/airbyte/pull/6893) | 🎉 Destination Oracle: implemented connection encryption | +| 0.1.9 | 2021-10-06 | [\#6611](https://github.com/airbytehq/airbyte/pull/6611) | 🐛 Destination Oracle: maxStringLength should be 128 | +| 0.1.8 | 2021-09-28 | [\#6370](https://github.com/airbytehq/airbyte/pull/6370) | Add SSH Support for Oracle Destination | +| 0.1.7 | 2021-08-30 | [\#5746](https://github.com/airbytehq/airbyte/pull/5746) | Use default column name for raw tables | +| 0.1.6 | 2021-08-23 | [\#5542](https://github.com/airbytehq/airbyte/pull/5542) | Remove support for Oracle 11g to allow normalization | +| 0.1.5 | 2021-08-10 | [\#5307](https://github.com/airbytehq/airbyte/pull/5307) | 🐛 Destination Oracle: Fix destination check for users without dba role | +| 0.1.4 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | +| 0.1.3 | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | +| 0.1.2 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | Require `sid` instead of `database` in connector specification | ### Changelog (Strict Encrypt) + | Version | Date | Pull Request | Subject | |:--------|:-----------|:--------------------------------------------------------| :--- | -| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.2 | 2021-01-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description | | 0.1.1 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md index 6ec3b4d743da1..3c10867e65d8e 100644 --- a/docs/integrations/destinations/postgres.md +++ b/docs/integrations/destinations/postgres.md @@ -82,11 +82,12 @@ Therefore, Airbyte Postgres destination will create tables and schemas using the ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--- | :--- | :--- | -| 0.3.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.3.13 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| Version | Date | Pull Request | Subject | +|:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------| +| 0.3.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.3.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.3.13 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | | 0.3.12 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | -| 0.3.11 | 2021-09-07 | [\#5743](https://github.com/airbytehq/airbyte/pull/5743) | Add SSH Tunnel support | -| 0.3.10 | 2021-08-11 | [\#5336](https://github.com/airbytehq/airbyte/pull/5336) | 🐛 Destination Postgres: fix \u0000\(NULL\) value processing | +| 0.3.11 | 2021-09-07 | [\#5743](https://github.com/airbytehq/airbyte/pull/5743) | Add SSH Tunnel support | +| 0.3.10 | 2021-08-11 | [\#5336](https://github.com/airbytehq/airbyte/pull/5336) | 🐛 Destination Postgres: fix \u0000\(NULL\) value processing | diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 7885ac484ec68..eabed6f7d7220 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -123,13 +123,14 @@ All Redshift connections are encrypted using SSL ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :-------- | :----- | :------ | -| 0.3.25 | 2022-02-14 | [#9920](https://github.com/airbytehq/airbyte/pull/9920) | Updated the size of staging files for S3 staging. Also, added closure of S3 writers to staging files when data has been written to an staging file. | -| 0.3.24 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.3.23 | 2021-12-16 | [\#8855](https://github.com/airbytehq/airbyte/pull/8855) | Add `purgeStagingData` option to enable/disable deleting the staging data | -| 0.3.22 | 2021-12-15 | [#8607](https://github.com/airbytehq/airbyte/pull/8607) | Accept a path for the staging data | -| 0.3.21 | 2021-12-10 | [#8562](https://github.com/airbytehq/airbyte/pull/8562) | Moving classes around for better dependency management | -| 0.3.20 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | +|:--------| :-------- | :----- | :------ | +| 0.3.26 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.3.25 | 2022-02-14 | [#9920](https://github.com/airbytehq/airbyte/pull/9920) | Updated the size of staging files for S3 staging. Also, added closure of S3 writers to staging files when data has been written to an staging file. | +| 0.3.24 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.3.23 | 2021-12-16 | [\#8855](https://github.com/airbytehq/airbyte/pull/8855) | Add `purgeStagingData` option to enable/disable deleting the staging data | +| 0.3.22 | 2021-12-15 | [#8607](https://github.com/airbytehq/airbyte/pull/8607) | Accept a path for the staging data | +| 0.3.21 | 2021-12-10 | [#8562](https://github.com/airbytehq/airbyte/pull/8562) | Moving classes around for better dependency management | +| 0.3.20 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | | 0.3.19 | 2021-10-21 | [7234](https://github.com/airbytehq/airbyte/pull/7234) | Allow SSL traffic only | | 0.3.17 | 2021-10-12 | [6965](https://github.com/airbytehq/airbyte/pull/6965) | Added SSL Support | | 0.3.16 | 2021-10-11 | [6949](https://github.com/airbytehq/airbyte/pull/6949) | Each stream was split into files of 10,000 records each for copying using S3 or GCS | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index aa39e657f2b7f..554f958f718fd 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -224,6 +224,7 @@ Finally, you need to add read/write permissions to your bucket with that email. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----- | :------ | +| 0.4.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.4.14 | 2022-02-17 | [\#10394](https://github.com/airbytehq/airbyte/pull/10394) | Reduce memory footprint. | | 0.4.13 | 2022-02-16 | [\#10212](https://github.com/airbytehq/airbyte/pull/10212) | Execute COPY command in parallel for S3 and GCS staging | | 0.4.12 | 2022-02-15 | [\#10342](https://github.com/airbytehq/airbyte/pull/10342) | Use connection pool, and fix connection leak. | From 39f8e0a01e5e4661b6195b93724330f96dd7f3df Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 25 Feb 2022 08:52:19 -0800 Subject: [PATCH 49/60] Update changelog --- docs/integrations/destinations/mssql.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md index 783d7f6fbfab6..8346c156fb1de 100644 --- a/docs/integrations/destinations/mssql.md +++ b/docs/integrations/destinations/mssql.md @@ -139,6 +139,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.1.5 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.1.4 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.3 | 2021-12-28 | [\#9158](https://github.com/airbytehq/airbyte/pull/9158) | Update connector fields title/description | | 0.1.2 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | From 9d9bf305aceddb0d1e1d65691dfb5078815f9b19 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 25 Feb 2022 08:53:10 -0800 Subject: [PATCH 50/60] update changelog --- docs/integrations/destinations/mysql.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/integrations/destinations/mysql.md b/docs/integrations/destinations/mysql.md index 86d0840bb9350..26ab85bbf7f2c 100644 --- a/docs/integrations/destinations/mysql.md +++ b/docs/integrations/destinations/mysql.md @@ -127,7 +127,8 @@ Using this feature requires additional configuration, when creating the destinat | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------| :--- | -| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.2 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | | 0.1.1 | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | | 0.1.0 | 06.10.2021 | [\#6763](https://github.com/airbytehq/airbyte/pull/6763) | Added destination-mysql-strict-encrypt that supports SSL connections only. | From 73f6d52d494187a2b4291f77394ac0641cb9d253 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 25 Feb 2022 09:09:22 -0800 Subject: [PATCH 51/60] Bump versions --- .../destination-clickhouse-strict-encrypt/Dockerfile | 2 +- .../connectors/destination-clickhouse/Dockerfile | 2 +- .../connectors/destination-mssql-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/destination-mssql/Dockerfile | 2 +- .../connectors/destination-mysql-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/destination-mysql/Dockerfile | 2 +- .../connectors/destination-oracle-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/destination-oracle/Dockerfile | 2 +- .../connectors/destination-postgres-strict-encrypt/Dockerfile | 2 +- .../connectors/destination-postgres/Dockerfile | 2 +- .../connectors/destination-redshift/Dockerfile | 2 +- .../connectors/destination-snowflake/Dockerfile | 4 ++-- docs/integrations/destinations/redshift.md | 2 +- docs/integrations/destinations/snowflake.md | 2 +- 14 files changed, 15 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile index f6069d7e1c60f..9fc1aa0f33db4 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index f62452f078213..dd60f557cc1c9 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-clickhouse diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile index 7cc7a7f59db19..7b094ef7293bf 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/destination-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mssql/Dockerfile b/airbyte-integrations/connectors/destination-mssql/Dockerfile index a48db180356c4..027c0699ecf1d 100644 --- a/airbyte-integrations/connectors/destination-mssql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.version=0.1.15 LABEL io.airbyte.name=airbyte/destination-mssql diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile index 530eb36c0b61b..ad65ba60ef064 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mysql/Dockerfile b/airbyte-integrations/connectors/destination-mysql/Dockerfile index b137bdd41e32b..bc324b2bff11f 100644 --- a/airbyte-integrations/connectors/destination-mysql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.17 +LABEL io.airbyte.version=0.1.18 LABEL io.airbyte.name=airbyte/destination-mysql diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile index bfbd7c875274e..f7ed56838c5fd 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-oracle-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-oracle/Dockerfile b/airbyte-integrations/connectors/destination-oracle/Dockerfile index 408b559fa4fa2..4bf980d8c5c64 100644 --- a/airbyte-integrations/connectors/destination-oracle/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-oracle COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.version=0.1.15 LABEL io.airbyte.name=airbyte/destination-oracle diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile index d7a514ace0319..1c32dea0e209b 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-postgres/Dockerfile b/airbyte-integrations/connectors/destination-postgres/Dockerfile index a58a4cfb3e532..4bc95fceba408 100644 --- a/airbyte-integrations/connectors/destination-postgres/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.14 +LABEL io.airbyte.version=0.3.15 LABEL io.airbyte.name=airbyte/destination-postgres diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index d909bb1d72e68..d6d8d7ad36597 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.26 +LABEL io.airbyte.version=0.3.27 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index 5912f54ed3ba0..6473177947fb5 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -18,8 +18,8 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -ENV APPLICATION_VERSION 0.4.16 +ENV APPLICATION_VERSION 0.4.17 ENV ENABLE_SENTRY true -LABEL io.airbyte.version=0.4.16 +LABEL io.airbyte.version=0.4.17 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index eabed6f7d7220..17d5c6fb3ed6d 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -124,7 +124,7 @@ All Redshift connections are encrypted using SSL | Version | Date | Pull Request | Subject | |:--------| :-------- | :----- | :------ | -| 0.3.26 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.3.27 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.3.25 | 2022-02-14 | [#9920](https://github.com/airbytehq/airbyte/pull/9920) | Updated the size of staging files for S3 staging. Also, added closure of S3 writers to staging files when data has been written to an staging file. | | 0.3.24 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.3.23 | 2021-12-16 | [\#8855](https://github.com/airbytehq/airbyte/pull/8855) | Add `purgeStagingData` option to enable/disable deleting the staging data | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index 554f958f718fd..8f0b165cfce7f 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -224,7 +224,7 @@ Finally, you need to add read/write permissions to your bucket with that email. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----- | :------ | -| 0.4.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.4.17 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.4.14 | 2022-02-17 | [\#10394](https://github.com/airbytehq/airbyte/pull/10394) | Reduce memory footprint. | | 0.4.13 | 2022-02-16 | [\#10212](https://github.com/airbytehq/airbyte/pull/10212) | Execute COPY command in parallel for S3 and GCS staging | | 0.4.12 | 2022-02-15 | [\#10342](https://github.com/airbytehq/airbyte/pull/10342) | Use connection pool, and fix connection leak. | From 91b5ebe989bba644e119c87793abc4c91071a61e Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 25 Feb 2022 10:11:26 -0800 Subject: [PATCH 52/60] bump version --- .../connectors/destination-mariadb-columnstore/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile b/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile index 0919436b17ef8..f33e8ba0b8d4d 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mariadb-columnstore COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-mariadb-columnstore From cf8128db938cee71f257d5c0c4adeab82d0a070e Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 25 Feb 2022 11:55:07 -0800 Subject: [PATCH 53/60] disable dbt support --- .../oracle/UnencryptedOracleDestinationAcceptanceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java index 8e57e31ef7ffc..f072506204abe 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java @@ -75,7 +75,7 @@ protected boolean implementsNamespaces() { @Override protected boolean supportsDBT() { - return true; + return false; } @Override From 830d86a44fda963ba7be2681edeb82484085ef8d Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 25 Feb 2022 13:34:41 -0800 Subject: [PATCH 54/60] update spec --- .../destination-oracle/src/main/resources/spec.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json index 3acaa8b981ae1..9422b0958de98 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json @@ -2,8 +2,11 @@ "documentationUrl": "https://docs.airbyte.io/integrations/destinations/oracle", "supportsIncremental": true, "supportsNormalization": false, - "supportsDBT": true, - "supported_destination_sync_modes": ["overwrite", "append"], + "supportsDBT": false, + "supported_destination_sync_modes": [ + "overwrite", + "append" + ], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Oracle Destination Spec", From 9300bc2d4240fa93cf42513a1e2486a18b7ff851 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 25 Feb 2022 14:57:39 -0800 Subject: [PATCH 55/60] update other oracle tests --- ...cleStrictEncryptDestinationAcceptanceTest.java | 15 ++++++++------- .../SshOracleDestinationAcceptanceTest.java | 2 +- ...nencryptedOracleDestinationAcceptanceTest.java | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java index 05b2a6d9eb435..04ab33055b98d 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java @@ -15,6 +15,7 @@ import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.destination.oracle.OracleDestination; import io.airbyte.integrations.destination.oracle.OracleNameTransformer; @@ -77,7 +78,7 @@ protected boolean implementsNamespaces() { @Override protected boolean supportsDBT() { - return true; + return false; } @Override @@ -113,8 +114,8 @@ private List retrieveRecordsFromTable(final String tableName, final St throws SQLException { final List result = getDatabase(config) .query(ctx -> ctx.fetch( - String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, - OracleDestination.COLUMN_NAME_EMITTED_AT)) + String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, + OracleDestination.COLUMN_NAME_EMITTED_AT)) .stream() .collect(Collectors.toList())); return result @@ -176,9 +177,9 @@ public void testEncryption() throws SQLException { config.get("port").asText(), config.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", - "oracle.net.encryption_client=REQUIRED;" + + JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + "oracle.net.encryption_types_client=( " - + algorithm + " )"); + + algorithm + " )")); final String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -202,9 +203,9 @@ public void testCheckProtocol() throws SQLException { clone.get("port").asText(), clone.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", - "oracle.net.encryption_client=REQUIRED;" + + JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + "oracle.net.encryption_types_client=( " - + algorithm + " )"); + + algorithm + " )")); final String network_service_banner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.query(network_service_banner).collect(Collectors.toList()); diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java index 1646e2f1dc0e8..e2e9054a7f4be 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java @@ -175,7 +175,7 @@ protected void tearDown(final TestDestinationEnv testEnv) throws Exception { @Override protected boolean supportsDBT() { - return true; + return false; } @Override diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java index f072506204abe..3201356782b27 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java @@ -111,8 +111,8 @@ private List retrieveRecordsFromTable(final String tableName, final St throws SQLException { final List result = getDatabase(config) .query(ctx -> ctx.fetch( - String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, - OracleDestination.COLUMN_NAME_EMITTED_AT)) + String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, + OracleDestination.COLUMN_NAME_EMITTED_AT)) .stream() .collect(Collectors.toList())); return result From 8150184589c6ee1a456200cf2dda8c92483133a8 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 25 Feb 2022 16:21:08 -0800 Subject: [PATCH 56/60] update doc --- docs/integrations/destinations/oracle.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md index 7c5d3c6fd953a..c2d244980c6f4 100644 --- a/docs/integrations/destinations/oracle.md +++ b/docs/integrations/destinations/oracle.md @@ -92,7 +92,7 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| -| 0.1.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.1.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling and remove DBT support | | 0.1.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.13 | 2021-12-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description | | 0.1.12 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | @@ -108,9 +108,9 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv ### Changelog (Strict Encrypt) -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:--------------------------------------------------------| :--- | -| 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | -| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.2 | 2021-01-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description | -| 0.1.1 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:--------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------| +| 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling and remove DBT support | +| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.2 | 2021-01-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description | +| 0.1.1 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | From 619d11fedcd5452b08a6d1b9365d0cf5c4259867 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 25 Feb 2022 18:07:07 -0800 Subject: [PATCH 57/60] bump seed --- .../resources/seed/destination_definitions.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index fd7b5772272cc..72d7d05a87247 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -37,7 +37,7 @@ - name: Clickhouse destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 dockerRepository: airbyte/destination-clickhouse - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse - name: DynamoDB destinationDefinitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89 @@ -108,7 +108,7 @@ - name: MS SQL Server destinationDefinitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf dockerRepository: airbyte/destination-mssql - dockerImageTag: 0.1.14 + dockerImageTag: 0.1.15 documentationUrl: https://docs.airbyte.io/integrations/destinations/mssql icon: mssql.svg - name: MeiliSearch @@ -126,19 +126,19 @@ - name: MySQL destinationDefinitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 dockerRepository: airbyte/destination-mysql - dockerImageTag: 0.1.17 + dockerImageTag: 0.1.18 documentationUrl: https://docs.airbyte.io/integrations/destinations/mysql icon: mysql.svg - name: Oracle destinationDefinitionId: 3986776d-2319-4de9-8af8-db14c0996e72 dockerRepository: airbyte/destination-oracle - dockerImageTag: 0.1.13 + dockerImageTag: 0.1.15 documentationUrl: https://docs.airbyte.io/integrations/destinations/oracle icon: oracle.svg - name: Postgres destinationDefinitionId: 25c5221d-dce2-4163-ade9-739ef790f503 dockerRepository: airbyte/destination-postgres - dockerImageTag: 0.3.14 + dockerImageTag: 0.3.15 documentationUrl: https://docs.airbyte.io/integrations/destinations/postgres icon: postgresql.svg - name: Pulsar @@ -162,7 +162,7 @@ - name: Redshift destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc dockerRepository: airbyte/destination-redshift - dockerImageTag: 0.3.26 + dockerImageTag: 0.3.27 documentationUrl: https://docs.airbyte.io/integrations/destinations/redshift icon: redshift.svg - name: Rockset @@ -185,7 +185,7 @@ - name: Snowflake destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba dockerRepository: airbyte/destination-snowflake - dockerImageTag: 0.4.16 + dockerImageTag: 0.4.17 documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake icon: snowflake.svg resourceRequirements: @@ -207,7 +207,7 @@ - name: MariaDB ColumnStore destinationDefinitionId: 294a4790-429b-40ae-9516-49826b9702e1 dockerRepository: airbyte/destination-mariadb-columnstore - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/destinations/mariadb-columnstore icon: mariadb.svg - name: Streamr From 6fe337ad3b4b6f6d5adf6b8884670ce191497ff2 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 28 Feb 2022 08:10:34 -0800 Subject: [PATCH 58/60] fix source test --- .../source/jdbc/test/JdbcSourceAcceptanceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java index 02ab8faef80bc..8511ad9d05206 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -24,6 +24,7 @@ import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcSourceOperations; +import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.relationaldb.models.DbState; @@ -196,7 +197,7 @@ public void setup() throws Exception { jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, jdbcConfig.get("jdbc_url").asText(), getDriverClass(), - jdbcConfig.has("connection_properties") ? jdbcConfig.get("connection_properties").asText() : null); + JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties")); if (supportsSchemas()) { createSchemas(); From c3c097769974ce82fbbfbd47aebf65db1acbf446 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 28 Feb 2022 08:14:40 -0800 Subject: [PATCH 59/60] update seed spec file --- .../main/resources/seed/destination_specs.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 091d757ed3a3b..1d54b454797d5 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -667,7 +667,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-clickhouse:0.1.3" +- dockerImage: "airbyte/destination-clickhouse:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: @@ -2077,7 +2077,7 @@ supportsDBT: false supported_destination_sync_modes: - "append" -- dockerImage: "airbyte/destination-mssql:0.1.14" +- dockerImage: "airbyte/destination-mssql:0.1.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql" connectionSpecification: @@ -2464,7 +2464,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-mysql:0.1.17" +- dockerImage: "airbyte/destination-mysql:0.1.18" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mysql" connectionSpecification: @@ -2629,7 +2629,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-oracle:0.1.13" +- dockerImage: "airbyte/destination-oracle:0.1.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/oracle" connectionSpecification: @@ -2853,11 +2853,11 @@ order: 4 supportsIncremental: true supportsNormalization: false - supportsDBT: true + supportsDBT: false supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-postgres:0.3.14" +- dockerImage: "airbyte/destination-postgres:0.3.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/postgres" connectionSpecification: @@ -3272,7 +3272,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-redshift:0.3.26" +- dockerImage: "airbyte/destination-redshift:0.3.27" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/redshift" connectionSpecification: @@ -3825,7 +3825,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-snowflake:0.4.16" +- dockerImage: "airbyte/destination-snowflake:0.4.17" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake" connectionSpecification: @@ -4078,7 +4078,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-mariadb-columnstore:0.1.3" +- dockerImage: "airbyte/destination-mariadb-columnstore:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mariadb-columnstore" connectionSpecification: From c327c0b172611738c8adab9d9b6bd68d0c97e230 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 28 Feb 2022 08:39:31 -0800 Subject: [PATCH 60/60] fix expected spec --- .../src/test/resources/expected_spec.json | 47 +++++++++++++++---- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json index 3d15378c40808..d27dea83a7f41 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json @@ -3,12 +3,21 @@ "supportsIncremental": true, "supportsNormalization": true, "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], + "supported_destination_sync_modes": [ + "overwrite", + "append", + "append_dedup" + ], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "ClickHouse Destination Spec", "type": "object", - "required": ["host", "port", "database", "username"], + "required": [ + "host", + "port", + "database", + "username" + ], "additionalProperties": true, "properties": { "host": { @@ -24,27 +33,41 @@ "minimum": 0, "maximum": 65536, "default": 8123, - "examples": ["8123"], + "examples": [ + "8123" + ], "order": 1 }, + "tcp-port": { + "title": "Native Port", + "description": "Native port (not the JDBC) of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 9000, + "examples": [ + "9000" + ], + "order": 2 + }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", - "order": 2 + "order": 3 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", - "order": 3 + "order": 4 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, - "order": 4 + "order": 5 }, "tunnel_method": { "type": "object", @@ -53,7 +76,9 @@ "oneOf": [ { "title": "No Tunnel", - "required": ["tunnel_method"], + "required": [ + "tunnel_method" + ], "properties": { "tunnel_method": { "description": "No ssh tunnel needed to connect to database", @@ -92,7 +117,9 @@ "minimum": 0, "maximum": 65536, "default": 22, - "examples": ["22"], + "examples": [ + "22" + ], "order": 2 }, "tunnel_user": { @@ -140,7 +167,9 @@ "minimum": 0, "maximum": 65536, "default": 22, - "examples": ["22"], + "examples": [ + "22" + ], "order": 2 }, "tunnel_user": {