Skip to content

Commit

Permalink
Fixes Spotless and runs spotless (#13040)
Browse files Browse the repository at this point in the history
zipped files with JSON extension made this task sad
  • Loading branch information
supertopher authored May 20, 2022
1 parent 3cf2e05 commit 013a886
Show file tree
Hide file tree
Showing 95 changed files with 327 additions and 333 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -137,9 +137,7 @@ private static JdbcDatabase getDatabase(final JsonNode config) {
config.get("username").asText(),
config.has("password") ? config.get("password").asText() : null,
ClickhouseDestination.DRIVER_CLASS,
jdbcStr
)
);
jdbcStr));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
import io.airbyte.integrations.base.JavaBaseConstants;
import io.airbyte.integrations.destination.ExtendedNameTransformer;
import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
import java.sql.SQLException;
import java.util.List;
import java.util.stream.Collectors;
import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
import org.junit.jupiter.api.Disabled;
import org.testcontainers.containers.ClickHouseContainer;

Expand Down Expand Up @@ -136,9 +136,7 @@ private static JdbcDatabase getDatabase(final JsonNode config) {
String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(),
config.get("host").asText(),
config.get("port").asInt(),
config.get("database").asText())
)
);
config.get("database").asText())));
}

@Override
Expand Down Expand Up @@ -197,4 +195,5 @@ public void testSyncWithNormalization(final String messagesFilename, final Strin
public void specNormalizationValueShouldBeCorrect() throws Exception {
super.specNormalizationValueShouldBeCorrect();
}

}
Original file line number Diff line number Diff line change
@@ -1,25 +1,29 @@
/*
* Copyright (c) 2021 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.clickhouse;

import io.airbyte.integrations.destination.ExtendedNameTransformer;
import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;

import java.util.ArrayList;
import java.util.List;

public class ClickhouseTestDataComparator extends AdvancedTestDataComparator {

private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer();
private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer();

@Override
protected List<String> resolveIdentifier(final String identifier) {
final List<String> result = new ArrayList<>();
final String resolved = namingResolver.getIdentifier(identifier);
result.add(identifier);
result.add(resolved);
if (!resolved.startsWith("\"")) {
result.add(resolved.toLowerCase());
result.add(resolved.toUpperCase());
}
return result;
@Override
protected List<String> resolveIdentifier(final String identifier) {
final List<String> result = new ArrayList<>();
final String resolved = namingResolver.getIdentifier(identifier);
result.add(identifier);
result.add(resolved);
if (!resolved.startsWith("\"")) {
result.add(resolved.toLowerCase());
result.add(resolved.toUpperCase());
}
return result;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,10 @@
import io.airbyte.integrations.base.ssh.SshTunnel;
import io.airbyte.integrations.destination.ExtendedNameTransformer;
import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
import org.junit.jupiter.api.Disabled;
import org.testcontainers.containers.ClickHouseContainer;

Expand Down Expand Up @@ -154,9 +153,7 @@ private static JdbcDatabase getDatabase(final JsonNode config) {
String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(),
config.get("host").asText(),
config.get("port").asInt(),
config.get("database").asText())
)
);
config.get("database").asText())));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,7 @@ void sanityTest() throws Exception {
String.format("jdbc:clickhouse://%s:%s/%s",
config.get("host").asText(),
config.get("port").asText(),
config.get("database").asText())
)
);
config.get("database").asText())));

final List<JsonNode> actualRecords = database.bufferedResultSetQuery(
connection -> connection.createStatement().executeQuery(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,5 @@ protected void cleanUpObjects(final String bucket, final List<KeyVersion> keysTo
protected Map<String, String> getMetadataMapping() {
return new HashMap<>();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,7 @@ protected DataSource getDataSource(final JsonNode config) {
jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null,
driverClass,
jdbcConfig.get("jdbc_url").asText(),
getConnectionProperties(config)
);
getConnectionProperties(config));
}

protected JdbcDatabase getDatabase(final DataSource dataSource) {
Expand Down Expand Up @@ -132,7 +131,8 @@ private void assertCustomParametersDontOverwriteDefaultParameters(final Map<Stri
public AirbyteMessageConsumer getConsumer(final JsonNode config,
final ConfiguredAirbyteCatalog catalog,
final Consumer<AirbyteMessage> outputRecordCollector) {
return JdbcBufferedConsumerFactory.create(outputRecordCollector, getDatabase(getDataSource(config)), sqlOperations, namingResolver, config, catalog);
return JdbcBufferedConsumerFactory.create(outputRecordCollector, getDatabase(getDataSource(config)), sqlOperations, namingResolver, config,
catalog);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,8 @@ private OnCloseFunction onCloseFunction(final JdbcDatabase database,
}
queryList.add(stagingOperations.copyTableQuery(database, schemaName, srcTableName, dstTableName));
}
stagingOperations.onDestinationCloseOperations(database, writeConfigs.stream().map(WriteConfig::getOutputSchemaName).collect(Collectors.toSet()));
stagingOperations.onDestinationCloseOperations(database,
writeConfigs.stream().map(WriteConfig::getOutputSchemaName).collect(Collectors.toSet()));
LOGGER.info("Executing finalization of tables.");
stagingOperations.executeTransaction(database, queryList);
LOGGER.info("Finalizing tables in destination completed.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,7 @@ void setup() {
config.get("username").asText(),
config.get("password").asText(),
DatabaseDriver.POSTGRESQL.getDriverClassName(),
config.get("jdbc_url").asText()
)
);
config.get("jdbc_url").asText()));

uuidSupplier = mock(Supplier.class);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,7 @@ private static JdbcDatabase getDatabase(final JsonNode config) {
String.format(DatabaseDriver.MARIADB.getUrlFormatString(),
config.get("host").asText(),
config.get("port").asInt(),
config.get("database").asText())
)
);
config.get("database").asText())));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,8 @@ private static Database getDatabaseFromConfig(final JsonNode config) {
String.format(DatabaseDriver.MARIADB.getUrlFormatString(),
config.get("host").asText(),
config.get("port").asInt(),
config.get("database").asText()), SQLDialect.MARIADB);
config.get("database").asText()),
SQLDialect.MARIADB);
return new Database(dslContext);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import java.util.List;
import java.util.stream.Collectors;
import org.jooq.DSLContext;
import org.jooq.SQLDialect;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -128,15 +127,15 @@ protected List<String> resolveIdentifier(final String identifier) {
private List<JsonNode> retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException {
final DSLContext dslContext = DatabaseConnectionHelper.createDslContext(db, null);
return new Database(dslContext).query(
ctx -> {
ctx.fetch(String.format("USE %s;", config.get("database")));
return ctx
.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT))
.stream()
.map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat()))
.map(Jsons::deserialize)
.collect(Collectors.toList());
});
ctx -> {
ctx.fetch(String.format("USE %s;", config.get("database")));
return ctx
.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT))
.stream()
.map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat()))
.map(Jsons::deserialize)
.collect(Collectors.toList());
});
}

private static DSLContext getDslContext(final JsonNode config) {
Expand All @@ -146,7 +145,8 @@ private static DSLContext getDslContext(final JsonNode config) {
DatabaseDriver.MSSQLSERVER.getDriverClassName(),
String.format("jdbc:sqlserver://%s:%d",
config.get("host").asText(),
config.get("port").asInt()), null);
config.get("port").asInt()),
null);
}

private static Database getDatabase(final DSLContext dslContext) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@
import io.airbyte.commons.json.Jsons;
import io.airbyte.commons.string.Strings;
import io.airbyte.db.Database;
import io.airbyte.db.factory.DSLContextFactory;
import io.airbyte.db.factory.DatabaseDriver;
import io.airbyte.integrations.base.JavaBaseConstants;
import io.airbyte.integrations.destination.ExtendedNameTransformer;
import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest;
import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
import io.airbyte.db.factory.DSLContextFactory;
import io.airbyte.db.factory.DatabaseDriver;
import io.airbyte.test.utils.DatabaseConnectionHelper;
import java.sql.SQLException;
import java.util.List;
Expand Down Expand Up @@ -125,7 +125,8 @@ private static DSLContext getDslContext(final JsonNode config) {
DatabaseDriver.MSSQLSERVER.getDriverClassName(),
String.format("jdbc:sqlserver://%s:%d",
config.get("host").asText(),
config.get("port").asInt()), null);
config.get("port").asInt()),
null);
}

private static Database getDatabase(final DSLContext dslContext) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,15 +126,15 @@ protected List<String> resolveIdentifier(final String identifier) {
private List<JsonNode> retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException {
final DSLContext dslContext = DatabaseConnectionHelper.createDslContext(db, SQLDialect.DEFAULT);
return new Database(dslContext).query(
ctx -> {
ctx.fetch(String.format("USE %s;", config.get("database")));
return ctx
.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT))
.stream()
.map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat()))
.map(Jsons::deserialize)
.collect(Collectors.toList());
});
ctx -> {
ctx.fetch(String.format("USE %s;", config.get("database")));
return ctx
.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT))
.stream()
.map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat()))
.map(Jsons::deserialize)
.collect(Collectors.toList());
});
}

@BeforeAll
Expand All @@ -151,7 +151,8 @@ private static DSLContext getDslContext(final JsonNode config) {
DatabaseDriver.MSSQLSERVER.getDriverClassName(),
String.format("jdbc:sqlserver://%s:%d",
config.get("host").asText(),
config.get("port").asInt()), null);
config.get("port").asInt()),
null);
}

private static Database getDatabase(final DSLContext dslContext) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import io.airbyte.commons.json.Jsons;
import io.airbyte.db.Database;
import io.airbyte.db.factory.DSLContextFactory;
import io.airbyte.db.factory.DataSourceFactory;
import io.airbyte.db.factory.DatabaseDriver;
import io.airbyte.db.jdbc.JdbcUtils;
import io.airbyte.integrations.base.JavaBaseConstants;
Expand All @@ -23,7 +22,6 @@
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.sql.DataSource;
import org.apache.commons.lang3.RandomStringUtils;
import org.jooq.DSLContext;
import org.testcontainers.containers.JdbcDatabaseContainer;
Expand Down Expand Up @@ -130,7 +128,8 @@ private static Database getDatabaseFromConfig(final JsonNode config) {
DatabaseDriver.MSSQLSERVER.getDriverClassName(),
String.format("jdbc:sqlserver://%s:%s",
config.get("host").asText(),
config.get("port").asInt()), null);
config.get("port").asInt()),
null);
return new Database(dslContext);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,8 @@ private static DSLContext getDslContext(final JsonNode config) {
String.format(DatabaseDriver.ORACLE.getUrlFormatString(),
config.get("host").asText(),
config.get("port").asInt(),
config.get("sid").asText()), null);
config.get("sid").asText()),
null);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,7 @@ public void testEncryption() throws SQLException {
config.get("host").asText(),
config.get("port").asInt(),
config.get("sid").asText()),
getAdditionalProperties(algorithm)
)
);
getAdditionalProperties(algorithm)));

final String networkServiceBanner =
"select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)";
Expand Down Expand Up @@ -78,9 +76,7 @@ public void testCheckProtocol() throws SQLException {
clone.get("host").asText(),
clone.get("port").asInt(),
clone.get("sid").asText()),
getAdditionalProperties(algorithm)
)
);
getAdditionalProperties(algorithm)));

final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual";
final List<JsonNode> collect = database.queryJsons(networkServiceBanner);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,8 @@ private Database getDatabaseFromConfig(final JsonNode config) {
String.format(DatabaseDriver.ORACLE.getUrlFormatString(),
config.get("host").asText(),
config.get("port").asInt(),
config.get("sid").asText()), null);
config.get("sid").asText()),
null);
return new Database(dslContext);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,8 @@ private List<JsonNode> retrieveRecordsFromTable(final String tableName, final St
try (final DSLContext dslContext = getDSLContext(config)) {
final List<org.jooq.Record> result = getDatabase(dslContext)
.query(ctx -> ctx.fetch(
String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName,
OracleDestination.COLUMN_NAME_EMITTED_AT))
String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName,
OracleDestination.COLUMN_NAME_EMITTED_AT))
.stream()
.collect(Collectors.toList()));
return result
Expand All @@ -142,7 +142,8 @@ private static DSLContext getDSLContext(final JsonNode config) {
String.format(DatabaseDriver.ORACLE.getUrlFormatString(),
config.get("host").asText(),
config.get("port").asInt(),
config.get("sid").asText()), null);
config.get("sid").asText()),
null);
}

private static Database getDatabase(final DSLContext dslContext) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,7 @@ private static Database getDatabaseFromConfig(final JsonNode config) {
config.get("host").asText(),
config.get("port").asInt(),
config.get("database").asText()),
SQLDialect.POSTGRES
)
);
SQLDialect.POSTGRES));
}

private List<JsonNode> retrieveRecordsFromTable(final String tableName, final String schemaName) throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
* {@link RedshiftInsertDestination} for more detail. The second inserts via streaming the data to
* an S3 bucket, and Cop-ing the date into Redshift. This is more efficient, and recommended for
* production workloads, but does require users to set up an S3 bucket and pass in additional
* credentials. See {@link RedshiftStagingS3Destination} for more detail. This class inspect the given
* arguments to determine which strategy to use.
* credentials. See {@link RedshiftStagingS3Destination} for more detail. This class inspect the
* given arguments to determine which strategy to use.
*/
public class RedshiftDestination extends SwitchingDestination<RedshiftDestination.DestinationType> {

Expand Down
Loading

0 comments on commit 013a886

Please sign in to comment.