From 3100d9789ded61101b405b12b1360543f9a33fae Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Wed, 14 Feb 2024 11:55:06 -0800 Subject: [PATCH] source-tidb: adopt CDK 0.20.4 (#35218) --- .../connectors/source-tidb/build.gradle | 25 ++------ .../connectors/source-tidb/metadata.yaml | 2 +- .../source/tidb/TiDBSourceAcceptanceTest.java | 59 ++++--------------- .../tidb/TiDBJdbcSourceAcceptanceTest.java | 10 +--- .../source/tidb/TiDBSourceTests.java | 45 -------------- .../source/tidb/TiDBTestDatabase.java | 49 ++++++++------- docs/integrations/sources/tidb.md | 1 + 7 files changed, 46 insertions(+), 145 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBSourceTests.java rename airbyte-integrations/connectors/source-tidb/src/{test => testFixtures}/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java (54%) diff --git a/airbyte-integrations/connectors/source-tidb/build.gradle b/airbyte-integrations/connectors/source-tidb/build.gradle index 7b59b6a2bab7b..8fe8b49931d2f 100644 --- a/airbyte-integrations/connectors/source-tidb/build.gradle +++ b/airbyte-integrations/connectors/source-tidb/build.gradle @@ -1,39 +1,22 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.13.2' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileTestJava { - options.compilerArgs.remove("-Werror") - } - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.tidb.TiDBSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { + implementation 'mysql:mysql-connector-java:8.0.33' - //TODO Add jdbc driver import here. Ex: implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' - implementation 'mysql:mysql-connector-java:8.0.22' - - testImplementation libs.testcontainers.tidb.source - - testImplementation 'org.apache.commons:commons-lang3:3.11' + testFixturesApi 'org.testcontainers:tidb:1.19.4' - integrationTestJavaImplementation libs.testcontainers.tidb.source + testImplementation 'org.hamcrest:hamcrest-all:1.3' } diff --git a/airbyte-integrations/connectors/source-tidb/metadata.yaml b/airbyte-integrations/connectors/source-tidb/metadata.yaml index 554020d41e1b0..1d277882a8e0c 100644 --- a/airbyte-integrations/connectors/source-tidb/metadata.yaml +++ b/airbyte-integrations/connectors/source-tidb/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: database connectorType: source definitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae - dockerImageTag: 0.3.1 + dockerImageTag: 0.3.2 dockerRepository: airbyte/source-tidb githubIssueLabel: source-tidb icon: tidb.svg diff --git a/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java index 7c402f0b86fbd..86cb85232832d 100644 --- a/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java @@ -5,16 +5,10 @@ package io.airbyte.integrations.source.tidb; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import io.airbyte.cdk.db.Database; -import io.airbyte.cdk.db.factory.DSLContextFactory; -import io.airbyte.cdk.db.factory.DatabaseDriver; -import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.base.ssh.SshHelpers; import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; @@ -25,11 +19,8 @@ import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; import java.util.HashMap; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; import org.junit.jupiter.api.Disabled; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.utility.DockerImageName; +import org.testcontainers.tidb.TiDBContainer; @Disabled public class TiDBSourceAcceptanceTest extends SourceAcceptanceTest { @@ -37,46 +28,22 @@ public class TiDBSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "public.starships"; - protected GenericContainer container; - protected JsonNode config; + protected TiDBContainer container = TiDBTestDatabase.container(); + protected TiDBTestDatabase testdb; @Override protected void setupEnvironment(final TestDestinationEnv testEnv) throws Exception { - container = new GenericContainer(DockerImageName.parse("pingcap/tidb:nightly")) - .withExposedPorts(4000); - container.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) - .put(JdbcUtils.USERNAME_KEY, "root") - .put(JdbcUtils.DATABASE_KEY, "test") - .build()); - - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - "", - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } + testdb = new TiDBTestDatabase(container) + .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');") + .with("CREATE TABLE starships(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); + testdb.with("DROP TABLE id_and_name;") + .with("DROP TABLE starships;"); } @Override @@ -91,7 +58,7 @@ protected ConnectorSpecification getSpec() throws Exception { @Override protected JsonNode getConfig() { - return config; + return testdb.integrationTestConfigBuilder().build(); } @Override @@ -102,7 +69,7 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withCursorField(Lists.newArrayList("id")) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s.%s", config.get(JdbcUtils.DATABASE_KEY).asText(), STREAM_NAME), + String.format("%s.%s", testdb.getDatabaseName(), STREAM_NAME), Field.of("id", JsonSchemaType.NUMBER), Field.of("name", JsonSchemaType.STRING)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), @@ -111,7 +78,7 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withCursorField(Lists.newArrayList("id")) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s.%s", config.get(JdbcUtils.DATABASE_KEY).asText(), STREAM_NAME2), + String.format("%s.%s", testdb.getDatabaseName(), STREAM_NAME2), Field.of("id", JsonSchemaType.NUMBER), Field.of("name", JsonSchemaType.STRING)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); diff --git a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBJdbcSourceAcceptanceTest.java index e3e2728f7aaf5..fff46004d7e10 100755 --- a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBJdbcSourceAcceptanceTest.java @@ -7,13 +7,12 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.commons.json.Jsons; -import org.junit.jupiter.api.Disabled; import org.testcontainers.tidb.TiDBContainer; -import org.testcontainers.utility.DockerImageName; -@Disabled class TiDBJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { + private final TiDBContainer container = TiDBTestDatabase.container(); + @Override public boolean supportsSchemas() { return false; @@ -21,7 +20,7 @@ public boolean supportsSchemas() { @Override public JsonNode config() { - return Jsons.clone(testdb.configBuilder().build()); + return Jsons.clone(testdb.testConfigBuilder().build()); } @Override @@ -31,9 +30,6 @@ protected TiDBSource source() { @Override protected TiDBTestDatabase createTestDatabase() { - final TiDBContainer container = new TiDBContainer(DockerImageName.parse("pingcap/tidb:nightly")) - .withExposedPorts(4000); - container.start(); return new TiDBTestDatabase(container).initialized(); } diff --git a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBSourceTests.java b/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBSourceTests.java deleted file mode 100644 index d2ceb40e6bc4d..0000000000000 --- a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBSourceTests.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.tidb; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.utility.DockerImageName; - -@Disabled -public class TiDBSourceTests { - - private JsonNode config; - private GenericContainer container; - - @Test - public void testSettingTimezones() throws Exception { - container = new GenericContainer(DockerImageName.parse("pingcap/tidb:nightly")) - .withExposedPorts(4000); - - container.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, "127.0.0.1") - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.USERNAME_KEY, "root") - .put(JdbcUtils.DATABASE_KEY, "test") - .build()); - - final AirbyteConnectionStatus check = new TiDBSource().check(config); - - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, check.getStatus()); - container.close(); - } - -} diff --git a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java b/airbyte-integrations/connectors/source-tidb/src/testFixtures/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java similarity index 54% rename from airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java rename to airbyte-integrations/connectors/source-tidb/src/testFixtures/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java index 26da58c7eff18..17b238db5d29f 100644 --- a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java +++ b/airbyte-integrations/connectors/source-tidb/src/testFixtures/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java @@ -5,42 +5,38 @@ package io.airbyte.integrations.source.tidb; import io.airbyte.cdk.db.factory.DatabaseDriver; -import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.ContainerFactory; import io.airbyte.cdk.testutils.TestDatabase; import java.util.stream.Stream; import org.jooq.SQLDialect; import org.testcontainers.tidb.TiDBContainer; +import org.testcontainers.utility.DockerImageName; public class TiDBTestDatabase extends - TestDatabase { - - protected static final String USER = "root"; - protected static final String DATABASE = "test"; - private final TiDBContainer container; + TestDatabase { protected TiDBTestDatabase(final TiDBContainer container) { super(container); - this.container = container; } @Override - public String getJdbcUrl() { - return container.getJdbcUrl(); + public String withNamespace(String name) { + return name; } @Override public String getDatabaseName() { - return DATABASE; + return getContainer().getDatabaseName(); } @Override public String getUserName() { - return container.getUsername(); + return getContainer().getUsername(); } @Override public String getPassword() { - return container.getPassword(); + return getContainer().getPassword(); } @Override @@ -64,25 +60,28 @@ public SQLDialect getSqlDialect() { } @Override - public void close() { - container.close(); - } - - @Override - public TiDBDbConfigBuilder configBuilder() { - return new TiDBDbConfigBuilder(this) - .with(JdbcUtils.HOST_KEY, "127.0.0.1") - .with(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .with(JdbcUtils.USERNAME_KEY, USER) - .with(JdbcUtils.DATABASE_KEY, DATABASE); + public TiDBConfigBuilder configBuilder() { + return new TiDBConfigBuilder(this); } - static public class TiDBDbConfigBuilder extends TestDatabase.ConfigBuilder { + static public class TiDBConfigBuilder extends ConfigBuilder { - protected TiDBDbConfigBuilder(final TiDBTestDatabase testdb) { + protected TiDBConfigBuilder(final TiDBTestDatabase testdb) { super(testdb); } } + static public TiDBContainer container() { + var factory = new ContainerFactory() { + + @Override + protected TiDBContainer createNewContainer(DockerImageName dockerImageName) { + return new TiDBContainer(dockerImageName).withExposedPorts(4000); + } + + }; + return factory.exclusive("pingcap/tidb:nightly"); + } + } diff --git a/docs/integrations/sources/tidb.md b/docs/integrations/sources/tidb.md index d29e5ba0e8754..197673c5e8cb6 100644 --- a/docs/integrations/sources/tidb.md +++ b/docs/integrations/sources/tidb.md @@ -128,6 +128,7 @@ Now that you have set up the TiDB source connector, check out the following TiDB | Version | Date | Pull Request | Subject | |:--------|:-----------| :----------- |-------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.2 | 2024-02-13 | [35218](https://github.com/airbytehq/airbyte/pull/35218) | Adopt CDK 0.20.4 | | 0.3.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | | 0.3.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Remove LEGACY state | | 0.2.5 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase |