Skip to content

Commit a8dc68e

Browse files
edgaoedgao
committed
mysql dv2 raw table impl
1 parent 97d1fa3 commit a8dc68e

File tree

51 files changed

+478
-46
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+478
-46
lines changed

‎airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ plugins {
66
airbyteJavaConnector {
77
cdkVersionRequired = '0.30.2'
88
features = ['db-destinations', 'typing-deduping']
9-
useLocalCdk = false
9+
useLocalCdk = true
1010
}
1111

1212
//remove once upgrading the CDK version to 0.4.x or later

‎airbyte-integrations/connectors/destination-mysql/build.gradle

+1-7
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ plugins {
66
airbyteJavaConnector {
77
cdkVersionRequired = '0.30.2'
88
features = ['db-destinations', 'typing-deduping']
9-
useLocalCdk = false
9+
useLocalCdk = true
1010
}
1111

1212
//remove once upgrading the CDK version to 0.4.x or later
@@ -27,9 +27,3 @@ dependencies {
2727
implementation 'mysql:mysql-connector-java:8.0.22'
2828
integrationTestJavaImplementation libs.testcontainers.mysql
2929
}
30-
31-
configurations.all {
32-
resolutionStrategy {
33-
force libs.jooq
34-
}
35-
}

‎airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java

+15-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination;
2222
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler;
2323
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator;
24+
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.NoOpJdbcDestinationHandler;
2425
import io.airbyte.commons.exceptions.ConnectionErrorException;
2526
import io.airbyte.commons.json.Jsons;
2627
import io.airbyte.commons.map.MoreMaps;
@@ -30,13 +31,15 @@
3031
import io.airbyte.integrations.base.destination.typing_deduping.migrators.Migration;
3132
import io.airbyte.integrations.base.destination.typing_deduping.migrators.MinimumDestinationState;
3233
import io.airbyte.integrations.destination.mysql.MySQLSqlOperations.VersionCompatibility;
34+
import io.airbyte.integrations.destination.mysql.typing_deduping.MysqlSqlGenerator;
3335
import io.airbyte.protocol.models.v0.AirbyteConnectionStatus;
3436
import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status;
3537
import java.util.Collections;
3638
import java.util.List;
3739
import java.util.Map;
3840
import javax.sql.DataSource;
3941
import org.jetbrains.annotations.NotNull;
42+
import org.jooq.SQLDialect;
4043
import org.slf4j.Logger;
4144
import org.slf4j.LoggerFactory;
4245

@@ -141,14 +144,24 @@ public JsonNode toJdbcConfig(final JsonNode config) {
141144

142145
@Override
143146
protected JdbcSqlGenerator getSqlGenerator(final JsonNode config) {
144-
throw new UnsupportedOperationException("mysql does not yet support DV2");
147+
return new MysqlSqlGenerator();
145148
}
146149

147150
@Override
148151
protected StreamAwareDataTransformer getDataTransformer(ParsedCatalog parsedCatalog, String defaultNamespace) {
149152
return new PropertyNameSimplifyingDataTransformer();
150153
}
151154

155+
@Override
156+
public boolean isV2Destination() {
157+
return true;
158+
}
159+
160+
@Override
161+
protected boolean shouldAlwaysDisableTypeDedupe() {
162+
return true;
163+
}
164+
152165
public static void main(final String[] args) throws Exception {
153166
final Destination destination = MySQLDestination.sshWrappedDestination();
154167
LOGGER.info("starting destination: {}", MySQLDestination.class);
@@ -161,7 +174,7 @@ public static void main(final String[] args) throws Exception {
161174
protected JdbcDestinationHandler<MinimumDestinationState> getDestinationHandler(@NotNull String databaseName,
162175
@NotNull JdbcDatabase database,
163176
@NotNull String rawTableSchema) {
164-
throw new UnsupportedOperationException("Mysql does not yet support DV2");
177+
return new NoOpJdbcDestinationHandler<>(databaseName, database, rawTableSchema, SQLDialect.DEFAULT);
165178
}
166179

167180
@NotNull

‎airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java

+31-2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@
44

55
package io.airbyte.integrations.destination.mysql;
66

7+
import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT;
8+
import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT;
9+
import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META;
10+
import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID;
11+
import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA;
12+
713
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
814
import io.airbyte.cdk.db.jdbc.JdbcDatabase;
915
import io.airbyte.cdk.integrations.base.JavaBaseConstants;
@@ -56,7 +62,8 @@ protected void insertRecordsInternalV2(final JdbcDatabase database,
5662
final String schemaName,
5763
final String tableName)
5864
throws Exception {
59-
throw new UnsupportedOperationException("mysql does not yet support DV2");
65+
// TODO ... how does this actually work?
66+
insertRecordsInternal(database, records, schemaName, tableName);
6067
}
6168

6269
private void loadDataIntoTable(final JdbcDatabase database,
@@ -129,7 +136,7 @@ private boolean checkIfLocalFileIsEnabled(final JdbcDatabase database) throws SQ
129136
}
130137

131138
@Override
132-
public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) {
139+
protected String createTableQueryV1(String schemaName, String tableName) {
133140
// MySQL requires byte information with VARCHAR. Since we are using uuid as value for the column,
134141
// 256 is enough
135142
return String.format(
@@ -141,6 +148,28 @@ public String createTableQuery(final JdbcDatabase database, final String schemaN
141148
schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT);
142149
}
143150

151+
protected String createTableQueryV2(String schemaName, String tableName) {
152+
// MySQL requires byte information with VARCHAR. Since we are using uuid as value for the column,
153+
// 256 is enough
154+
return String.format(
155+
"""
156+
CREATE TABLE IF NOT EXISTS %s.%s (\s
157+
%s VARCHAR(256) PRIMARY KEY,
158+
%s JSON,
159+
%s TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP(6),
160+
%s TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP(6),
161+
%s JSON
162+
);
163+
""",
164+
schemaName,
165+
tableName,
166+
JavaBaseConstants.COLUMN_NAME_AB_RAW_ID,
167+
JavaBaseConstants.COLUMN_NAME_DATA,
168+
JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT,
169+
JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT,
170+
JavaBaseConstants.COLUMN_NAME_AB_META);
171+
}
172+
144173
public static class VersionCompatibility {
145174

146175
private final double version;

‎airbyte-integrations/connectors/destination-mysql/src/main/kotlin/io/airbyte/integrations/destination/mysql/typing_deduping/MysqlSqlGenerator.kt

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package io.airbyte.integrations.destination.mysql.typing_deduping
2+
3+
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.RawOnlySqlGenerator
4+
import io.airbyte.integrations.base.destination.typing_deduping.StreamId
5+
import io.airbyte.integrations.base.destination.typing_deduping.StreamId.Companion.concatenateRawTableName
6+
import io.airbyte.integrations.destination.mysql.MySQLNameTransformer
7+
8+
class MysqlSqlGenerator : RawOnlySqlGenerator(MySQLNameTransformer()) {
9+
10+
override fun buildStreamId(
11+
namespace: String,
12+
name: String,
13+
rawNamespaceOverride: String
14+
): StreamId {
15+
return StreamId(
16+
namingTransformer.getNamespace(namespace),
17+
namingTransformer.convertStreamName(name),
18+
namingTransformer.getNamespace(rawNamespaceOverride),
19+
// The default implementation is just convertStreamName(concatenate()).
20+
// Wrap in getIdentifier to also truncate.
21+
// This is probably only necessary because the mysql name transformer
22+
// doesn't call convertStreamName in getIdentifier (probably a bug?).
23+
// But that entire NameTransformer interface is a hot mess anyway.
24+
namingTransformer.getIdentifier(
25+
namingTransformer.convertStreamName(
26+
concatenateRawTableName(namespace, name),
27+
),
28+
),
29+
namespace,
30+
name,
31+
)
32+
}
33+
}

‎airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json

+6
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,12 @@
5858
"title": "JDBC URL Params",
5959
"type": "string",
6060
"order": 6
61+
},
62+
"raw_table_database": {
63+
"type": "string",
64+
"description": "The database to write raw tables into",
65+
"title": "Raw table database (defaults to airbyte_internal)",
66+
"order": 7
6167
}
6268
}
6369
}

‎airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java

+34-28
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,11 @@ protected boolean supportObjectDataTypeTest() {
8383

8484
@Override
8585
protected JsonNode getConfig() {
86-
return Jsons.jsonNode(ImmutableMap.builder()
86+
return getConfigFromTestContainer(db);
87+
}
88+
89+
public static ObjectNode getConfigFromTestContainer(final MySQLContainer<?> db) {
90+
return (ObjectNode) Jsons.jsonNode(ImmutableMap.builder()
8791
.put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db))
8892
.put(JdbcUtils.USERNAME_KEY, db.getUsername())
8993
.put(JdbcUtils.PASSWORD_KEY, db.getPassword())
@@ -132,23 +136,22 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
132136
}
133137

134138
private List<JsonNode> retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException {
135-
try (final DSLContext dslContext = DSLContextFactory.create(
139+
final DSLContext dslContext = DSLContextFactory.create(
136140
db.getUsername(),
137141
db.getPassword(),
138142
db.getDriverClassName(),
139143
String.format(DatabaseDriver.MYSQL.getUrlFormatString(),
140144
db.getHost(),
141145
db.getFirstMappedPort(),
142146
db.getDatabaseName()),
143-
SQLDialect.MYSQL)) {
144-
return new Database(dslContext).query(
145-
ctx -> ctx
146-
.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName,
147-
JavaBaseConstants.COLUMN_NAME_EMITTED_AT))
148-
.stream()
149-
.map(this::getJsonFromRecord)
150-
.collect(Collectors.toList()));
151-
}
147+
SQLDialect.MYSQL);
148+
return new Database(dslContext).query(
149+
ctx -> ctx
150+
.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName,
151+
JavaBaseConstants.COLUMN_NAME_EMITTED_AT))
152+
.stream()
153+
.map(this::getJsonFromRecord)
154+
.collect(Collectors.toList()));
152155
}
153156

154157
@Override
@@ -163,36 +166,39 @@ protected List<JsonNode> retrieveNormalizedRecords(final TestDestinationEnv test
163166
protected void setup(final TestDestinationEnv testEnv, final HashSet<String> TEST_SCHEMAS) {
164167
db = new MySQLContainer<>("mysql:8.0");
165168
db.start();
166-
setLocalInFileToTrue();
167-
revokeAllPermissions();
168-
grantCorrectPermissions();
169+
configureTestContainer(db);
170+
}
171+
172+
public static void configureTestContainer(final MySQLContainer<?> db) {
173+
setLocalInFileToTrue(db);
174+
revokeAllPermissions(db);
175+
grantCorrectPermissions(db);
169176
}
170177

171-
private void setLocalInFileToTrue() {
172-
executeQuery("set global local_infile=true");
178+
private static void setLocalInFileToTrue(final MySQLContainer<?> db) {
179+
executeQuery(db, "set global local_infile=true");
173180
}
174181

175-
private void revokeAllPermissions() {
176-
executeQuery("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + db.getUsername() + "@'%';");
182+
private static void revokeAllPermissions(final MySQLContainer<?> db) {
183+
executeQuery(db, "REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + db.getUsername() + "@'%';");
177184
}
178185

179-
private void grantCorrectPermissions() {
180-
executeQuery("GRANT ALTER, CREATE, INSERT, SELECT, DROP ON *.* TO " + db.getUsername() + "@'%';");
186+
private static void grantCorrectPermissions(final MySQLContainer<?> db) {
187+
executeQuery(db, "GRANT ALTER, CREATE, INSERT, INDEX, UPDATE, DELETE, SELECT, DROP ON *.* TO " + db.getUsername() + "@'%';");
181188
}
182189

183-
private void executeQuery(final String query) {
184-
try (final DSLContext dslContext = DSLContextFactory.create(
190+
private static void executeQuery(final MySQLContainer<?> db, final String query) {
191+
final DSLContext dslContext = DSLContextFactory.create(
185192
"root",
186193
"test",
187194
db.getDriverClassName(),
188195
String.format(DatabaseDriver.MYSQL.getUrlFormatString(),
189196
db.getHost(),
190197
db.getFirstMappedPort(),
191198
db.getDatabaseName()),
192-
SQLDialect.MYSQL)) {
193-
new Database(dslContext).query(
194-
ctx -> ctx
195-
.execute(query));
199+
SQLDialect.MYSQL);
200+
try {
201+
new Database(dslContext).query(ctx -> ctx.execute(query));
196202
} catch (final SQLException e) {
197203
throw new RuntimeException(e);
198204
}
@@ -208,7 +214,7 @@ protected void tearDown(final TestDestinationEnv testEnv) {
208214
@Test
209215
public void testCustomDbtTransformations() throws Exception {
210216
// We need to create view for testing custom dbt transformations
211-
executeQuery("GRANT CREATE VIEW ON *.* TO " + db.getUsername() + "@'%';");
217+
executeQuery(db, "GRANT CREATE VIEW ON *.* TO " + db.getUsername() + "@'%';");
212218
super.testCustomDbtTransformations();
213219
}
214220

@@ -330,7 +336,7 @@ public void testCheckIncorrectDataBaseFailure() {
330336
unit = SECONDS)
331337
@Test
332338
public void testUserHasNoPermissionToDataBase() {
333-
executeQuery("create user '" + USERNAME_WITHOUT_PERMISSION + "'@'%' IDENTIFIED BY '" + PASSWORD_WITHOUT_PERMISSION + "';\n");
339+
executeQuery(db, "create user '" + USERNAME_WITHOUT_PERMISSION + "'@'%' IDENTIFIED BY '" + PASSWORD_WITHOUT_PERMISSION + "';\n");
334340
final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.USERNAME_KEY, USERNAME_WITHOUT_PERMISSION);
335341
((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, PASSWORD_WITHOUT_PERMISSION);
336342
final MySQLDestination destination = new MySQLDestination();

‎airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java

+4-6
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,6 @@ protected void setup(final TestDestinationEnv testEnv, final HashSet<String> TES
100100

101101
@Override
102102
protected void tearDown(final TestDestinationEnv testEnv) {
103-
dslContext.close();
104103
db.stop();
105104
db.close();
106105
}
@@ -128,18 +127,17 @@ private void grantCorrectPermissions() {
128127
}
129128

130129
private void executeQuery(final String query) {
131-
try (final DSLContext dslContext = DSLContextFactory.create(
130+
final DSLContext dslContext = DSLContextFactory.create(
132131
"root",
133132
"test",
134133
db.getDriverClassName(),
135134
String.format("jdbc:mysql://%s:%s/%s?useSSL=true&requireSSL=true&verifyServerCertificate=false",
136135
db.getHost(),
137136
db.getFirstMappedPort(),
138137
db.getDatabaseName()),
139-
SQLDialect.DEFAULT)) {
140-
new Database(dslContext).query(
141-
ctx -> ctx
142-
.execute(query));
138+
SQLDialect.DEFAULT);
139+
try {
140+
new Database(dslContext).query(ctx -> ctx.execute(query));
143141
} catch (final SQLException e) {
144142
throw new RuntimeException(e);
145143
}

‎airbyte-integrations/connectors/destination-mysql/src/test-integration/kotlin/io/airbyte/integrations/destination/mysql/MysqlTestSourceOperations.kt

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package io.airbyte.integrations.destination.mysql
2+
3+
import com.fasterxml.jackson.databind.JsonNode
4+
import com.fasterxml.jackson.databind.node.ObjectNode
5+
import io.airbyte.cdk.db.jdbc.JdbcSourceOperations
6+
import io.airbyte.commons.json.Jsons
7+
import java.sql.ResultSet
8+
import java.sql.SQLException
9+
import java.util.Locale
10+
11+
class MysqlTestSourceOperations : JdbcSourceOperations() {
12+
@Throws(SQLException::class)
13+
override fun copyToJsonField(resultSet: ResultSet, colIndex: Int, json: ObjectNode) {
14+
val columnName = resultSet.metaData.getColumnName(colIndex)
15+
val columnTypeName = resultSet.metaData.getColumnTypeName(colIndex).lowercase(Locale.getDefault())
16+
17+
// JSON has no equivalent in JDBCType
18+
if ("json" == columnTypeName) {
19+
json.set<JsonNode>(columnName, Jsons.deserializeExact(resultSet.getString(colIndex)))
20+
} else {
21+
super.copyToJsonField(resultSet, colIndex, json)
22+
}
23+
}
24+
}

0 commit comments

Comments
 (0)