JDBC sources: consolidate date/time values mapping (#20346)

* JDBC sources: consolidate date/time values mapping * fixed Snowflake acceptance tests * fixed TestJdbcUtils * refactoring * update setDate for incremental sync * fixed CockroachDbJdbcSourceAcceptanceTest * fixed CockroachDbSourceDatatypeTest * fixed CockroachDbSourceDatatypeTest * fixed MsSQLSourceDatatypeTest * fixed MsSQL cdc datatype test * refactoring * fixed mssql cdc test * fixed Redshift tests * fixed Redshift tests * updated db2 datatype tests * fixed DB2 tests * fixed DB2 strict-encrypt tests * fixed Oracle strict-encrypt tests * refactoring * updated changelog * refactoring * removed redundant overrides * bump versions * auto-bump connector version * resolved merge conflicts * auto-bump connector version * auto-bump connector version * auto-bump connector version * auto-bump connector version * bump postgres version * auto-bump connector version * updated seed definitions * revert generated definitions for Postgres * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Jan 18, 2023 · f5f4de2 · f5f4de2
1 parent a0b6b9c
commit f5f4de2
Show file tree

Hide file tree

Showing 53 changed files with 245 additions and 155 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -41,7 +41,7 @@
 - name: AlloyDB for PostgreSQL
   sourceDefinitionId: 1fa90628-2b9e-11ed-a261-0242ac120002
   dockerRepository: airbyte/source-alloydb
-  dockerImageTag: 1.0.34
+  dockerImageTag: 1.0.35
   documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   icon: alloydb.svg
   sourceType: database
@@ -249,7 +249,7 @@
 - name: ClickHouse
   sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c
   dockerRepository: airbyte/source-clickhouse
-  dockerImageTag: 0.1.14
+  dockerImageTag: 0.1.15
   documentationUrl: https://docs.airbyte.com/integrations/sources/clickhouse
   icon: clickhouse.svg
   sourceType: database
@@ -281,7 +281,7 @@
 - name: Cockroachdb
   sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003
   dockerRepository: airbyte/source-cockroachdb
-  dockerImageTag: 0.1.18
+  dockerImageTag: 0.1.19
   documentationUrl: https://docs.airbyte.com/integrations/sources/cockroachdb
   icon: cockroachdb.svg
   sourceType: database
@@ -780,7 +780,7 @@
 - name: IBM Db2
   sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2
   dockerRepository: airbyte/source-db2
-  dockerImageTag: 0.1.16
+  dockerImageTag: 0.1.17
   documentationUrl: https://docs.airbyte.com/integrations/sources/db2
   icon: db2.svg
   sourceType: database
@@ -1019,7 +1019,7 @@
 - name: Microsoft SQL Server (MSSQL)
   sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1
   dockerRepository: airbyte/source-mssql
-  dockerImageTag: 0.4.26
+  dockerImageTag: 0.4.27
   documentationUrl: https://docs.airbyte.com/integrations/sources/mssql
   icon: mssql.svg
   sourceType: database
@@ -1075,7 +1075,7 @@
 - name: MySQL
   sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad
   dockerRepository: airbyte/source-mysql
-  dockerImageTag: 1.0.18
+  dockerImageTag: 1.0.19
   documentationUrl: https://docs.airbyte.com/integrations/sources/mysql
   icon: mysql.svg
   sourceType: database
@@ -1170,7 +1170,7 @@
 - name: Oracle DB
   sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83
   dockerRepository: airbyte/source-oracle
-  dockerImageTag: 0.3.21
+  dockerImageTag: 0.3.22
   documentationUrl: https://docs.airbyte.com/integrations/sources/oracle
   icon: oracle.svg
   sourceType: database
@@ -1338,7 +1338,7 @@
 - name: Postgres
   sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   dockerRepository: airbyte/source-postgres
-  dockerImageTag: 1.0.37
+  dockerImageTag: 1.0.38
   documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
   icon: postgresql.svg
   sourceType: database
@@ -1450,7 +1450,7 @@
 - name: Redshift
   sourceDefinitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6
   dockerRepository: airbyte/source-redshift
-  dockerImageTag: 0.3.15
+  dockerImageTag: 0.3.16
   documentationUrl: https://docs.airbyte.com/integrations/sources/redshift
   icon: redshift.svg
   sourceType: database
@@ -1632,7 +1632,7 @@
 - name: Snowflake
   sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2
   dockerRepository: airbyte/source-snowflake
-  dockerImageTag: 0.1.28
+  dockerImageTag: 0.1.29
   documentationUrl: https://docs.airbyte.com/integrations/sources/snowflake
   icon: snowflake.svg
   sourceType: database
@@ -1728,7 +1728,7 @@
 - name: TiDB
   sourceDefinitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae
   dockerRepository: airbyte/source-tidb
-  dockerImageTag: 0.2.1
+  dockerImageTag: 0.2.2
   documentationUrl: https://docs.airbyte.com/integrations/sources/tidb
   icon: tidb.svg
   sourceType: database

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -280,7 +280,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-alloydb:1.0.34"
+- dockerImage: "airbyte/source-alloydb:1.0.35"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
     connectionSpecification:
@@ -2185,7 +2185,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-clickhouse:0.1.14"
+- dockerImage: "airbyte/source-clickhouse:0.1.15"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/destinations/clickhouse"
     connectionSpecification:
@@ -2476,7 +2476,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-cockroachdb:0.1.18"
+- dockerImage: "airbyte/source-cockroachdb:0.1.19"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/cockroachdb"
     connectionSpecification:
@@ -6428,7 +6428,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-db2:0.1.16"
+- dockerImage: "airbyte/source-db2:0.1.17"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/db2"
     connectionSpecification:
@@ -6492,9 +6492,6 @@
               encryption_method:
                 type: "string"
                 const: "unencrypted"
-                enum:
-                - "unencrypted"
-                default: "unencrypted"
           - title: "TLS Encrypted (verify certificate)"
             description: "Verify and use the cert provided by the server."
             required:
@@ -6504,9 +6501,6 @@
               encryption_method:
                 type: "string"
                 const: "encrypted_verify_certificate"
-                enum:
-                - "encrypted_verify_certificate"
-                default: "encrypted_verify_certificate"
               ssl_certificate:
                 title: "SSL PEM file"
                 description: "Privacy Enhanced Mail (PEM) files are concatenated certificate\
@@ -8021,7 +8015,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-mssql:0.4.26"
+- dockerImage: "airbyte/source-mssql:0.4.27"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/destinations/mssql"
     connectionSpecification:
@@ -8884,7 +8878,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-mysql:1.0.18"
+- dockerImage: "airbyte/source-mysql:1.0.19"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/mysql"
     connectionSpecification:
@@ -10135,7 +10129,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-oracle:0.3.21"
+- dockerImage: "airbyte/source-oracle:0.3.22"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/oracle"
     connectionSpecification:
@@ -10177,7 +10171,6 @@
               connection_type:
                 type: "string"
                 const: "service_name"
-                default: "service_name"
                 order: 0
               service_name:
                 title: "Service name"
@@ -10191,7 +10184,6 @@
               connection_type:
                 type: "string"
                 const: "sid"
-                default: "sid"
                 order: 0
               sid:
                 title: "System ID (SID)"
@@ -10239,9 +10231,6 @@
               encryption_method:
                 type: "string"
                 const: "unencrypted"
-                enum:
-                - "unencrypted"
-                default: "unencrypted"
           - title: "Native Network Encryption (NNE)"
             description: "The native network encryption gives you the ability to encrypt\
               \ database connections, without the configuration overhead of TCP/IP\
@@ -10252,9 +10241,6 @@
               encryption_method:
                 type: "string"
                 const: "client_nne"
-                enum:
-                - "client_nne"
-                default: "client_nne"
               encryption_algorithm:
                 type: "string"
                 description: "This parameter defines what encryption algorithm is\
@@ -10274,9 +10260,6 @@
               encryption_method:
                 type: "string"
                 const: "encrypted_verify_certificate"
-                enum:
-                - "encrypted_verify_certificate"
-                default: "encrypted_verify_certificate"
               ssl_certificate:
                 title: "SSL PEM File"
                 description: "Privacy Enhanced Mail (PEM) files are concatenated certificate\
@@ -11395,7 +11378,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-postgres:1.0.37"
+- dockerImage: "airbyte/source-postgres:1.0.38"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
     connectionSpecification:
@@ -12233,7 +12216,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-redshift:0.3.15"
+- dockerImage: "airbyte/source-redshift:0.3.16"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/destinations/redshift"
     connectionSpecification:
@@ -13709,7 +13692,7 @@
         - - "client_secret"
         oauthFlowOutputParameters:
         - - "refresh_token"
-- dockerImage: "airbyte/source-snowflake:0.1.28"
+- dockerImage: "airbyte/source-snowflake:0.1.29"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/snowflake"
     connectionSpecification:
@@ -14557,7 +14540,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-tidb:0.2.1"
+- dockerImage: "airbyte/source-tidb:0.2.2"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/tidb"
     connectionSpecification:

diff --git a/...te-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/...te-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java
@@ -23,9 +23,11 @@
 import java.text.ParseException;
 import java.time.Instant;
 import java.time.LocalDate;
+import java.time.LocalTime;
 import java.time.OffsetDateTime;
 import java.time.OffsetTime;
 import java.time.chrono.IsoEra;
+import java.time.format.DateTimeParseException;
 import java.util.Collections;
 import javax.xml.bind.DatatypeConverter;
 
@@ -121,11 +123,11 @@ protected void putString(final ObjectNode node, final String columnName, final R
   }
 
   protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
-    node.put(columnName, DataTypeUtils.toISO8601String(resultSet.getDate(index)));
+    node.put(columnName, DateTimeConverter.convertToDate(getObject(resultSet, index, LocalDate.class)));
   }
 
   protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
-    node.put(columnName, DataTypeUtils.toISO8601String(resultSet.getTime(index)));
+    node.put(columnName, DateTimeConverter.convertToTime(getObject(resultSet, index, LocalTime.class)));
   }
 
   protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
@@ -143,7 +145,11 @@ protected void putDefault(final ObjectNode node, final String columnName, final
   }
 
   protected void setTime(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException {
-    setTimestamp(preparedStatement, parameterIndex, value);
+    try {
+      preparedStatement.setObject(parameterIndex, LocalTime.parse(value));
+    } catch (final DateTimeParseException e) {
+      setTimestamp(preparedStatement, parameterIndex, value);
+    }
   }
 
   protected void setTimestamp(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException {
@@ -174,6 +180,14 @@ protected void setTimestamp(final PreparedStatement preparedStatement, final int
   }
 
   protected void setDate(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException {
+    try {
+      preparedStatement.setObject(parameterIndex, LocalDate.parse(value));
+    } catch (final DateTimeParseException e) {
+      setDateAsTimestamp(preparedStatement, parameterIndex, value);
+    }
+  }
+
+  private void setDateAsTimestamp(PreparedStatement preparedStatement, int parameterIndex, String value) throws SQLException {
     try {
       final Timestamp from = Timestamp.from(DataTypeUtils.getDateFormat().parse(value).toInstant());
       preparedStatement.setDate(parameterIndex, new Date(from.getTime()));

diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java
@@ -12,7 +12,10 @@
 import static io.airbyte.db.jdbc.AbstractJdbcCompatibleSourceOperations.resolveEra;
 import static java.time.ZoneOffset.UTC;
 
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import java.sql.Date;
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.sql.Time;
 import java.sql.Timestamp;
 import java.time.Duration;
@@ -168,4 +171,15 @@ public static String convertToTime(final Object time) {
     }
   }
 
+  public static void putJavaSQLDate(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException {
+    final Date date = resultSet.getDate(index);
+    node.put(columnName, convertToDate(date));
+  }
+
+  public static void putJavaSQLTime(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException {
+    // resultSet.getTime() will lose nanoseconds precision
+    final LocalTime localTime = resultSet.getTimestamp(index).toLocalDateTime().toLocalTime();
+    node.put(columnName, convertToTime(localTime));
+  }
+
 }
diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java
@@ -163,8 +163,8 @@ void testSetStatementField() throws SQLException {
       sourceOperations.setCursorField(ps, 10, JDBCType.DECIMAL, "1");
       sourceOperations.setCursorField(ps, 11, JDBCType.CHAR, "a");
       sourceOperations.setCursorField(ps, 12, JDBCType.VARCHAR, "a");
-      sourceOperations.setCursorField(ps, 13, JDBCType.DATE, "2020-11-01T00:00:00Z");
-      sourceOperations.setCursorField(ps, 14, JDBCType.TIME, "1970-01-01T05:00:00.000Z");
+      sourceOperations.setCursorField(ps, 13, JDBCType.DATE, "2020-11-01");
+      sourceOperations.setCursorField(ps, 14, JDBCType.TIME, "05:00:00.000");
       sourceOperations.setCursorField(ps, 15, JDBCType.TIMESTAMP, "2001-09-29T03:00:00.000Z");
       sourceOperations.setCursorField(ps, 16, JDBCType.BINARY, "61616161");
 
@@ -391,10 +391,8 @@ private ObjectNode expectedValues() {
     expected.put("decimal", new BigDecimal(1));
     expected.put("char", "a");
     expected.put("varchar", "a");
-    // todo (cgardens) we should parse this to a date string
-    expected.put("date", "2020-11-01T00:00:00Z");
-    // todo (cgardens) we should parse this to a time string
-    expected.put("time", "1970-01-01T05:00:00Z");
+    expected.put("date", "2020-11-01");
+    expected.put("time", "05:00:00.000000");
     expected.put("timestamp", "2001-09-29T03:00:00.000000Z");
     expected.put("binary1", "aaaa".getBytes(Charsets.UTF_8));
     return expected;

diff --git a/...ezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java b/...ezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java
@@ -8,6 +8,7 @@
 import com.microsoft.sqlserver.jdbc.Geometry;
 import com.microsoft.sqlserver.jdbc.SQLServerException;
 import io.airbyte.db.DataTypeUtils;
+import io.airbyte.db.jdbc.DateTimeConverter;
 import io.debezium.spi.converter.CustomConverter;
 import io.debezium.spi.converter.RelationalColumn;
 import java.math.BigDecimal;
@@ -107,7 +108,9 @@ private void registerDate(final RelationalColumn field,
       if (Objects.isNull(input)) {
         return DebeziumConverterUtils.convertDefaultValue(field);
       }
-
+      if (field.typeName().equalsIgnoreCase("DATE")) {
+        return DateTimeConverter.convertToDate(input);
+      }
       return DebeziumConverterUtils.convertDate(input);
     });
   }

diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=1.0.34
+LABEL io.airbyte.version=1.0.35
 LABEL io.airbyte.name=airbyte/source-alloydb-strict-encrypt