-
Notifications
You must be signed in to change notification settings - Fork 4.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Destination Postgres : Enable DAT and fix the data fetch. (#12543)
* Enable DAT for Postgres and fix the data fetch. Move JDBC abstract part for tests to the JdbcDestinationAcceptanceTest.java * Remove unnecessary deserialization + add jsonb to json transformation. * Remove unnecessary deserialization from ssh
- Loading branch information
1 parent
38390e5
commit 781be94
Showing
6 changed files
with
164 additions
and
89 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
41 changes: 41 additions & 0 deletions
41
.../java/io/airbyte/integrations/standardtest/destination/JdbcDestinationAcceptanceTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| /* | ||
| * Copyright (c) 2021 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.standardtest.destination; | ||
|
|
||
| import com.fasterxml.jackson.databind.JsonNode; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import com.fasterxml.jackson.databind.node.ObjectNode; | ||
| import io.airbyte.commons.json.Jsons; | ||
| import java.util.Arrays; | ||
| import org.jooq.Record; | ||
|
|
||
| public abstract class JdbcDestinationAcceptanceTest extends DestinationAcceptanceTest { | ||
|
|
||
| protected final ObjectMapper mapper = new ObjectMapper(); | ||
|
|
||
| protected JsonNode getJsonFromRecord(Record record) { | ||
| ObjectNode node = mapper.createObjectNode(); | ||
|
|
||
| Arrays.stream(record.fields()).forEach(field -> { | ||
| var value = record.get(field); | ||
|
|
||
| switch (field.getDataType().getTypeName()) { | ||
| case "varchar", "jsonb", "other": | ||
| var stringValue = (value != null ? value.toString() : null); | ||
| if (stringValue != null && (stringValue.replaceAll("[^\\x00-\\x7F]", "").matches("^\\[.*\\]$") | ||
| || stringValue.replaceAll("[^\\x00-\\x7F]", "").matches("^\\{.*\\}$"))) { | ||
| node.set(field.getName(), Jsons.deserialize(stringValue)); | ||
| } else { | ||
| node.put(field.getName(), stringValue); | ||
| } | ||
| break; | ||
| default: | ||
| node.put(field.getName(), (value != null ? value.toString() : null)); | ||
| } | ||
| }); | ||
| return node; | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
58 changes: 58 additions & 0 deletions
58
...gration/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| /* | ||
| * Copyright (c) 2021 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.destination.postgres; | ||
|
|
||
| import io.airbyte.integrations.destination.ExtendedNameTransformer; | ||
| import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; | ||
| import java.time.LocalDate; | ||
| import java.time.LocalDateTime; | ||
| import java.time.ZoneOffset; | ||
| import java.time.ZonedDateTime; | ||
| import java.time.format.DateTimeFormatter; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| public class PostgresTestDataComparator extends AdvancedTestDataComparator { | ||
|
|
||
| private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer(); | ||
|
|
||
| private static final String POSTGRES_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; | ||
| private static final String POSTGRES_DATETIME_WITH_TZ_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; | ||
|
|
||
| @Override | ||
| protected List<String> resolveIdentifier(final String identifier) { | ||
| final List<String> result = new ArrayList<>(); | ||
| final String resolved = namingResolver.getIdentifier(identifier); | ||
| result.add(identifier); | ||
| result.add(resolved); | ||
| if (!resolved.startsWith("\"")) { | ||
| result.add(resolved.toLowerCase()); | ||
| result.add(resolved.toUpperCase()); | ||
| } | ||
| return result; | ||
| } | ||
|
|
||
| private LocalDate parseLocalDate(String dateTimeValue) { | ||
| if (dateTimeValue != null) { | ||
| var format = (dateTimeValue.matches(".+Z") ? POSTGRES_DATETIME_FORMAT : AIRBYTE_DATETIME_FORMAT); | ||
| return LocalDate.parse(dateTimeValue, DateTimeFormatter.ofPattern(format)); | ||
| } else { | ||
| return null; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| protected boolean compareDateTimeValues(String expectedValue, String actualValue) { | ||
| var destinationDate = parseLocalDate(actualValue); | ||
| var expectedDate = LocalDate.parse(expectedValue, DateTimeFormatter.ofPattern(AIRBYTE_DATETIME_FORMAT)); | ||
| return expectedDate.equals(destinationDate); | ||
| } | ||
|
|
||
| @Override | ||
| protected ZonedDateTime parseDestinationDateWithTz(String destinationValue) { | ||
| return ZonedDateTime.of(LocalDateTime.parse(destinationValue, DateTimeFormatter.ofPattern(POSTGRES_DATETIME_WITH_TZ_FORMAT)), ZoneOffset.UTC); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters