From d3683626a0796afb875fa0ffefab287f4fc85dd9 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Mon, 14 Feb 2022 20:05:01 +0200 Subject: [PATCH 01/10] add data type test with compatibility support --- .../DataTypeTestArgumentProvider.java | 86 +++++++++++++++++++ .../DestinationAcceptanceTest.java | 36 ++++++++ .../data_type_array_object_test_catalog.json | 0 .../data_type_array_object_test_messages.txt | 0 .../data_type_array_test_catalog.json | 0 .../data_type_array_test_messages.txt | 0 .../data_type_basic_test_catalog.json | 0 .../data_type_basic_test_messages.txt | 0 .../data_type_object_test_catalog.json | 0 .../data_type_object_test_messages.txt | 0 10 files changed, 122 insertions(+) create mode 100644 airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java create mode 100644 airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json create mode 100644 airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_messages.txt create mode 100644 airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json create mode 100644 airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_messages.txt create mode 100644 airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_catalog.json create mode 100644 airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_messages.txt create mode 100644 airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json create mode 100644 airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java new file mode 100644 index 0000000000000..943ec8359b2e0 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.standardtest.destination; + +import java.util.stream.Stream; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DataTypeTestArgumentProvider implements ArgumentsProvider { + + private static final Logger LOGGER = LoggerFactory.getLogger(DataTypeTestArgumentProvider.class); + + public static final CatalogMessageTestConfigWithCompatibility BASIC_TEST = + new CatalogMessageTestConfigWithCompatibility("data_type_basic_test_catalog.json", "data_type_basic_test_messages.txt", + new TestCompatibility(true, false, false)); + public static final CatalogMessageTestConfigWithCompatibility ARRAY_TEST = + new CatalogMessageTestConfigWithCompatibility("data_type_array_test_catalog.json", "data_type_array_test_messages.txt", + new TestCompatibility(true, true, false)); + public static final CatalogMessageTestConfigWithCompatibility OBJECT_TEST = + new CatalogMessageTestConfigWithCompatibility("data_type_object_test_catalog.json", "data_type_object_test_messages.txt", + new TestCompatibility(true, false, true)); + public static final CatalogMessageTestConfigWithCompatibility OBJECT_WITH_ARRAY_TEST = + new CatalogMessageTestConfigWithCompatibility("data_type_array_object_test_catalog.json", "data_type_array_object_test_messages.txt", + new TestCompatibility(true, true, true)); + + @Override + public Stream provideArguments(ExtensionContext context) throws Exception { + return Stream.of( + getArguments(BASIC_TEST), + getArguments(ARRAY_TEST), + getArguments(OBJECT_TEST), + getArguments(OBJECT_WITH_ARRAY_TEST)); + } + + private Arguments getArguments(CatalogMessageTestConfigWithCompatibility testConfig) { + return Arguments.of(testConfig.messageFile, testConfig.catalogFile, testConfig.testCompatibility); + } + + public record TestCompatibility(boolean requireBasicCompatibility, + boolean requireArrayCompatibility, + boolean requireObjectCompatibility) { + + public boolean isTestCompatible(boolean supportBasicDataTypeTest, boolean supportArrayDataTypeTest, boolean supportObjectDataTypeTest) { + LOGGER.info("---- Data type test compatibility ----"); + LOGGER.info("| Data type test | Require | Support |"); + LOGGER.info("| Basic test | {} | {} |", (requireBasicCompatibility ? "true " : "false"), (supportBasicDataTypeTest ? "true " : "false")); + LOGGER.info("| Array test | {} | {} |", (requireArrayCompatibility ? "true " : "false"), (supportArrayDataTypeTest ? "true " : "false")); + LOGGER.info("| Object test | {} | {} |", (requireObjectCompatibility ? "true " : "false"), + (supportObjectDataTypeTest ? "true " : "false")); + LOGGER.info("--------------------------------------"); + + if (requireBasicCompatibility && !supportBasicDataTypeTest) { + LOGGER.info("The destination doesn't support required Basic data type test. The test is skipped!"); + return false; + } + if (requireArrayCompatibility && !supportArrayDataTypeTest) { + LOGGER.info("The destination doesn't support required Array data type test. The test is skipped!"); + return false; + } + if (requireObjectCompatibility && !supportObjectDataTypeTest) { + LOGGER.info("The destination doesn't support required Object data type test. The test is skipped!"); + return false; + } + + return true; + } + + } + + public static class CatalogMessageTestConfigWithCompatibility extends DataArgumentsProvider.CatalogMessageTestConfigPair { + + final TestCompatibility testCompatibility; + + public CatalogMessageTestConfigWithCompatibility(String catalogFile, String messageFile, TestCompatibility testCompatibility) { + super(catalogFile, messageFile); + this.testCompatibility = testCompatibility; + } + + } + +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index c450abf2e970f..82eaf4e12662c 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -1293,4 +1293,40 @@ public void testStressPerformance() throws Exception { + "\n" + "Praesent finibus scelerisque elit, accumsan condimentum risus mattis vitae. Donec tristique hendrerit facilisis. Curabitur metus purus, venenatis non elementum id, finibus eu augue. Quisque posuere rhoncus ligula, et vehicula erat pulvinar at. Pellentesque vel quam vel lectus tincidunt congue quis id sapien. Ut efficitur mauris vitae pretium iaculis. Aliquam consectetur iaculis nisi vitae laoreet. Integer vel odio quis diam mattis tempor eget nec est. Donec iaculis facilisis neque, at dictum magna vestibulum ut. Sed malesuada non nunc ac consequat. Maecenas tempus lectus a nisl congue, ac venenatis diam viverra. Nam ac justo id nulla iaculis lobortis in eu ligula. Vivamus et ligula id sapien efficitur aliquet. Curabitur est justo, tempus vitae mollis quis, tincidunt vitae felis. Vestibulum molestie laoreet justo, nec mollis purus vulputate at."; + protected boolean supportBasicDataTypeTest() { + return false; + } + + protected boolean supportArrayDataTypeTest() { + return false; + } + + protected boolean supportObjectDataTypeTest() { + return false; + } + + private boolean checkTestCompatibility(final DataTypeTestArgumentProvider.TestCompatibility testCompatibility) { + return testCompatibility.isTestCompatible(supportBasicDataTypeTest(), supportArrayDataTypeTest(), supportObjectDataTypeTest()); + } + + @ParameterizedTest + @ArgumentsSource(DataTypeTestArgumentProvider.class) + public void testDataTypeTest(final String messagesFilename, + final String catalogFilename, + final DataTypeTestArgumentProvider.TestCompatibility testCompatibility) + throws Exception { + if (!checkTestCompatibility(testCompatibility)) + return; + + final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); + final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); + final List messages = MoreResources.readResource(messagesFilename).lines() + .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList()); + + final JsonNode config = getConfig(); + final String defaultSchema = getDefaultSchema(config); + runSyncAndVerifyStateOutput(config, messages, configuredCatalog, false); + retrieveRawRecordsAndAssertSameMessages(catalog, messages, defaultSchema); + } + } diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_messages.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_messages.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_catalog.json new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_messages.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_messages.txt new file mode 100644 index 0000000000000..e69de29bb2d1d From d1eddaccb4508701da88905d4ecdd375b73d4e7f Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Mon, 14 Feb 2022 22:43:09 +0200 Subject: [PATCH 02/10] add data for basic data type tests --- .../data_type_basic_test_catalog.json | 101 ++++++++++++++++++ .../data_type_basic_test_messages.txt | 26 +++++ 2 files changed, 127 insertions(+) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_catalog.json index e69de29bb2d1d..aecc94b9d4876 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_catalog.json +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_catalog.json @@ -0,0 +1,101 @@ +{ + "streams": [ + { + "name": "string_test_1", + "json_schema": { + "properties": { + "data": { + "type": "string" + } + } + } + }, + { + "name": "date_test_1", + "json_schema": { + "properties": { + "data": { + "type": "string", + "format": "date" + } + } + } + }, + { + "name": "datetime_test_1", + "json_schema": { + "properties": { + "data": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + } + }, + { + "name": "datetime_test_2", + "json_schema": { + "properties": { + "data": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + } + } + } + }, + { + "name": "number_test_1", + "json_schema": { + "properties": { + "data": { + "type": "number" + } + } + } + }, + { + "name": "bignumber_test_1", + "json_schema": { + "properties": { + "data": { + "type": "string", + "airbyte_type": "big_number" + } + } + } + }, + { + "name": "integer_test_1", + "json_schema": { + "properties": { + "data": { + "type": "integer" + } + } + } + }, + { + "name": "big_integer_test_1", + "json_schema": { + "properties": { + "data": { + "type": "string", + "airbyte_type": "big_integer" + } + } + } + }, + { + "name": "boolean_test_1", + "json_schema": { + "properties": { + "data": { + "type": "boolean" + } + } + } + } + ] +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_messages.txt index e69de29bb2d1d..438ba2698dad6 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_messages.txt +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_messages.txt @@ -0,0 +1,26 @@ +{"type": "RECORD", "record": {"stream": "string_test_1", "emitted_at": 1602637589100, "data": { "data" : "foo bar" }}} +{"type": "RECORD", "record": {"stream": "string_test_1", "emitted_at": 1602637589200, "data": { "data" : "" }}} +{"type": "RECORD", "record": {"stream": "string_test_1", "emitted_at": 1602637589300, "data": { "data" : "some random special characters: ࠈൡሗ" }}} +{"type": "RECORD", "record": {"stream": "date_test_1", "emitted_at": 1602637589100, "data": { "data" : "2021-01-23" }}} +{"type": "RECORD", "record": {"stream": "date_test_1", "emitted_at": 1602637589200, "data": { "data" : "1504-02-29" }}} +{"type": "RECORD", "record": {"stream": "date_test_1", "emitted_at": 1602637589300, "data": { "data" : "9999-12-23" }}} +{"type": "RECORD", "record": {"stream": "datetime_test_1", "emitted_at": 1602637589100, "data": { "data" : "2022-11-22T01:23:45+05:00" }}} +{"type": "RECORD", "record": {"stream": "datetime_test_1", "emitted_at": 1602637589200, "data": { "data" : "1504-02-29T01:23:45+05:00" }}} +{"type": "RECORD", "record": {"stream": "datetime_test_1", "emitted_at": 1602637589300, "data": { "data" : "9999-12-21T01:23:45+05:00" }}} +{"type": "RECORD", "record": {"stream": "datetime_test_2", "emitted_at": 1602637589100, "data": { "data" : "2022-11-22T01:23:45" }}} +{"type": "RECORD", "record": {"stream": "datetime_test_2", "emitted_at": 1602637589200, "data": { "data" : "1504-02-29T01:23:45" }}} +{"type": "RECORD", "record": {"stream": "datetime_test_2", "emitted_at": 1602637589300, "data": { "data" : "9999-12-21T01:23:45" }}} +{"type": "RECORD", "record": {"stream": "number_test_1", "emitted_at": 1602637589100, "data": { "data" : 56.78 }}} +{"type": "RECORD", "record": {"stream": "number_test_1", "emitted_at": 1602637589200, "data": { "data" : 0 }}} +{"type": "RECORD", "record": {"stream": "number_test_1", "emitted_at": 1602637589300, "data": { "data" : -12345.678 }}} +{"type": "RECORD", "record": {"stream": "bignumber_test_1", "emitted_at": 1602637589100, "data": { "data" : "100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.1234" }}} +{"type": "RECORD", "record": {"stream": "bignumber_test_1", "emitted_at": 1602637589200, "data": { "data" : "0" }}} +{"type": "RECORD", "record": {"stream": "bignumber_test_1", "emitted_at": 1602637589300, "data": { "data" : "-12345.678" }}} +{"type": "RECORD", "record": {"stream": "integer_test_1", "emitted_at": 1602637589100, "data": { "data" : 42 }}} +{"type": "RECORD", "record": {"stream": "integer_test_1", "emitted_at": 1602637589200, "data": { "data" : 0 }}} +{"type": "RECORD", "record": {"stream": "integer_test_1", "emitted_at": 1602637589300, "data": { "data" : -12345 }}} +{"type": "RECORD", "record": {"stream": "big_integer_test_1", "emitted_at": 1602637589100, "data": { "data" : "123141241234124123141241234124123141241234124123141241234124123141241234124" }}} +{"type": "RECORD", "record": {"stream": "big_integer_test_1", "emitted_at": 1602637589200, "data": { "data" : "0" }}} +{"type": "RECORD", "record": {"stream": "big_integer_test_1", "emitted_at": 1602637589300, "data": { "data" : "-1234" }}} +{"type": "RECORD", "record": {"stream": "boolean_test_1", "emitted_at": 1602637589100, "data": { "data" : true }}} +{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} \ No newline at end of file From 35d94a7a5fe727149f5e0ede74579529146ccd6f Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Wed, 16 Feb 2022 20:19:08 +0200 Subject: [PATCH 03/10] use test with normalization --- .../DataTypeTestArgumentProvider.java | 6 +-- .../DestinationAcceptanceTest.java | 53 ++++++++++--------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java index 943ec8359b2e0..28882c6553d7c 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java @@ -55,15 +55,15 @@ public boolean isTestCompatible(boolean supportBasicDataTypeTest, boolean suppor LOGGER.info("--------------------------------------"); if (requireBasicCompatibility && !supportBasicDataTypeTest) { - LOGGER.info("The destination doesn't support required Basic data type test. The test is skipped!"); + LOGGER.warn("The destination doesn't support required Basic data type test. The test is skipped!"); return false; } if (requireArrayCompatibility && !supportArrayDataTypeTest) { - LOGGER.info("The destination doesn't support required Array data type test. The test is skipped!"); + LOGGER.warn("The destination doesn't support required Array data type test. The test is skipped!"); return false; } if (requireObjectCompatibility && !supportObjectDataTypeTest) { - LOGGER.info("The destination doesn't support required Object data type test. The test is skipped!"); + LOGGER.warn("The destination doesn't support required Object data type test. The test is skipped!"); return false; } diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 82eaf4e12662c..0284437fac833 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -71,7 +71,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; import org.slf4j.Logger; @@ -321,7 +320,7 @@ void tearDownInternal() throws Exception { /** * Verify that when the integrations returns a valid spec. */ - @Test + //@Test public void testGetSpec() throws WorkerException { assertNotNull(runSpec()); } @@ -330,7 +329,7 @@ public void testGetSpec() throws WorkerException { * Verify that when given valid credentials, that check connection returns a success response. * Assume that the {@link DestinationAcceptanceTest#getConfig()} is valid. */ - @Test + //@Test public void testCheckConnection() throws Exception { assertEquals(Status.SUCCEEDED, runCheck(getConfig()).getStatus()); } @@ -339,7 +338,7 @@ public void testCheckConnection() throws Exception { * Verify that when given invalid credentials, that check connection returns a failed response. * Assume that the {@link DestinationAcceptanceTest#getFailCheckConfig()} is invalid. */ - @Test + //@Test public void testCheckConnectionInvalidCredentials() throws Exception { assertEquals(Status.FAILED, runCheck(getFailCheckConfig()).getStatus()); } @@ -348,7 +347,7 @@ public void testCheckConnectionInvalidCredentials() throws Exception { * Verify that the integration successfully writes records. Tests a wide variety of messages and * schemas (aspirationally, anyway). */ - @ParameterizedTest + //@ParameterizedTest @ArgumentsSource(DataArgumentsProvider.class) public void testSync(final String messagesFilename, final String catalogFilename) throws Exception { final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); @@ -366,7 +365,7 @@ public void testSync(final String messagesFilename, final String catalogFilename * This serves to test MSSQL 2100 limit parameters in a single query. this means that for Airbyte * insert data need to limit to ~ 700 records (3 columns for the raw tables) = 2100 params */ - @ParameterizedTest + //@ParameterizedTest @ArgumentsSource(DataArgumentsProvider.class) public void testSyncWithLargeRecordBatch(final String messagesFilename, final String catalogFilename) throws Exception { final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); @@ -383,7 +382,7 @@ public void testSyncWithLargeRecordBatch(final String messagesFilename, final St /** * Verify that the integration overwrites the first sync with the second sync. */ - @Test + //@Test public void testSecondSync() throws Exception { if (!implementsOverwrite()) { LOGGER.info("Destination's spec.json does not support overwrite sync mode."); @@ -426,7 +425,7 @@ public void testSecondSync() throws Exception { * Tests that we are able to read over special characters properly when processing line breaks in * destinations. */ - @Test + //@Test public void testLineBreakCharacters() throws Exception { final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), AirbyteCatalog.class); @@ -457,7 +456,7 @@ public void testLineBreakCharacters() throws Exception { retrieveRawRecordsAndAssertSameMessages(catalog, secondSyncMessages, defaultSchema); } - @Test + //@Test public void specNormalizationValueShouldBeCorrect() throws Exception { final boolean normalizationFromSpec = normalizationFromSpec(); assertEquals(normalizationFromSpec, supportsNormalization()); @@ -473,7 +472,7 @@ public void specNormalizationValueShouldBeCorrect() throws Exception { } } - @Test + //@Test public void specDBTValueShouldBeCorrect() throws WorkerException { assertEquals(dbtFromSpec(), supportsDBT()); } @@ -482,7 +481,7 @@ public void specDBTValueShouldBeCorrect() throws WorkerException { * Verify that the integration successfully writes records incrementally. The second run should * append records to the datastore instead of overwriting the previous run. */ - @Test + //@Test public void testIncrementalSync() throws Exception { if (!implementsAppend()) { LOGGER.info("Destination's spec.json does not include '\"supportsIncremental\" ; true'"); @@ -533,7 +532,7 @@ public void testIncrementalSync() throws Exception { * Verify that the integration successfully writes records successfully both raw and normalized. * Tests a wide variety of messages an schemas (aspirationally, anyway). */ - @ParameterizedTest + //@ParameterizedTest @ArgumentsSource(DataArgumentsProvider.class) public void testSyncWithNormalization(final String messagesFilename, final String catalogFilename) throws Exception { if (!normalizationFromSpec()) { @@ -560,7 +559,7 @@ public void testSyncWithNormalization(final String messagesFilename, final Strin * Although this test assumes append-dedup requires normalization, and almost all our Destinations * do so, this is not necessarily true. This explains {@link #implementsAppendDedup()}. */ - @Test + //@Test public void testIncrementalDedupeSync() throws Exception { if (!implementsAppendDedup()) { LOGGER.info("Destination's spec.json does not include 'append_dedupe' in its '\"supportedDestinationSyncModes\"'"); @@ -654,7 +653,7 @@ public void testIncrementalDedupeSync() throws Exception { * The first big message should be small enough to fit into the destination while the second message * would be too big and fails to replicate. */ - @Test + //@Test void testSyncVeryBigRecords() throws Exception { if (!implementsRecordSizeLimitChecks()) { return; @@ -717,7 +716,7 @@ protected int getMaxRecordValueLimit() { return 1000000000; } - @Test + //@Test public void testCustomDbtTransformations() throws Exception { if (!dbtFromSpec()) { return; @@ -791,7 +790,7 @@ public void testCustomDbtTransformations() throws Exception { runner.close(); } - @Test + //@Test void testCustomDbtTransformationsFailure() throws Exception { if (!normalizationFromSpec() || !dbtFromSpec()) { // we require normalization implementation for this destination, because we make sure to install @@ -826,7 +825,7 @@ void testCustomDbtTransformationsFailure() throws Exception { /** * Verify the destination uses the namespace field if it is set. */ - @Test + //@Test void testSyncUsesAirbyteStreamNamespaceIfNotNull() throws Exception { if (!implementsNamespaces()) { return; @@ -857,7 +856,7 @@ void testSyncUsesAirbyteStreamNamespaceIfNotNull() throws Exception { /** * Verify a destination is able to write tables with the same name to different namespaces. */ - @Test + //@Test void testSyncWriteSameTableNameDifferentNamespace() throws Exception { if (!implementsNamespaces()) { return; @@ -911,7 +910,7 @@ void testSyncWriteSameTableNameDifferentNamespace() throws Exception { * The source connector must specify its entrypoint in the AIRBYTE_ENTRYPOINT variable. This test * ensures that the entrypoint environment variable is set. */ - @Test + //@Test public void testEntrypointEnvVar() throws Exception { final String entrypoint = EntrypointEnvChecker.getEntrypointEnvVariable( processFactory, @@ -1181,7 +1180,7 @@ public String toString() { * "docker ps" command in console to find the container's id. Then run "docker container attach * your_containers_id" (ex. docker container attach 18cc929f44c8) to see the container's output */ - @Test + //@Test @Disabled public void testStressPerformance() throws Exception { final int streamsSize = 5; // number of generated streams @@ -1311,22 +1310,24 @@ private boolean checkTestCompatibility(final DataTypeTestArgumentProvider.TestCo @ParameterizedTest @ArgumentsSource(DataTypeTestArgumentProvider.class) - public void testDataTypeTest(final String messagesFilename, + public void testDataTypeTestWithNormalization(final String messagesFilename, final String catalogFilename, final DataTypeTestArgumentProvider.TestCompatibility testCompatibility) - throws Exception { - if (!checkTestCompatibility(testCompatibility)) + throws Exception { + if (!checkTestCompatibility(testCompatibility) || !normalizationFromSpec()) return; final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); final List messages = MoreResources.readResource(messagesFilename).lines() - .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList()); + .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList()); final JsonNode config = getConfig(); + runSyncAndVerifyStateOutput(config, messages, configuredCatalog, true); + final String defaultSchema = getDefaultSchema(config); - runSyncAndVerifyStateOutput(config, messages, configuredCatalog, false); - retrieveRawRecordsAndAssertSameMessages(catalog, messages, defaultSchema); + final List actualMessages = retrieveNormalizedRecords(catalog, defaultSchema); + assertSameMessages(messages, actualMessages, true); } } From 07fae9d478b46952eddc8ab0e113d4af5d66d94b Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Wed, 16 Feb 2022 23:20:59 +0200 Subject: [PATCH 04/10] run the test using or not using normalization based on a destination --- .../DestinationAcceptanceTest.java | 80 +++++++++++++------ 1 file changed, 55 insertions(+), 25 deletions(-) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 0284437fac833..0d229dc9c3673 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -53,6 +53,7 @@ import io.airbyte.workers.protocols.airbyte.AirbyteDestination; import io.airbyte.workers.protocols.airbyte.DefaultAirbyteDestination; import io.airbyte.workers.test_helpers.EntrypointEnvChecker; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; @@ -71,6 +72,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; import org.slf4j.Logger; @@ -320,7 +322,7 @@ void tearDownInternal() throws Exception { /** * Verify that when the integrations returns a valid spec. */ - //@Test + @Test public void testGetSpec() throws WorkerException { assertNotNull(runSpec()); } @@ -329,7 +331,7 @@ public void testGetSpec() throws WorkerException { * Verify that when given valid credentials, that check connection returns a success response. * Assume that the {@link DestinationAcceptanceTest#getConfig()} is valid. */ - //@Test + @Test public void testCheckConnection() throws Exception { assertEquals(Status.SUCCEEDED, runCheck(getConfig()).getStatus()); } @@ -338,7 +340,7 @@ public void testCheckConnection() throws Exception { * Verify that when given invalid credentials, that check connection returns a failed response. * Assume that the {@link DestinationAcceptanceTest#getFailCheckConfig()} is invalid. */ - //@Test + @Test public void testCheckConnectionInvalidCredentials() throws Exception { assertEquals(Status.FAILED, runCheck(getFailCheckConfig()).getStatus()); } @@ -347,7 +349,7 @@ public void testCheckConnectionInvalidCredentials() throws Exception { * Verify that the integration successfully writes records. Tests a wide variety of messages and * schemas (aspirationally, anyway). */ - //@ParameterizedTest + @ParameterizedTest @ArgumentsSource(DataArgumentsProvider.class) public void testSync(final String messagesFilename, final String catalogFilename) throws Exception { final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); @@ -365,7 +367,7 @@ public void testSync(final String messagesFilename, final String catalogFilename * This serves to test MSSQL 2100 limit parameters in a single query. this means that for Airbyte * insert data need to limit to ~ 700 records (3 columns for the raw tables) = 2100 params */ - //@ParameterizedTest + @ParameterizedTest @ArgumentsSource(DataArgumentsProvider.class) public void testSyncWithLargeRecordBatch(final String messagesFilename, final String catalogFilename) throws Exception { final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); @@ -382,7 +384,7 @@ public void testSyncWithLargeRecordBatch(final String messagesFilename, final St /** * Verify that the integration overwrites the first sync with the second sync. */ - //@Test + @Test public void testSecondSync() throws Exception { if (!implementsOverwrite()) { LOGGER.info("Destination's spec.json does not support overwrite sync mode."); @@ -425,7 +427,7 @@ public void testSecondSync() throws Exception { * Tests that we are able to read over special characters properly when processing line breaks in * destinations. */ - //@Test + @Test public void testLineBreakCharacters() throws Exception { final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), AirbyteCatalog.class); @@ -456,7 +458,7 @@ public void testLineBreakCharacters() throws Exception { retrieveRawRecordsAndAssertSameMessages(catalog, secondSyncMessages, defaultSchema); } - //@Test + @Test public void specNormalizationValueShouldBeCorrect() throws Exception { final boolean normalizationFromSpec = normalizationFromSpec(); assertEquals(normalizationFromSpec, supportsNormalization()); @@ -472,7 +474,7 @@ public void specNormalizationValueShouldBeCorrect() throws Exception { } } - //@Test + @Test public void specDBTValueShouldBeCorrect() throws WorkerException { assertEquals(dbtFromSpec(), supportsDBT()); } @@ -481,7 +483,7 @@ public void specDBTValueShouldBeCorrect() throws WorkerException { * Verify that the integration successfully writes records incrementally. The second run should * append records to the datastore instead of overwriting the previous run. */ - //@Test + @Test public void testIncrementalSync() throws Exception { if (!implementsAppend()) { LOGGER.info("Destination's spec.json does not include '\"supportsIncremental\" ; true'"); @@ -532,7 +534,7 @@ public void testIncrementalSync() throws Exception { * Verify that the integration successfully writes records successfully both raw and normalized. * Tests a wide variety of messages an schemas (aspirationally, anyway). */ - //@ParameterizedTest + @ParameterizedTest @ArgumentsSource(DataArgumentsProvider.class) public void testSyncWithNormalization(final String messagesFilename, final String catalogFilename) throws Exception { if (!normalizationFromSpec()) { @@ -559,7 +561,7 @@ public void testSyncWithNormalization(final String messagesFilename, final Strin * Although this test assumes append-dedup requires normalization, and almost all our Destinations * do so, this is not necessarily true. This explains {@link #implementsAppendDedup()}. */ - //@Test + @Test public void testIncrementalDedupeSync() throws Exception { if (!implementsAppendDedup()) { LOGGER.info("Destination's spec.json does not include 'append_dedupe' in its '\"supportedDestinationSyncModes\"'"); @@ -653,7 +655,7 @@ public void testIncrementalDedupeSync() throws Exception { * The first big message should be small enough to fit into the destination while the second message * would be too big and fails to replicate. */ - //@Test + @Test void testSyncVeryBigRecords() throws Exception { if (!implementsRecordSizeLimitChecks()) { return; @@ -716,7 +718,7 @@ protected int getMaxRecordValueLimit() { return 1000000000; } - //@Test + @Test public void testCustomDbtTransformations() throws Exception { if (!dbtFromSpec()) { return; @@ -790,7 +792,7 @@ public void testCustomDbtTransformations() throws Exception { runner.close(); } - //@Test + @Test void testCustomDbtTransformationsFailure() throws Exception { if (!normalizationFromSpec() || !dbtFromSpec()) { // we require normalization implementation for this destination, because we make sure to install @@ -825,7 +827,7 @@ void testCustomDbtTransformationsFailure() throws Exception { /** * Verify the destination uses the namespace field if it is set. */ - //@Test + @Test void testSyncUsesAirbyteStreamNamespaceIfNotNull() throws Exception { if (!implementsNamespaces()) { return; @@ -856,7 +858,7 @@ void testSyncUsesAirbyteStreamNamespaceIfNotNull() throws Exception { /** * Verify a destination is able to write tables with the same name to different namespaces. */ - //@Test + @Test void testSyncWriteSameTableNameDifferentNamespace() throws Exception { if (!implementsNamespaces()) { return; @@ -910,7 +912,7 @@ void testSyncWriteSameTableNameDifferentNamespace() throws Exception { * The source connector must specify its entrypoint in the AIRBYTE_ENTRYPOINT variable. This test * ensures that the entrypoint environment variable is set. */ - //@Test + @Test public void testEntrypointEnvVar() throws Exception { final String entrypoint = EntrypointEnvChecker.getEntrypointEnvVariable( processFactory, @@ -1180,7 +1182,7 @@ public String toString() { * "docker ps" command in console to find the container's id. Then run "docker container attach * your_containers_id" (ex. docker container attach 18cc929f44c8) to see the container's output */ - //@Test + @Test @Disabled public void testStressPerformance() throws Exception { final int streamsSize = 5; // number of generated streams @@ -1314,20 +1316,48 @@ public void testDataTypeTestWithNormalization(final String messagesFilename, final String catalogFilename, final DataTypeTestArgumentProvider.TestCompatibility testCompatibility) throws Exception { - if (!checkTestCompatibility(testCompatibility) || !normalizationFromSpec()) + if (!checkTestCompatibility(testCompatibility)) return; - final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); - final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); - final List messages = MoreResources.readResource(messagesFilename).lines() + final AirbyteCatalog catalog = readCatalogFromFile(catalogFilename); + final ConfiguredAirbyteCatalog configuredCatalog = getDefaultAirbyteCatalog(catalog); + final List messages = reedMessagesFromFile(messagesFilename); + + if (supportsNormalization()) { + LOGGER.info("Normalization is supported! Run test with normalization."); + runAndCheckWithNormalization(messages, configuredCatalog, catalog); + } + else { + LOGGER.info("Normalization is not supported! Run test without normalization."); + runAndCheckWithoutNormalization(messages, configuredCatalog, catalog); + } + } + + private AirbyteCatalog readCatalogFromFile(String catalogFilename) throws IOException { + return Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); + } + + private ConfiguredAirbyteCatalog getDefaultAirbyteCatalog(AirbyteCatalog catalog) { + return CatalogHelpers.toDefaultConfiguredCatalog(catalog); + } + + private List reedMessagesFromFile(String messagesFilename) throws IOException { + return MoreResources.readResource(messagesFilename).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList()); + } + private void runAndCheckWithNormalization(final List messages, final ConfiguredAirbyteCatalog configuredCatalog, final AirbyteCatalog catalog) throws Exception { final JsonNode config = getConfig(); runSyncAndVerifyStateOutput(config, messages, configuredCatalog, true); - final String defaultSchema = getDefaultSchema(config); - final List actualMessages = retrieveNormalizedRecords(catalog, defaultSchema); + final List actualMessages = retrieveNormalizedRecords(catalog, getDefaultSchema(config)); assertSameMessages(messages, actualMessages, true); } + private void runAndCheckWithoutNormalization(final List messages, final ConfiguredAirbyteCatalog configuredCatalog, final AirbyteCatalog catalog) throws Exception { + final JsonNode config = getConfig(); + runSyncAndVerifyStateOutput(config, messages, configuredCatalog, false); + retrieveRawRecordsAndAssertSameMessages(catalog, messages, getDefaultSchema(config)); + } + } From 1c7f6453845188a934c1a9a16a376406972f47ea Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Wed, 16 Feb 2022 23:21:17 +0200 Subject: [PATCH 05/10] add object test data + schema --- .../data_type_object_test_catalog.json | 48 +++++++++++++++++++ .../data_type_object_test_messages.txt | 2 + 2 files changed, 50 insertions(+) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json index e69de29bb2d1d..e055d58f909ba 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json @@ -0,0 +1,48 @@ +{ + "streams": [ + { + "name": "object_test_1", + "json_schema": { + "type": [ + "object" + ], + "properties": { + "property_string": { + "type": "string" + }, + "property_date": { + "type": "string", + "format": "date" + }, + "property_timestamp_with_timezone": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "property_timestamp_without_timezone": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "property_number": { + "type": "number" + }, + "property_big_number": { + "type": "string", + "airbyte_type": "big_number" + }, + "property_integer": { + "type": "integer" + }, + "property_big_integer": { + "type": "string", + "airbyte_type": "big_integer" + }, + "property_boolean": { + "type": "boolean" + } + } + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_messages.txt index e69de29bb2d1d..b42a63188a53f 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_messages.txt +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_messages.txt @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "object_test_1", "emitted_at": 1602637589100, "data": {"property_string": "foo bar", "property_date": "2021-01-23", "property_timestamp_with_timezone": "2022-11-22T01:23:45+00:00", "property_timestamp_without_timezone": "2022-11-22T01:23:45", "property_number": 56.78, "property_big_number": "100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.1234", "property_integer": 42, "property_big_integer": "123141241234124123141241234124123141241234124123141241234124123141241234124", "property_boolean": true }}} +{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} \ No newline at end of file From ba6967a53e9547d0ac15ebde90c80582538cbece Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Wed, 16 Feb 2022 23:21:29 +0200 Subject: [PATCH 06/10] add array test data + schema --- .../data_type_array_test_catalog.json | 92 +++++++++++++++++++ .../data_type_array_test_messages.txt | 2 + 2 files changed, 94 insertions(+) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json index e69de29bb2d1d..5695a163663d0 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json @@ -0,0 +1,92 @@ +{ + "streams": [ + { + "name": "array_test_1", + "json_schema": { + "properties": { + "string_array": { + "type": [ + "array" + ], + "items": { + "type": [ + "string" + ] + } + }, + "array_date": { + "type": [ + "array" + ], + "items": { + "type": "string", + "format": "date" + } + }, + "array_timestamp_with_timezone": { + "type": [ + "array" + ], + "items": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + }, + "array_timestamp_without_timezone": { + "type": [ + "array" + ], + "items": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + } + }, + "array_number": { + "type": [ + "array" + ], + "items": { + "type": "number" + } + }, + "array_big_number": { + "type": [ + "array" + ], + "items": { + "type": "string", + "airbyte_type": "big_number" + } + }, + "array_integer": { + "type": [ + "array" + ], + "items": { + "type": "integer" + } + }, + "array_big_integer": { + "type": [ + "array" + ], + "items": { + "type": "string", + "airbyte_type": "big_integer" + } + }, + "array_boolean": { + "type": [ + "array" + ], + "items": { + "type": "boolean" + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_messages.txt index e69de29bb2d1d..1c0bb5224a0a3 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_messages.txt +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_messages.txt @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "array_test_1", "emitted_at": 1602637589100, "data": { "string_array" : ["foo bar", "some random special characters: ࠈൡሗ"], "array_date" : ["2021-01-23", "1504-02-29"], "array_timestamp_with_timezone" : ["2022-11-22T01:23:45+05:00", "9999-12-21T01:23:45-05:00"], "array_timestamp_without_timezone" : ["2022-11-22T01:23:45", "1504-02-29T01:23:45"], "array_number" : [56.78, 0, -12345.678], "array_big_number" : ["-12345.678", "100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.1234"], "array_integer" : [42, 0, 12345], "array_big_integer" : ["0", "123141241234124123141241234124123141241234124123141241234124123141241234124"], "array_boolean" : [true, false] }}} +{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} \ No newline at end of file From a1515a723a24ec6feecefd33e9973e072a3a43d7 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Thu, 17 Feb 2022 17:48:01 +0200 Subject: [PATCH 07/10] add object/array/object test data + schema --- .../data_type_array_object_test_catalog.json | 40 +++++++++++++++++++ .../data_type_array_object_test_messages.txt | 2 + 2 files changed, 42 insertions(+) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json index e69de29bb2d1d..66788617f163b 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "name": "object_array_test_1", + "json_schema": { + "type": [ + "object" + ], + "properties": { + "property_string": { + "type": [ + "string" + ] + }, + "property_array": { + "type": [ + "array" + ], + "items": { + "type": [ + "object" + ], + "properties": { + "property_string": {"type": "string"}, + "property_date": {"type": "string", "format": "date"}, + "property_timestamp_with_timezone": {"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"}, + "property_timestamp_without_timezone": {"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}, + "property_number": {"type": "number"}, + "property_big_number": {"type": "string", "airbyte_type": "big_number"}, + "property_integer": {"type": "integer"}, + "property_big_integer": {"type": "string", "airbyte_type": "big_integer"}, + "property_boolean": {"type": "boolean"} + } + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_messages.txt index e69de29bb2d1d..15a6eebf906dc 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_messages.txt +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_messages.txt @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "object_array_test_1", "emitted_at": 1602637589100, "data": { "property_string" : "qqq", "property_array" : [ { "property_string": "foo bar", "property_date": "2021-01-23", "property_timestamp_with_timezone": "2022-11-22T01:23:45+00:00", "property_timestamp_without_timezone": "2022-11-22T01:23:45", "property_number": 56.78, "property_big_number": "100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.1234", "property_integer": 42, "property_big_integer": "123141241234124123141241234124123141241234124123141241234124123141241234124", "property_boolean": true } ] }}} +{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} \ No newline at end of file From 1749b032dd68cd6e61f037d4dad92131dd65e03e Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Thu, 17 Feb 2022 19:15:35 +0200 Subject: [PATCH 08/10] format --- .../DataTypeTestArgumentProvider.java | 6 ++- .../DestinationAcceptanceTest.java | 21 +++++--- .../data_type_array_object_test_catalog.json | 50 +++++++++++-------- .../data_type_array_test_catalog.json | 42 ++++------------ .../data_type_object_test_catalog.json | 6 +-- 5 files changed, 58 insertions(+), 67 deletions(-) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java index 28882c6553d7c..7d32d983dcd7a 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java @@ -48,8 +48,10 @@ public record TestCompatibility(boolean requireBasicCompatibility, public boolean isTestCompatible(boolean supportBasicDataTypeTest, boolean supportArrayDataTypeTest, boolean supportObjectDataTypeTest) { LOGGER.info("---- Data type test compatibility ----"); LOGGER.info("| Data type test | Require | Support |"); - LOGGER.info("| Basic test | {} | {} |", (requireBasicCompatibility ? "true " : "false"), (supportBasicDataTypeTest ? "true " : "false")); - LOGGER.info("| Array test | {} | {} |", (requireArrayCompatibility ? "true " : "false"), (supportArrayDataTypeTest ? "true " : "false")); + LOGGER.info("| Basic test | {} | {} |", (requireBasicCompatibility ? "true " : "false"), + (supportBasicDataTypeTest ? "true " : "false")); + LOGGER.info("| Array test | {} | {} |", (requireArrayCompatibility ? "true " : "false"), + (supportArrayDataTypeTest ? "true " : "false")); LOGGER.info("| Object test | {} | {} |", (requireObjectCompatibility ? "true " : "false"), (supportObjectDataTypeTest ? "true " : "false")); LOGGER.info("--------------------------------------"); diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 0d229dc9c3673..0f046f529628e 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -1313,9 +1313,9 @@ private boolean checkTestCompatibility(final DataTypeTestArgumentProvider.TestCo @ParameterizedTest @ArgumentsSource(DataTypeTestArgumentProvider.class) public void testDataTypeTestWithNormalization(final String messagesFilename, - final String catalogFilename, - final DataTypeTestArgumentProvider.TestCompatibility testCompatibility) - throws Exception { + final String catalogFilename, + final DataTypeTestArgumentProvider.TestCompatibility testCompatibility) + throws Exception { if (!checkTestCompatibility(testCompatibility)) return; @@ -1326,8 +1326,7 @@ public void testDataTypeTestWithNormalization(final String messagesFilename, if (supportsNormalization()) { LOGGER.info("Normalization is supported! Run test with normalization."); runAndCheckWithNormalization(messages, configuredCatalog, catalog); - } - else { + } else { LOGGER.info("Normalization is not supported! Run test without normalization."); runAndCheckWithoutNormalization(messages, configuredCatalog, catalog); } @@ -1343,10 +1342,13 @@ private ConfiguredAirbyteCatalog getDefaultAirbyteCatalog(AirbyteCatalog catalog private List reedMessagesFromFile(String messagesFilename) throws IOException { return MoreResources.readResource(messagesFilename).lines() - .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList()); + .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList()); } - private void runAndCheckWithNormalization(final List messages, final ConfiguredAirbyteCatalog configuredCatalog, final AirbyteCatalog catalog) throws Exception { + private void runAndCheckWithNormalization(final List messages, + final ConfiguredAirbyteCatalog configuredCatalog, + final AirbyteCatalog catalog) + throws Exception { final JsonNode config = getConfig(); runSyncAndVerifyStateOutput(config, messages, configuredCatalog, true); @@ -1354,7 +1356,10 @@ private void runAndCheckWithNormalization(final List messages, f assertSameMessages(messages, actualMessages, true); } - private void runAndCheckWithoutNormalization(final List messages, final ConfiguredAirbyteCatalog configuredCatalog, final AirbyteCatalog catalog) throws Exception { + private void runAndCheckWithoutNormalization(final List messages, + final ConfiguredAirbyteCatalog configuredCatalog, + final AirbyteCatalog catalog) + throws Exception { final JsonNode config = getConfig(); runSyncAndVerifyStateOutput(config, messages, configuredCatalog, false); retrieveRawRecordsAndAssertSameMessages(catalog, messages, getDefaultSchema(config)); diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json index 66788617f163b..2ce588e732ebc 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json @@ -3,33 +3,39 @@ { "name": "object_array_test_1", "json_schema": { - "type": [ - "object" - ], + "type": ["object"], "properties": { "property_string": { - "type": [ - "string" - ] + "type": ["string"] }, "property_array": { - "type": [ - "array" - ], + "type": ["array"], "items": { - "type": [ - "object" - ], + "type": ["object"], "properties": { - "property_string": {"type": "string"}, - "property_date": {"type": "string", "format": "date"}, - "property_timestamp_with_timezone": {"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"}, - "property_timestamp_without_timezone": {"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}, - "property_number": {"type": "number"}, - "property_big_number": {"type": "string", "airbyte_type": "big_number"}, - "property_integer": {"type": "integer"}, - "property_big_integer": {"type": "string", "airbyte_type": "big_integer"}, - "property_boolean": {"type": "boolean"} + "property_string": { "type": "string" }, + "property_date": { "type": "string", "format": "date" }, + "property_timestamp_with_timezone": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "property_timestamp_without_timezone": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "property_number": { "type": "number" }, + "property_big_number": { + "type": "string", + "airbyte_type": "big_number" + }, + "property_integer": { "type": "integer" }, + "property_big_integer": { + "type": "string", + "airbyte_type": "big_integer" + }, + "property_boolean": { "type": "boolean" } } } } @@ -37,4 +43,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json index 5695a163663d0..45e614559a143 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json @@ -5,28 +5,20 @@ "json_schema": { "properties": { "string_array": { - "type": [ - "array" - ], + "type": ["array"], "items": { - "type": [ - "string" - ] + "type": ["string"] } }, "array_date": { - "type": [ - "array" - ], + "type": ["array"], "items": { "type": "string", "format": "date" } }, "array_timestamp_with_timezone": { - "type": [ - "array" - ], + "type": ["array"], "items": { "type": "string", "format": "date-time", @@ -34,9 +26,7 @@ } }, "array_timestamp_without_timezone": { - "type": [ - "array" - ], + "type": ["array"], "items": { "type": "string", "format": "date-time", @@ -44,43 +34,33 @@ } }, "array_number": { - "type": [ - "array" - ], + "type": ["array"], "items": { "type": "number" } }, "array_big_number": { - "type": [ - "array" - ], + "type": ["array"], "items": { "type": "string", "airbyte_type": "big_number" } }, "array_integer": { - "type": [ - "array" - ], + "type": ["array"], "items": { "type": "integer" } }, "array_big_integer": { - "type": [ - "array" - ], + "type": ["array"], "items": { "type": "string", "airbyte_type": "big_integer" } }, "array_boolean": { - "type": [ - "array" - ], + "type": ["array"], "items": { "type": "boolean" } @@ -89,4 +69,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json index e055d58f909ba..f143cbcea7283 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json @@ -3,9 +3,7 @@ { "name": "object_test_1", "json_schema": { - "type": [ - "object" - ], + "type": ["object"], "properties": { "property_string": { "type": "string" @@ -45,4 +43,4 @@ } } ] -} \ No newline at end of file +} From 2690fed3c37783eda408d9b18bed0177fd2b0e2c Mon Sep 17 00:00:00 2001 From: Andrii Leonets <30464745+DoNotPanicUA@users.noreply.github.com> Date: Wed, 23 Feb 2022 15:11:55 +0200 Subject: [PATCH 09/10] Update airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java Co-authored-by: Edward Gao --- .../standardtest/destination/DestinationAcceptanceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 0f046f529628e..fae2d68cfba94 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -1321,7 +1321,7 @@ public void testDataTypeTestWithNormalization(final String messagesFilename, final AirbyteCatalog catalog = readCatalogFromFile(catalogFilename); final ConfiguredAirbyteCatalog configuredCatalog = getDefaultAirbyteCatalog(catalog); - final List messages = reedMessagesFromFile(messagesFilename); + final List messages = readMessagesFromFile(messagesFilename); if (supportsNormalization()) { LOGGER.info("Normalization is supported! Run test with normalization."); From a1a229428752a47e6ae3ec1e0e0e197296ab8e60 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Wed, 23 Feb 2022 16:07:49 +0200 Subject: [PATCH 10/10] review --- .../destination/DestinationAcceptanceTest.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 82cfcc6a5ab4e..74d33a60c3717 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -1328,7 +1328,7 @@ public void testDataTypeTestWithNormalization(final String messagesFilename, return; final AirbyteCatalog catalog = readCatalogFromFile(catalogFilename); - final ConfiguredAirbyteCatalog configuredCatalog = getDefaultAirbyteCatalog(catalog); + final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); final List messages = readMessagesFromFile(messagesFilename); if (supportsNormalization()) { @@ -1344,11 +1344,7 @@ private AirbyteCatalog readCatalogFromFile(String catalogFilename) throws IOExce return Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); } - private ConfiguredAirbyteCatalog getDefaultAirbyteCatalog(AirbyteCatalog catalog) { - return CatalogHelpers.toDefaultConfiguredCatalog(catalog); - } - - private List reedMessagesFromFile(String messagesFilename) throws IOException { + private List readMessagesFromFile(String messagesFilename) throws IOException { return MoreResources.readResource(messagesFilename).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList()); }