diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json b/airbyte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json index 30a6e6e4ba668..1de59155ead55 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json @@ -188,18 +188,6 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "alerts", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": ["date_updated"] - }, - "cursor_field": ["date_updated"], - "sync_mode": "incremental", - "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/accounts.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/accounts.json index e134c867fbd8d..524578cb8489a 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/accounts.json @@ -4,10 +4,12 @@ "type": ["null", "string"] }, "date_created": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "date_updated": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "friendly_name": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json index 280b2b2aacc9d..0671f851378c9 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json @@ -10,9 +10,11 @@ "type": ["null", "string"] }, "date_created": { + "format": "date-time", "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "emergency_enabled": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json index f5e494febf557..56fe7b0dcac05 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json @@ -7,9 +7,11 @@ "type": ["null", "string"] }, "date_created": { + "format": "date-time", "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "friendly_name": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json index b8243a59e6b85..3daf6f426b73b 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json @@ -4,9 +4,11 @@ "type": ["null", "string"] }, "date_created": { + "format": "date-time", "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "parent_call_sid": { @@ -34,6 +36,7 @@ "type": ["null", "string"] }, "start_time": { + "format": "date-time", "type": ["null", "string"] }, "end_time": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json index 35e232774fa2d..23f5de105e8e7 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json @@ -13,9 +13,11 @@ "type": ["null", "string"] }, "date_created": { + "format": "date-time", "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "end_conference_on_exit": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json index ea79de25c22e8..4d74793e4e09b 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json @@ -4,6 +4,7 @@ "type": ["null", "string"] }, "date_created": { + "format": "date-time", "type": ["null", "string"] }, "date_updated": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json index d17c0c0392380..58413bbccf617 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json @@ -28,9 +28,11 @@ "type": ["null", "boolean"] }, "date_created": { + "format": "date-time", "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "sms_url": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json index ca1c7057d9616..b493a73f0a857 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json @@ -28,9 +28,11 @@ "type": ["null", "boolean"] }, "date_created": { + "format": "date-time", "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "sms_url": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json index 22fbeb7ad14f2..7f30792f83ffd 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json @@ -1,9 +1,11 @@ { "properties": { "date_created": { + "format": "date-time", "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "friendly_name": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/message_media.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/message_media.json index c9adb472aef14..e73a3b2143945 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/message_media.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/message_media.json @@ -17,6 +17,7 @@ "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "uri": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json index ef1fe4ad2abee..24e0952f49a7d 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json @@ -10,6 +10,7 @@ "type": ["null", "string"] }, "date_created": { + "format": "date-time", "type": ["null", "string"] }, "date_sent": { @@ -17,6 +18,7 @@ "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "direction": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/outgoing_caller_ids.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/outgoing_caller_ids.json index 0ec183661c928..ce99aa4817dbd 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/outgoing_caller_ids.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/outgoing_caller_ids.json @@ -4,9 +4,11 @@ "type": ["null", "string"] }, "date_created": { + "format": "date-time", "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "friendly_name": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json index 7fa032e8f8513..594aacf683d78 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json @@ -10,9 +10,11 @@ "type": ["null", "integer"] }, "date_created": { + "format": "date-time", "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "friendly_name": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json index e00942d80aae6..2949880e2d42c 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json @@ -17,9 +17,11 @@ "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "start_time": { + "format": "date-time", "type": ["null", "string"] }, "duration": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json index 2a292f1258abf..ca6293568c2dc 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json @@ -7,9 +7,11 @@ "type": ["null", "string"] }, "date_created": { + "format": "date-time", "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "duration": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json index f639dbd5c00d4..658fa01a241f7 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json @@ -4,9 +4,11 @@ "type": ["null", "string"] }, "date_updated": { + "format": "date-time", "type": ["null", "string"] }, "date_fired": { + "format": "date-time", "type": ["null", "string"] }, "friendly_name": { @@ -31,6 +33,7 @@ "type": ["null", "number"] }, "date_created": { + "format": "date-time", "type": ["null", "string"] }, "callback_url": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py index 5350bba5d0404..528e8ddbb0b57 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py @@ -21,7 +21,7 @@ class TwilioStream(HttpStream, ABC): url_base = TWILIO_API_URL_BASE primary_key = "sid" page_size = 100 - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) @property def data_field(self): @@ -66,6 +66,20 @@ def request_params( params.update(**next_page_token) return params + @transformer.registerCustomTransform + def custom_transform_function(original_value: Any, field_schema: Mapping[str, Any]) -> Any: + if original_value and "format" in field_schema and field_schema["format"] == "date-time": + try: + return pendulum.from_format(original_value, "ddd, D MMM YYYY HH:mm:ss ZZ").in_timezone('UTC').to_iso8601_string() + except ValueError: + # Twilio API returns datetime in two formats: + # - RFC2822, like "Fri, 11 Dec 2020 04:28:40 +0000"; + # - ISO8601, like "2020-12-11T04:29:09Z". + # If `ValueError` exception was raised this means that datetime was already in ISO8601 format and there + # is no need in transforming anything. + pass + return original_value + class IncrementalTwilioStream(TwilioStream, ABC): cursor_field = "date_updated" diff --git a/docs/integrations/sources/twilio.md b/docs/integrations/sources/twilio.md index 36bd01b93e6c4..616f78ae72afc 100644 --- a/docs/integrations/sources/twilio.md +++ b/docs/integrations/sources/twilio.md @@ -66,6 +66,6 @@ See [docs](https://www.twilio.com/docs/iam/api) for more details. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.1 | 2021-10-?? | [7034](https://github.com/airbytehq/airbyte/pull/7034) | Update schemas + turn on transformer for all streams | +| 0.1.1 | 2021-10-18 | [7034](https://github.com/airbytehq/airbyte/pull/7034) | Update schemas and transform data types according to the API schema | | 0.1.0 | 2021-07-02 | [4070](https://github.com/airbytehq/airbyte/pull/4070) | Native Twilio connector implemented |