Transform RFC2822 datetime into ISO8601

airbytehq · Oct 18, 2021 · 0237554 · 0237554
1 parent be3df09
commit 0237554
Show file tree

Hide file tree

Showing 19 changed files with 50 additions and 16 deletions.
diff --git a/...yte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json b/...yte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json
@@ -188,18 +188,6 @@
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
-    },
-    {
-      "stream": {
-        "name": "alerts",
-        "json_schema": {},
-        "supported_sync_modes": ["incremental", "full_refresh"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["date_updated"]
-      },
-      "cursor_field": ["date_updated"],
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/accounts.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/accounts.json
@@ -4,10 +4,12 @@
       "type": ["null", "string"]
     },
     "date_created": {
-      "type": ["null", "string"]
+      "type": ["null", "string"],
+      "format": "date-time"
     },
     "date_updated": {
-      "type": ["null", "string"]
+      "type": ["null", "string"],
+      "format": "date-time"
     },
     "friendly_name": {
       "type": ["null", "string"]

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json
@@ -10,9 +10,11 @@
       "type": ["null", "string"]
     },
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "emergency_enabled": {

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json
@@ -7,9 +7,11 @@
       "type": ["null", "string"]
     },
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "friendly_name": {

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json
@@ -4,9 +4,11 @@
       "type": ["null", "string"]
     },
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "parent_call_sid": {
@@ -34,6 +36,7 @@
       "type": ["null", "string"]
     },
     "start_time": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "end_time": {

diff --git a/...-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json b/...-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json
@@ -13,9 +13,11 @@
       "type": ["null", "string"]
     },
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "end_conference_on_exit": {

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json
@@ -4,6 +4,7 @@
       "type": ["null", "string"]
     },
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_updated": {

diff --git a/...-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json b/...-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json
@@ -28,9 +28,11 @@
       "type": ["null", "boolean"]
     },
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "sms_url": {

diff --git a/...e-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json b/...e-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json
@@ -28,9 +28,11 @@
       "type": ["null", "boolean"]
     },
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "sms_url": {

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json
@@ -1,9 +1,11 @@
 {
   "properties": {
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "friendly_name": {

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/message_media.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/message_media.json
@@ -17,6 +17,7 @@
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "uri": {

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json
@@ -10,13 +10,15 @@
       "type": ["null", "string"]
     },
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_sent": {
       "format": "date-time",
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "direction": {

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/outgoing_caller_ids.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/outgoing_caller_ids.json
@@ -4,9 +4,11 @@
       "type": ["null", "string"]
     },
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "friendly_name": {

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json
@@ -10,9 +10,11 @@
       "type": ["null", "integer"]
     },
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "friendly_name": {

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json
@@ -17,9 +17,11 @@
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "start_time": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "duration": {

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json
@@ -7,9 +7,11 @@
       "type": ["null", "string"]
     },
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "duration": {

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json
@@ -4,9 +4,11 @@
       "type": ["null", "string"]
     },
     "date_updated": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "date_fired": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "friendly_name": {
@@ -31,6 +33,7 @@
       "type": ["null", "number"]
     },
     "date_created": {
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "callback_url": {

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py
@@ -21,7 +21,7 @@ class TwilioStream(HttpStream, ABC):
     url_base = TWILIO_API_URL_BASE
     primary_key = "sid"
     page_size = 100
-    transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
+    transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization)
 
     @property
     def data_field(self):
@@ -66,6 +66,20 @@ def request_params(
             params.update(**next_page_token)
         return params
 
+    @transformer.registerCustomTransform
+    def custom_transform_function(original_value: Any, field_schema: Mapping[str, Any]) -> Any:
+        if original_value and "format" in field_schema and field_schema["format"] == "date-time":
+            try:
+                return pendulum.from_format(original_value, "ddd, D MMM YYYY HH:mm:ss ZZ").in_timezone('UTC').to_iso8601_string()
+            except ValueError:
+                # Twilio API returns datetime in two formats:
+                #   - RFC2822, like "Fri, 11 Dec 2020 04:28:40 +0000";
+                #   - ISO8601, like "2020-12-11T04:29:09Z".
+                # If `ValueError` exception was raised this means that datetime was already in ISO8601 format and there
+                # is no need in transforming anything.
+                pass
+        return original_value
+
 
 class IncrementalTwilioStream(TwilioStream, ABC):
     cursor_field = "date_updated"

diff --git a/docs/integrations/sources/twilio.md b/docs/integrations/sources/twilio.md
@@ -66,6 +66,6 @@ See [docs](https://www.twilio.com/docs/iam/api) for more details.
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
-| 0.1.1 | 2021-10-?? | [7034](https://github.com/airbytehq/airbyte/pull/7034) | Update schemas + turn on transformer for all streams |
+| 0.1.1 | 2021-10-18 | [7034](https://github.com/airbytehq/airbyte/pull/7034) | Update schemas and transform data types according to the API schema |
 | 0.1.0 | 2021-07-02 | [4070](https://github.com/airbytehq/airbyte/pull/4070) | Native Twilio connector implemented |