Source Twilio: shift start date for stream if it exceeds 400 days (ai…

…rbytehq#15972)
robbinhan · Sep 29, 2022 · e4c9f6a · e4c9f6a
1 parent 8cbb5d0
commit e4c9f6a
Show file tree

Hide file tree

Showing 30 changed files with 132 additions and 116 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -1036,7 +1036,7 @@
 - name: Twilio
   sourceDefinitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1
   dockerRepository: airbyte/source-twilio
-  dockerImageTag: 0.1.6
+  dockerImageTag: 0.1.7
   documentationUrl: https://docs.airbyte.io/integrations/sources/twilio
   icon: twilio.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -10280,7 +10280,7 @@
         oauthFlowOutputParameters:
         - - "token"
         - - "key"
-- dockerImage: "airbyte/source-twilio:0.1.6"
+- dockerImage: "airbyte/source-twilio:0.1.7"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/twilio"
     connectionSpecification:
@@ -10291,7 +10291,7 @@
       - "account_sid"
       - "auth_token"
       - "start_date"
-      additionalProperties: false
+      additionalProperties: true
       properties:
         account_sid:
           title: "Account ID"

diff --git a/airbyte-integrations/connectors/source-twilio/Dockerfile b/airbyte-integrations/connectors/source-twilio/Dockerfile
@@ -12,5 +12,5 @@ COPY main.py ./
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.6
+LABEL io.airbyte.version=0.1.7
 LABEL io.airbyte.name=airbyte/source-twilio
diff --git a/airbyte-integrations/connectors/source-twilio/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-twilio/acceptance-test-docker.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env sh
 
 # Build latest connector image
-docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2)
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-)
 
 # Pull latest acctest image
 docker pull airbyte/source-acceptance-test:latest

diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/accounts.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/accounts.json
@@ -25,7 +25,7 @@
     },
     "subresource_uris": {
       "type": ["null", "object"],
-      "additionalProperties": false,
+      "additionalProperties": true,
       "properties": {
         "addresses": {
           "type": ["null", "string"]
@@ -97,5 +97,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json
@@ -52,5 +52,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/alerts.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/alerts.json
@@ -50,5 +50,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json
@@ -32,6 +32,9 @@
     "sms_method": {
       "type": ["null", "string"]
     },
+    "public_application_connect_enabled": {
+      "type": ["null", "boolean"]
+    },
     "sms_status_callback": {
       "type": ["null", "string"]
     },
@@ -64,5 +67,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/...ions/connectors/source-twilio/source_twilio/schemas/available_phone_number_countries.json b/...ions/connectors/source-twilio/source_twilio/schemas/available_phone_number_countries.json
@@ -14,7 +14,7 @@
     },
     "subresource_uris": {
       "type": ["null", "object"],
-      "additionalProperties": false,
+      "additionalProperties": true,
       "properties": {
         "local": {
           "type": ["null", "string"]
@@ -29,5 +29,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/...rations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_local.json b/...rations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_local.json
@@ -52,9 +52,9 @@
         }
       },
       "type": ["null", "object"],
-      "additionalProperties": false
+      "additionalProperties": true
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/...ations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_mobile.json b/...ations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_mobile.json
@@ -52,9 +52,9 @@
         }
       },
       "type": ["null", "object"],
-      "additionalProperties": false
+      "additionalProperties": true
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/...ons/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_toll_free.json b/...ons/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_toll_free.json
@@ -52,9 +52,9 @@
         }
       },
       "type": ["null", "object"],
-      "additionalProperties": false
+      "additionalProperties": true
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json
@@ -84,7 +84,7 @@
     },
     "subresource_uris": {
       "type": ["null", "object"],
-      "additionalProperties": false,
+      "additionalProperties": true,
       "properties": {
         "recordings": {
           "type": ["null", "string"]
@@ -114,5 +114,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/...-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json b/...-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json
@@ -46,5 +46,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json
@@ -37,7 +37,7 @@
     },
     "subresource_uris": {
       "type": ["null", "object"],
-      "additionalProperties": false,
+      "additionalProperties": true,
       "properties": {
         "participants": {
           "type": ["null", "string"]
@@ -49,5 +49,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/...-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json b/...-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json
@@ -63,7 +63,7 @@
         }
       },
       "type": ["null", "object"],
-      "additionalProperties": false
+      "additionalProperties": true
     },
     "status_callback": {
       "type": ["null", "string"]
@@ -94,5 +94,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/...e-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json b/...e-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json
@@ -69,7 +69,7 @@
         }
       },
       "type": ["null", "object"],
-      "additionalProperties": false
+      "additionalProperties": true
     },
     "voice_receive_mode": {
       "type": ["null", "string"]
@@ -121,5 +121,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json
@@ -16,5 +16,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/message_media.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/message_media.json
@@ -25,5 +25,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json
@@ -56,7 +56,7 @@
     },
     "subresource_uris": {
       "type": ["null", "object"],
-      "additionalProperties": false,
+      "additionalProperties": true,
       "properties": {
         "feedback": {
           "type": ["null", "string"]
@@ -74,5 +74,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/outgoing_caller_ids.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/outgoing_caller_ids.json
@@ -25,5 +25,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json
@@ -31,7 +31,7 @@
     },
     "subresource_uris": {
       "type": ["null", "object"],
-      "additionalProperties": false,
+      "additionalProperties": true,
       "properties": {
         "members": {
           "type": ["null", "string"]
@@ -40,5 +40,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json
@@ -73,11 +73,11 @@
         }
       },
       "type": ["null", "object"],
-      "additionalProperties": false
+      "additionalProperties": true
     },
     "subresource_uris": {
       "type": ["null", "object"],
-      "additionalProperties": false,
+      "additionalProperties": true,
       "properties": {
         "transcriptions": {
           "type": ["null", "string"]
@@ -89,5 +89,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json
@@ -43,5 +43,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_records.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_records.json
@@ -11,7 +11,7 @@
     },
     "subresource_uris": {
       "type": ["null", "object"],
-      "additionalProperties": false,
+      "additionalProperties": true,
       "properties": {
         "yesterday": {
           "type": ["null", "string"]
@@ -76,5 +76,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json
@@ -53,5 +53,5 @@
     }
   },
   "type": "object",
-  "additionalProperties": false
+  "additionalProperties": true
 }
diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py
@@ -2,8 +2,10 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+import datetime
 from typing import Any, List, Mapping, Tuple
 
+import pendulum
 from airbyte_cdk.logger import AirbyteLogger
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources import AbstractSource
@@ -34,6 +36,8 @@
     UsageTriggers,
 )
 
+RETENTION_WINDOW_LIMIT = 400
+
 
 class SourceTwilio(AbstractSource):
     def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
@@ -67,6 +71,14 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             "lookback_window": config["lookback_window"],
         }
 
+        # Fix for `Date range specified in query is partially or entirely outside of retention window of 400 days`
+        # See: https://app.zenhub.com/workspaces/python-connectors-6262f8b593bb82001df56c65/issues/airbytehq/airbyte/10418
+        incremental_stream_kwargs_message_stream = dict(**incremental_stream_kwargs)
+        if pendulum.now().diff(pendulum.parse(config["start_date"])).days >= RETENTION_WINDOW_LIMIT:
+            incremental_stream_kwargs_message_stream["start_date"] = (
+                pendulum.now() - datetime.timedelta(days=RETENTION_WINDOW_LIMIT - 1)
+            ).to_iso8601_string()
+
         streams = [
             Accounts(**full_refresh_stream_kwargs),
             Addresses(**full_refresh_stream_kwargs),
@@ -82,8 +94,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             DependentPhoneNumbers(**full_refresh_stream_kwargs),
             IncomingPhoneNumbers(**full_refresh_stream_kwargs),
             Keys(**full_refresh_stream_kwargs),
-            MessageMedia(**incremental_stream_kwargs),
-            Messages(**incremental_stream_kwargs),
+            MessageMedia(**incremental_stream_kwargs_message_stream),
+            Messages(**incremental_stream_kwargs_message_stream),
             OutgoingCallerIds(**full_refresh_stream_kwargs),
             Queues(**full_refresh_stream_kwargs),
             Recordings(**incremental_stream_kwargs),

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/spec.json b/airbyte-integrations/connectors/source-twilio/source_twilio/spec.json
@@ -5,7 +5,7 @@
     "title": "Twilio Spec",
     "type": "object",
     "required": ["account_sid", "auth_token", "start_date"],
-    "additionalProperties": false,
+    "additionalProperties": true,
     "properties": {
       "account_sid": {
         "title": "Account ID",

diff --git a/docs/integrations/sources/twilio.md b/docs/integrations/sources/twilio.md
@@ -80,6 +80,7 @@ For more information, see [the Twilio docs for rate limitations](https://support
 
 | Version | Date        | Pull Request                                             | Subject                                                                                                 |
 |:--------|:------------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------|
+| 0.1.7   | 2022-08-26  | [15972](https://github.com/airbytehq/airbyte/pull/15972) | Shift start date for stream if it exceeds 400 days                            |
 | 0.1.6   | 2022-06-22  | [14000](https://github.com/airbytehq/airbyte/pull/14000) | Update Records stream schema and align tests with connectors' best practices                            |
 | 0.1.5   | 2022-06-22  | [13896](https://github.com/airbytehq/airbyte/pull/13896) | Add lookback window parameters to fetch messages with a rolling window and catch status updates         |
 | 0.1.4   | 2022-04-22  | [12157](https://github.com/airbytehq/airbyte/pull/12157) | Use Retry-After header for backoff                                                                      |