🎉 Source Zendesk Chat: make Chats stream incremental + add tests fo…

…r all streams (airbytehq#8166) * Make `Chats` stream incremental + add tests for all streams * Update docs * Bump connector's version + update docs
schlattk · Jan 4, 2022 · 2c8ed97 · 2c8ed97
1 parent b81cdc3
commit 2c8ed97
Show file tree

Hide file tree

Showing 11 changed files with 192 additions and 63 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -621,7 +621,7 @@
 - name: Zendesk Chat
   sourceDefinitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4
   dockerRepository: airbyte/source-zendesk-chat
-  dockerImageTag: 0.1.3
+  dockerImageTag: 0.1.4
   documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-chat
   icon: zendesk.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -6155,7 +6155,7 @@
         - - "client_secret"
         oauthFlowOutputParameters:
         - - "refresh_token"
-- dockerImage: "airbyte/source-zendesk-chat:0.1.3"
+- dockerImage: "airbyte/source-zendesk-chat:0.1.4"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-chat"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile
@@ -16,5 +16,5 @@ RUN pip install .
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main_dev.py"]
 
-LABEL io.airbyte.version=0.1.3
+LABEL io.airbyte.version=0.1.4
 LABEL io.airbyte.name=airbyte/source-zendesk-chat
diff --git a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml
@@ -11,16 +11,17 @@ tests:
     - config_path: "secrets/config.json"
   basic_read:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "sample_files/configured_catalog.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
   incremental:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog_incremental.json"
-#      Unable to use 'state_path' because Zendesk Chat API returns an error when specifying a date in the future.
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+#      Unable to use 'future_state_path' because Zendesk Chat API returns an error when specifying a date in the future.
 #      future_state_path: "integration_tests/abnormal_state.json"
       cursor_paths:
-        agents: ["id"]
-        bans: ["id"]
+        agents: [ "id" ]
+        bans: [ "id" ]
         agent_timeline: [ "start_time" ]
+        chats: [ "update_timestamp" ]
   full_refresh:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "sample_files/configured_catalog.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
diff --git a/...yte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog.json b/...yte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog.json
@@ -1,9 +1,120 @@
 {
   "streams": [
+    {
+      "stream": {
+        "name": "accounts",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "agent_timeline",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["start_time"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append",
+      "cursor_field": ["start_time"]
+    },
+    {
+      "stream": {
+        "name": "agents",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["id"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append",
+      "cursor_field": ["id"]
+    },
+    {
+      "stream": {
+        "name": "bans",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["id"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append",
+      "cursor_field": ["id"]
+    },
     {
       "stream": {
         "name": "chats",
         "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["update_timestamp"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append",
+      "cursor_field": ["update_timestamp"]
+    },
+    {
+      "stream": {
+        "name": "departments",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "goals",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "roles",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "shortcuts",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "skills",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "triggers",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "routing_settings",
+        "json_schema": {},
         "supported_sync_modes": ["full_refresh"]
       },
       "sync_mode": "full_refresh",

diff --git a/...ions/connectors/source-zendesk-chat/integration_tests/configured_catalog_incremental.json b/...ions/connectors/source-zendesk-chat/integration_tests/configured_catalog_incremental.json
diff --git a/airbyte-integrations/connectors/source-zendesk-chat/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-zendesk-chat/sample_files/configured_catalog.json
@@ -9,18 +9,66 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "agent_timeline",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["start_time"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append",
+      "cursor_field": ["start_time"]
+    },
+    {
+      "stream": {
+        "name": "agents",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["id"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append",
+      "cursor_field": ["id"]
+    },
+    {
+      "stream": {
+        "name": "bans",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["id"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append",
+      "cursor_field": ["id"]
+    },
     {
       "stream": {
         "name": "chats",
         "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["update_timestamp"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append",
+      "cursor_field": ["update_timestamp"]
+    },
+    {
+      "stream": {
+        "name": "departments",
+        "json_schema": {},
         "supported_sync_modes": ["full_refresh"]
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
     {
       "stream": {
-        "name": "shortcuts",
+        "name": "goals",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"]
       },
@@ -29,7 +77,7 @@
     },
     {
       "stream": {
-        "name": "triggers",
+        "name": "roles",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"]
       },
@@ -38,7 +86,7 @@
     },
     {
       "stream": {
-        "name": "departments",
+        "name": "shortcuts",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"]
       },
@@ -47,7 +95,7 @@
     },
     {
       "stream": {
-        "name": "goals",
+        "name": "skills",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"]
       },
@@ -56,7 +104,7 @@
     },
     {
       "stream": {
-        "name": "skills",
+        "name": "triggers",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"]
       },
@@ -65,7 +113,7 @@
     },
     {
       "stream": {
-        "name": "roles",
+        "name": "routing_settings",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"]
       },

diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/chats.json b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/chats.json
@@ -223,6 +223,10 @@
       "format": "date-time",
       "type": ["null", "string"]
     },
+    "update_timestamp": {
+      "format": "date-time",
+      "type": ["null", "string"]
+    },
     "triggered": {
       "type": ["null", "boolean"]
     },
@@ -281,6 +285,9 @@
     "message": {
       "type": ["null", "string"]
     },
+    "deleted": {
+      "type": ["null", "boolean"]
+    },
     "zendesk_ticket_id": {
       "type": ["null", "integer"]
     }

diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py
@@ -26,16 +26,16 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         authenticator = TokenAuthenticator(config["access_token"])
         return [
-            Agents(authenticator=authenticator),
-            AgentTimelines(authenticator=authenticator, start_date=config["start_date"]),
             Accounts(authenticator=authenticator),
-            Chats(authenticator=authenticator),
-            Shortcuts(authenticator=authenticator),
-            Triggers(authenticator=authenticator),
+            AgentTimelines(authenticator=authenticator, start_date=config["start_date"]),
+            Agents(authenticator=authenticator),
             Bans(authenticator=authenticator),
+            Chats(authenticator=authenticator, start_date=config["start_date"]),
             Departments(authenticator=authenticator),
             Goals(authenticator=authenticator),
-            Skills(authenticator=authenticator),
             Roles(authenticator=authenticator),
             RoutingSettings(authenticator=authenticator),
+            Shortcuts(authenticator=authenticator),
+            Skills(authenticator=authenticator),
+            Triggers(authenticator=authenticator),
         ]
diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py
@@ -209,11 +209,12 @@ def path(self, **kwargs) -> str:
         return "account"
 
 
-class Chats(Stream):
+class Chats(TimeIncrementalStream):
     """
-    Chats Stream: https://developer.zendesk.com/rest_api/docs/chat/chats#list-chats
+    Chats Stream: https://developer.zendesk.com/api-reference/live-chat/chat-api/incremental_export/#incremental-chat-export
     """
 
+    cursor_field = "update_timestamp"
     data_field = "chats"
 
 

diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md
@@ -62,6 +62,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.4 | 2021-11-22 | [8166](https://github.com/airbytehq/airbyte/pull/8166) | Make `Chats` stream incremental + add tests for all streams |
 | 0.1.3 | 2021-10-21 | [7210](https://github.com/airbytehq/airbyte/pull/7210) | Chats stream is only getting data from first page |
 | 0.1.2 | 2021-08-17 | [5476](https://github.com/airbytehq/airbyte/pull/5476) | Correct field unread to boolean type |
 | 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support |