diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 9e7bd4ba8d2ee..cf393bc8202ae 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -621,7 +621,7 @@ - name: Zendesk Chat sourceDefinitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 dockerRepository: airbyte/source-zendesk-chat - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-chat icon: zendesk.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 0a209ff93984c..bd1179c9b16b4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6155,7 +6155,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-zendesk-chat:0.1.3" +- dockerImage: "airbyte/source-zendesk-chat:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-chat" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile index 3d61adfccd498..4852cee384645 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile @@ -16,5 +16,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main_dev.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-zendesk-chat diff --git a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml index f6fbd69197e59..266704ef4f2f5 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml @@ -11,16 +11,17 @@ tests: - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" - configured_catalog_path: "sample_files/configured_catalog.json" + configured_catalog_path: "integration_tests/configured_catalog.json" incremental: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_incremental.json" -# Unable to use 'state_path' because Zendesk Chat API returns an error when specifying a date in the future. + configured_catalog_path: "integration_tests/configured_catalog.json" +# Unable to use 'future_state_path' because Zendesk Chat API returns an error when specifying a date in the future. # future_state_path: "integration_tests/abnormal_state.json" cursor_paths: - agents: ["id"] - bans: ["id"] + agents: [ "id" ] + bans: [ "id" ] agent_timeline: [ "start_time" ] + chats: [ "update_timestamp" ] full_refresh: - config_path: "secrets/config.json" - configured_catalog_path: "sample_files/configured_catalog.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog.json index d22ffc702c98e..ff6d03c582340 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog.json @@ -1,9 +1,120 @@ { "streams": [ + { + "stream": { + "name": "accounts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "agent_timeline", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["start_time"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["start_time"] + }, + { + "stream": { + "name": "agents", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["id"] + }, + { + "stream": { + "name": "bans", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["id"] + }, { "stream": { "name": "chats", "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["update_timestamp"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["update_timestamp"] + }, + { + "stream": { + "name": "departments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "goals", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "roles", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "shortcuts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "skills", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "triggers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "routing_settings", + "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog_incremental.json deleted file mode 100644 index e350c92fb6cd5..0000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog_incremental.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "agents", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["id"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["id"] - }, - { - "stream": { - "name": "agent_timeline", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["start_time"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["start_time"] - }, - { - "stream": { - "name": "bans", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["id"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["id"] - } - ] -} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-zendesk-chat/sample_files/configured_catalog.json index 4cde9b055ac56..ff6d03c582340 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-zendesk-chat/sample_files/configured_catalog.json @@ -9,10 +9,58 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "agent_timeline", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["start_time"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["start_time"] + }, + { + "stream": { + "name": "agents", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["id"] + }, + { + "stream": { + "name": "bans", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["id"] + }, { "stream": { "name": "chats", "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["update_timestamp"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["update_timestamp"] + }, + { + "stream": { + "name": "departments", + "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", @@ -20,7 +68,7 @@ }, { "stream": { - "name": "shortcuts", + "name": "goals", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, @@ -29,7 +77,7 @@ }, { "stream": { - "name": "triggers", + "name": "roles", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, @@ -38,7 +86,7 @@ }, { "stream": { - "name": "departments", + "name": "shortcuts", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, @@ -47,7 +95,7 @@ }, { "stream": { - "name": "goals", + "name": "skills", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, @@ -56,7 +104,7 @@ }, { "stream": { - "name": "skills", + "name": "triggers", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, @@ -65,7 +113,7 @@ }, { "stream": { - "name": "roles", + "name": "routing_settings", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/chats.json b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/chats.json index f2005c056e357..e1d4f83877aa6 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/chats.json +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/chats.json @@ -223,6 +223,10 @@ "format": "date-time", "type": ["null", "string"] }, + "update_timestamp": { + "format": "date-time", + "type": ["null", "string"] + }, "triggered": { "type": ["null", "boolean"] }, @@ -281,6 +285,9 @@ "message": { "type": ["null", "string"] }, + "deleted": { + "type": ["null", "boolean"] + }, "zendesk_ticket_id": { "type": ["null", "integer"] } diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py index 4d3f8a255c30d..caa3ebc6e4de7 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py @@ -26,16 +26,16 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = TokenAuthenticator(config["access_token"]) return [ - Agents(authenticator=authenticator), - AgentTimelines(authenticator=authenticator, start_date=config["start_date"]), Accounts(authenticator=authenticator), - Chats(authenticator=authenticator), - Shortcuts(authenticator=authenticator), - Triggers(authenticator=authenticator), + AgentTimelines(authenticator=authenticator, start_date=config["start_date"]), + Agents(authenticator=authenticator), Bans(authenticator=authenticator), + Chats(authenticator=authenticator, start_date=config["start_date"]), Departments(authenticator=authenticator), Goals(authenticator=authenticator), - Skills(authenticator=authenticator), Roles(authenticator=authenticator), RoutingSettings(authenticator=authenticator), + Shortcuts(authenticator=authenticator), + Skills(authenticator=authenticator), + Triggers(authenticator=authenticator), ] diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py index 6caa79ad81b47..7eaf9ae288ea8 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py @@ -209,11 +209,12 @@ def path(self, **kwargs) -> str: return "account" -class Chats(Stream): +class Chats(TimeIncrementalStream): """ - Chats Stream: https://developer.zendesk.com/rest_api/docs/chat/chats#list-chats + Chats Stream: https://developer.zendesk.com/api-reference/live-chat/chat-api/incremental_export/#incremental-chat-export """ + cursor_field = "update_timestamp" data_field = "chats" diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md index ba0629792372a..c2782c88e74c5 100644 --- a/docs/integrations/sources/zendesk-chat.md +++ b/docs/integrations/sources/zendesk-chat.md @@ -62,6 +62,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.4 | 2021-11-22 | [8166](https://github.com/airbytehq/airbyte/pull/8166) | Make `Chats` stream incremental + add tests for all streams | | 0.1.3 | 2021-10-21 | [7210](https://github.com/airbytehq/airbyte/pull/7210) | Chats stream is only getting data from first page | | 0.1.2 | 2021-08-17 | [5476](https://github.com/airbytehq/airbyte/pull/5476) | Correct field unread to boolean type | | 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support |