From f3fe0fd3f16232f2e236e9e55017e0150a44e006 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 14 Jun 2022 17:39:13 +0300 Subject: [PATCH 1/8] fixed the bug, bumped the version --- .../source-zendesk-support/Dockerfile | 2 +- .../source_zendesk_support/streams.py | 22 ++++++++++++------- .../unit_tests/unit_test.py | 6 +++-- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile index f07d4a07ed6ca..feab96bbdec3c 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile @@ -25,5 +25,5 @@ COPY source_zendesk_support ./source_zendesk_support ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.9 +LABEL io.airbyte.version=0.2.10 LABEL io.airbyte.name=airbyte/source-zendesk-support diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index 1dcb8bab758a4..ee9c55015216e 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -335,7 +335,7 @@ class SourceZendeskSupportCursorPaginationStream(SourceZendeskSupportFullRefresh """ Endpoints provide a cursor pagination and sorting mechanism """ - + cursor_field = "updated_at" next_page_field = "next_page" prev_start_time = None @@ -379,7 +379,7 @@ class SourceZendeskIncrementalExportStream(SourceZendeskSupportCursorPaginationS more info: https://developer.zendesk.com/documentation/ticketing/using-the-zendesk-api/side_loading/#supported-endpoints """ - cursor_field = "updated_at" + # cursor_field = "updated_at" response_list_name: str = None sideload_param: str = None @@ -483,8 +483,6 @@ class Groups(SourceZendeskSupportStream): class GroupMemberships(SourceZendeskSupportCursorPaginationStream): """GroupMemberships stream: https://developer.zendesk.com/api-reference/ticketing/groups/group_memberships/""" - cursor_field = "updated_at" - class SatisfactionRatings(SourceZendeskSupportStream): """SatisfactionRatings stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/satisfaction_ratings/ @@ -517,12 +515,20 @@ class TicketFields(SourceZendeskSupportStream): class TicketForms(SourceZendeskSupportCursorPaginationStream): """TicketForms stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_forms/""" - cursor_field = "updated_at" - -class TicketMetrics(SourceZendeskSupportStream): +class TicketMetrics(SourceZendeskSupportCursorPaginationStream): """TicketMetric stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metrics/""" - + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + next_page = self._parse_next_page_number(response) + return next_page if next_page else None + + def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = {"start_time": self.check_stream_state(stream_state)} + if next_page_token: + params.update({"page": next_page_token or 1, "per_page": self.page_size}) + return params + class TicketMetricEvents(SourceZendeskSupportCursorPaginationStream): """TicketMetricEvents stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metric_events/""" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index b815ca81aef58..1c3eed72a7b4e 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -408,7 +408,7 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte (Groups, None), (SatisfactionRatings, None), (TicketFields, None), - (TicketMetrics, None), + # (TicketMetrics, None), ], ids=[ "Macros", @@ -416,7 +416,7 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte "Groups", "SatisfactionRatings", "TicketFields", - "TicketMetrics", + # "TicketMetrics", ], ) def test_next_page_token(self, stream_cls, expected): @@ -555,12 +555,14 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte (TicketForms), (TicketMetricEvents), (TicketAudits), + (TicketMetrics), ], ids=[ "GroupMemberships", "TicketForms", "TicketMetricEvents", "TicketAudits", + "TicketMetrics", ], ) def test_next_page_token(self, requests_mock, stream_cls): From f5e06950a78017590039e9845b4e74dd5c9db86d Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 14 Jun 2022 17:43:47 +0300 Subject: [PATCH 2/8] updated changelog --- docs/integrations/sources/zendesk-support.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/zendesk-support.md b/docs/integrations/sources/zendesk-support.md index 131632b3b52c7..19413e16b82f4 100644 --- a/docs/integrations/sources/zendesk-support.md +++ b/docs/integrations/sources/zendesk-support.md @@ -85,6 +85,7 @@ The Zendesk connector should not run into Zendesk API limitations under normal u | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `0.2.10` | 2022-06-14 | [13757](https://github.com/airbytehq/airbyte/pull/13757) | Fixed the bug with `TicketMetrics` stream, HTTP Error 429, caused by lots of API requests | | `0.2.9` | 2022-05-27 | [13261](https://github.com/airbytehq/airbyte/pull/13261) | Bugfix for the unhandled [ChunkedEncodingError](https://github.com/airbytehq/airbyte/issues/12591) and [ConnectionError](https://github.com/airbytehq/airbyte/issues/12155) | | `0.2.8` | 2022-05-20 | [13055](https://github.com/airbytehq/airbyte/pull/13055) | Fixed minor issue for stream `ticket_audits` schema | | `0.2.7` | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Adding fixtures to mock time.sleep for connectors that explicitly sleep | From 34b6441c7e4f7cf631bc98a054dbf882fa942b87 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 14 Jun 2022 17:45:11 +0300 Subject: [PATCH 3/8] cleared code --- .../source-zendesk-support/source_zendesk_support/streams.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index ee9c55015216e..fc046a74f292a 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -379,7 +379,6 @@ class SourceZendeskIncrementalExportStream(SourceZendeskSupportCursorPaginationS more info: https://developer.zendesk.com/documentation/ticketing/using-the-zendesk-api/side_loading/#supported-endpoints """ - # cursor_field = "updated_at" response_list_name: str = None sideload_param: str = None From c167488d9ec3d76357523ae0e819c18b7efe13dd Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 14 Jun 2022 17:46:27 +0300 Subject: [PATCH 4/8] adopted unit_tests --- .../connectors/source-zendesk-support/unit_tests/unit_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index 1c3eed72a7b4e..ede0ef4f260a4 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -408,7 +408,6 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte (Groups, None), (SatisfactionRatings, None), (TicketFields, None), - # (TicketMetrics, None), ], ids=[ "Macros", @@ -416,7 +415,6 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte "Groups", "SatisfactionRatings", "TicketFields", - # "TicketMetrics", ], ) def test_next_page_token(self, stream_cls, expected): From 7c47ea2b24e3b2025f38ce25f7f3fff236480350 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 15 Jun 2022 16:07:50 +0300 Subject: [PATCH 5/8] updated SatisfactionRatings stream --- .../source_zendesk_support/streams.py | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index fc046a74f292a..a276f2e275eb3 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -483,27 +483,28 @@ class GroupMemberships(SourceZendeskSupportCursorPaginationStream): """GroupMemberships stream: https://developer.zendesk.com/api-reference/ticketing/groups/group_memberships/""" -class SatisfactionRatings(SourceZendeskSupportStream): +class SatisfactionRatings(SourceZendeskSupportCursorPaginationStream): """SatisfactionRatings stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/satisfaction_ratings/ The ZenDesk API for this stream provides the filter "start_time" that can be used for incremental logic """ + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + next_page = self._parse_next_page_number(response) + return next_page if next_page else None + def request_params( self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs ) -> MutableMapping[str, Any]: - """Adds the filtering field 'start_time'""" - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) + params = { + "page": 1, + "per_page": self.page_size, + "sort_by": "asc" + } start_time = self.str2unixtime((stream_state or {}).get(self.cursor_field)) - - if not start_time: - start_time = self.str2unixtime(self._start_date) - params.update( - { - "start_time": start_time, - "sort_by": "asc", - } - ) + params["start_time"] = start_time if start_time else self.str2unixtime(self._start_date) + if next_page_token: + params["page"] = next_page_token return params @@ -523,9 +524,13 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, return next_page if next_page else None def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = {"start_time": self.check_stream_state(stream_state)} + params = { + "start_time": self.check_stream_state(stream_state), + "page": 1, + "per_page": self.page_size, + } if next_page_token: - params.update({"page": next_page_token or 1, "per_page": self.page_size}) + params['page'] = next_page_token return params From 7d0d0ea9d5a70406df408e95e31649cbd8facbf4 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 15 Jun 2022 16:09:04 +0300 Subject: [PATCH 6/8] updated code --- .../source_zendesk_support/streams.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index a276f2e275eb3..3892e0eb9b6f1 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -484,9 +484,8 @@ class GroupMemberships(SourceZendeskSupportCursorPaginationStream): class SatisfactionRatings(SourceZendeskSupportCursorPaginationStream): - """SatisfactionRatings stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/satisfaction_ratings/ - - The ZenDesk API for this stream provides the filter "start_time" that can be used for incremental logic + """ + SatisfactionRatings stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/satisfaction_ratings/ """ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -535,7 +534,9 @@ def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token class TicketMetricEvents(SourceZendeskSupportCursorPaginationStream): - """TicketMetricEvents stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metric_events/""" + """ + TicketMetricEvents stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metric_events/ + """ cursor_field = "time" From c86f9718f4d4570aa7e2305ea75f27d50d6658e6 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 15 Jun 2022 16:30:19 +0300 Subject: [PATCH 7/8] updated unit_tests --- .../source-zendesk-support/unit_tests/unit_test.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index ede0ef4f260a4..f7165121c809d 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -15,7 +15,7 @@ import requests from airbyte_cdk import AirbyteLogger from source_zendesk_support.source import BasicApiTokenAuthenticator, SourceZendeskSupport -from source_zendesk_support.streams import ( # streams +from source_zendesk_support.streams import ( DATETIME_FORMAT, END_OF_STREAM_KEY, LAST_END_TIME_KEY, @@ -406,14 +406,12 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte (Macros, None), (Organizations, None), (Groups, None), - (SatisfactionRatings, None), (TicketFields, None), ], ids=[ "Macros", "Organizations", "Groups", - "SatisfactionRatings", "TicketFields", ], ) @@ -428,17 +426,13 @@ def test_next_page_token(self, stream_cls, expected): (Macros, {"start_time": 1622505600}), (Organizations, {"start_time": 1622505600}), (Groups, {"start_time": 1622505600}), - (SatisfactionRatings, {"start_time": 1622505600, "sort_by": "asc"}), (TicketFields, {"start_time": 1622505600}), - (TicketMetrics, {"start_time": 1622505600}), ], ids=[ "Macros", "Organizations", "Groups", - "SatisfactionRatings", "TicketFields", - "TicketMetrics", ], ) def test_request_params(self, stream_cls, expected): @@ -554,6 +548,7 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte (TicketMetricEvents), (TicketAudits), (TicketMetrics), + (SatisfactionRatings), ], ids=[ "GroupMemberships", @@ -561,6 +556,7 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte "TicketMetricEvents", "TicketAudits", "TicketMetrics", + "SatisfactionRatings", ], ) def test_next_page_token(self, requests_mock, stream_cls): @@ -598,12 +594,16 @@ def test_check_stream_state(self, stream_cls, expected): (TicketForms, {"start_time": 1622505600}), (TicketMetricEvents, {"start_time": 1622505600}), (TicketAudits, {"sort_by": "created_at", "sort_order": "desc", "limit": 1000}), + (SatisfactionRatings, {'page': 1, 'per_page': 100, 'sort_by': 'asc', 'start_time': 1622505600}), + (TicketMetrics, {'page': 1, 'per_page': 100, 'start_time': 1622505600}), ], ids=[ "GroupMemberships", "TicketForms", "TicketMetricEvents", "TicketAudits", + "SatisfactionRatings", + "TicketMetrics", ], ) def test_request_params(self, stream_cls, expected): From fa3d58fcb029611d187e630a5762c68689557763 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 15 Jun 2022 14:32:27 +0000 Subject: [PATCH 8/8] auto-bump connector version --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../source_zendesk_support/streams.py | 19 +++++++++---------- .../unit_tests/unit_test.py | 4 ++-- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 9cd6d3c45c76c..51585b9a763d8 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1001,7 +1001,7 @@ - name: Zendesk Support sourceDefinitionId: 79c1aa37-dae3-42ae-b333-d1c105477715 dockerRepository: airbyte/source-zendesk-support - dockerImageTag: 0.2.9 + dockerImageTag: 0.2.10 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-support icon: zendesk.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index a8c059ce8f6af..2d3215dd0fd73 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -9678,7 +9678,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-zendesk-support:0.2.9" +- dockerImage: "airbyte/source-zendesk-support:0.2.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-support" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index 3892e0eb9b6f1..65b31762195b8 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -335,6 +335,7 @@ class SourceZendeskSupportCursorPaginationStream(SourceZendeskSupportFullRefresh """ Endpoints provide a cursor pagination and sorting mechanism """ + cursor_field = "updated_at" next_page_field = "next_page" prev_start_time = None @@ -495,11 +496,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs ) -> MutableMapping[str, Any]: - params = { - "page": 1, - "per_page": self.page_size, - "sort_by": "asc" - } + params = {"page": 1, "per_page": self.page_size, "sort_by": "asc"} start_time = self.str2unixtime((stream_state or {}).get(self.cursor_field)) params["start_time"] = start_time if start_time else self.str2unixtime(self._start_date) if next_page_token: @@ -517,21 +514,23 @@ class TicketForms(SourceZendeskSupportCursorPaginationStream): class TicketMetrics(SourceZendeskSupportCursorPaginationStream): """TicketMetric stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metrics/""" - + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: next_page = self._parse_next_page_number(response) return next_page if next_page else None - - def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + + def request_params( + self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: params = { "start_time": self.check_stream_state(stream_state), "page": 1, "per_page": self.page_size, } if next_page_token: - params['page'] = next_page_token + params["page"] = next_page_token return params - + class TicketMetricEvents(SourceZendeskSupportCursorPaginationStream): """ diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index f7165121c809d..35537455fe54a 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -594,8 +594,8 @@ def test_check_stream_state(self, stream_cls, expected): (TicketForms, {"start_time": 1622505600}), (TicketMetricEvents, {"start_time": 1622505600}), (TicketAudits, {"sort_by": "created_at", "sort_order": "desc", "limit": 1000}), - (SatisfactionRatings, {'page': 1, 'per_page': 100, 'sort_by': 'asc', 'start_time': 1622505600}), - (TicketMetrics, {'page': 1, 'per_page': 100, 'start_time': 1622505600}), + (SatisfactionRatings, {"page": 1, "per_page": 100, "sort_by": "asc", "start_time": 1622505600}), + (TicketMetrics, {"page": 1, "per_page": 100, "start_time": 1622505600}), ], ids=[ "GroupMemberships",