🐛 Source Zendesk Support: fixed 429 error for TicketMetrics stream

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -1001,7 +1001,7 @@
 - name: Zendesk Support
   sourceDefinitionId: 79c1aa37-dae3-42ae-b333-d1c105477715
   dockerRepository: airbyte/source-zendesk-support
-  dockerImageTag: 0.2.9
+  dockerImageTag: 0.2.10
   documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-support
   icon: zendesk.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -9678,7 +9678,7 @@
               path_in_connector_config:
               - "credentials"
               - "client_secret"
-- dockerImage: "airbyte/source-zendesk-support:0.2.9"
+- dockerImage: "airbyte/source-zendesk-support:0.2.10"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-support"
     connectionSpecification:

airbyte-integrations/connectors/source-zendesk-support/Dockerfile

@@ -25,5 +25,5 @@ COPY source_zendesk_support ./source_zendesk_support
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.9
+LABEL io.airbyte.version=0.2.10
 LABEL io.airbyte.name=airbyte/source-zendesk-support

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py

@@ -336,6 +336,7 @@ class SourceZendeskSupportCursorPaginationStream(SourceZendeskSupportFullRefresh
     Endpoints provide a cursor pagination and sorting mechanism
     """
 
+    cursor_field = "updated_at"
     next_page_field = "next_page"
     prev_start_time = None
 
@@ -379,7 +380,6 @@ class SourceZendeskIncrementalExportStream(SourceZendeskSupportCursorPaginationS
         more info: https://developer.zendesk.com/documentation/ticketing/using-the-zendesk-api/side_loading/#supported-endpoints
     """
 
-    cursor_field = "updated_at"
     response_list_name: str = None
     sideload_param: str = None
 
@@ -483,30 +483,24 @@ class Groups(SourceZendeskSupportStream):
 class GroupMemberships(SourceZendeskSupportCursorPaginationStream):
     """GroupMemberships stream: https://developer.zendesk.com/api-reference/ticketing/groups/group_memberships/"""
 
-    cursor_field = "updated_at"
-
 
-class SatisfactionRatings(SourceZendeskSupportStream):
-    """SatisfactionRatings stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/satisfaction_ratings/
-
-    The ZenDesk API for this stream provides the filter "start_time" that can be used for incremental logic
+class SatisfactionRatings(SourceZendeskSupportCursorPaginationStream):
+    """
+    SatisfactionRatings stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/satisfaction_ratings/
     """
 
+    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+        next_page = self._parse_next_page_number(response)
+        return next_page if next_page else None
+
     def request_params(
         self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs
     ) -> MutableMapping[str, Any]:
-        """Adds the filtering field 'start_time'"""
-        params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs)
+        params = {"page": 1, "per_page": self.page_size, "sort_by": "asc"}
         start_time = self.str2unixtime((stream_state or {}).get(self.cursor_field))
-
-        if not start_time:
-            start_time = self.str2unixtime(self._start_date)
-        params.update(
-            {
-                "start_time": start_time,
-                "sort_by": "asc",
-            }
-        )
+        params["start_time"] = start_time if start_time else self.str2unixtime(self._start_date)
+        if next_page_token:
+            params["page"] = next_page_token
         return params
 
 
@@ -517,15 +511,31 @@ class TicketFields(SourceZendeskSupportStream):
 class TicketForms(SourceZendeskSupportCursorPaginationStream):
     """TicketForms stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_forms/"""
 
-    cursor_field = "updated_at"
-
 
-class TicketMetrics(SourceZendeskSupportStream):
+class TicketMetrics(SourceZendeskSupportCursorPaginationStream):
     """TicketMetric stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metrics/"""
 
+    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+        next_page = self._parse_next_page_number(response)
+        return next_page if next_page else None
+
+    def request_params(
+        self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs
+    ) -> MutableMapping[str, Any]:
+        params = {
+            "start_time": self.check_stream_state(stream_state),
+            "page": 1,
+            "per_page": self.page_size,
+        }
+        if next_page_token:
+            params["page"] = next_page_token
+        return params
+
 
 class TicketMetricEvents(SourceZendeskSupportCursorPaginationStream):
-    """TicketMetricEvents stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metric_events/"""
+    """
+    TicketMetricEvents stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metric_events/
+    """
 
     cursor_field = "time"
 

airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py

@@ -15,7 +15,7 @@
 import requests
 from airbyte_cdk import AirbyteLogger
 from source_zendesk_support.source import BasicApiTokenAuthenticator, SourceZendeskSupport
-from source_zendesk_support.streams import (  # streams
+from source_zendesk_support.streams import (
     DATETIME_FORMAT,
     END_OF_STREAM_KEY,
     LAST_END_TIME_KEY,
@@ -406,17 +406,13 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte
             (Macros, None),
             (Organizations, None),
             (Groups, None),
-            (SatisfactionRatings, None),
             (TicketFields, None),
-            (TicketMetrics, None),
         ],
         ids=[
             "Macros",
             "Organizations",
             "Groups",
-            "SatisfactionRatings",
             "TicketFields",
-            "TicketMetrics",
         ],
     )
     def test_next_page_token(self, stream_cls, expected):
@@ -430,17 +426,13 @@ def test_next_page_token(self, stream_cls, expected):
             (Macros, {"start_time": 1622505600}),
             (Organizations, {"start_time": 1622505600}),
             (Groups, {"start_time": 1622505600}),
-            (SatisfactionRatings, {"start_time": 1622505600, "sort_by": "asc"}),
             (TicketFields, {"start_time": 1622505600}),
-            (TicketMetrics, {"start_time": 1622505600}),
         ],
         ids=[
             "Macros",
             "Organizations",
             "Groups",
-            "SatisfactionRatings",
             "TicketFields",
-            "TicketMetrics",
         ],
     )
     def test_request_params(self, stream_cls, expected):
@@ -555,12 +547,16 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte
             (TicketForms),
             (TicketMetricEvents),
             (TicketAudits),
+            (TicketMetrics),
+            (SatisfactionRatings),
         ],
         ids=[
             "GroupMemberships",
             "TicketForms",
             "TicketMetricEvents",
             "TicketAudits",
+            "TicketMetrics",
+            "SatisfactionRatings",
         ],
     )
     def test_next_page_token(self, requests_mock, stream_cls):
@@ -598,12 +594,16 @@ def test_check_stream_state(self, stream_cls, expected):
             (TicketForms, {"start_time": 1622505600}),
             (TicketMetricEvents, {"start_time": 1622505600}),
             (TicketAudits, {"sort_by": "created_at", "sort_order": "desc", "limit": 1000}),
+            (SatisfactionRatings, {"page": 1, "per_page": 100, "sort_by": "asc", "start_time": 1622505600}),
+            (TicketMetrics, {"page": 1, "per_page": 100, "start_time": 1622505600}),
         ],
         ids=[
             "GroupMemberships",
             "TicketForms",
             "TicketMetricEvents",
             "TicketAudits",
+            "SatisfactionRatings",
+            "TicketMetrics",
         ],
     )
     def test_request_params(self, stream_cls, expected):

docs/integrations/sources/zendesk-support.md

@@ -85,6 +85,7 @@ The Zendesk connector should not run into Zendesk API limitations under normal u
 
 | Version  | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                            |
 |:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `0.2.10`  | 2022-06-14 | [13757](https://github.com/airbytehq/airbyte/pull/13757) | Fixed the bug with `TicketMetrics` stream, HTTP Error 429, caused by lots of API requests | 
 | `0.2.9`  | 2022-05-27 | [13261](https://github.com/airbytehq/airbyte/pull/13261) | Bugfix for the unhandled [ChunkedEncodingError](https://github.com/airbytehq/airbyte/issues/12591) and [ConnectionError](https://github.com/airbytehq/airbyte/issues/12155)                                                        |
 | `0.2.8`  | 2022-05-20 | [13055](https://github.com/airbytehq/airbyte/pull/13055) | Fixed minor issue for stream `ticket_audits` schema                                                                                                                                                                                |
 | `0.2.7`  | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Adding fixtures to mock time.sleep for connectors that explicitly sleep                                                                                                                                                            |