From 305b1eb7ef6affd2fa9e81dc7caba6884c3c2d20 Mon Sep 17 00:00:00 2001 From: Henri Blancke Date: Fri, 16 Dec 2022 13:18:48 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Source=20Retently:=20Add=20campa?= =?UTF-8?q?igns,=20feedback,=20outbox,=20templates=20streams=20(#19456)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [ADD] missing streams Signed-off-by: Henri Blancke * [UPD] increase request limit Signed-off-by: Henri Blancke * [UPD] update customers schema Signed-off-by: Henri Blancke * [UPD] formatting, fix unit and acceptance tests Signed-off-by: Henri Blancke * [UPD] bump version Signed-off-by: Henri Blancke * [FIX] feedback additional questions answer can be string Signed-off-by: Henri Blancke * [UPD] docs Signed-off-by: Henri Blancke * [UPD] companies pagination limit Signed-off-by: Henri Blancke * [UPD] add all streams to configured catalog Signed-off-by: Henri Blancke * [UPD] update sample catalog Signed-off-by: Henri Blancke * [UPD] add newline Co-authored-by: Haithem SOUALA * fix: modify schema files to pass SAT * Update configured_catalog.json Signed-off-by: Henri Blancke Co-authored-by: Haithem SOUALA Co-authored-by: Sajarin --- .../acceptance-test-config.yml | 3 + .../integration_tests/configured_catalog.json | 28 +++++- .../sample_files/configured_catalog.json | 50 ++++++++++- .../connectors/source-retently/setup.py | 2 +- .../source_retently/schemas/campaigns.json | 31 +++++-- .../source_retently/schemas/outbox.json | 77 ++++++++++++++++ .../source_retently/schemas/templates.json | 18 ++++ .../source-retently/source_retently/source.py | 89 ++++++++++++++++--- .../source-retently/source_retently/spec.json | 2 +- .../source-retently/unit_tests/test_source.py | 2 +- .../unit_tests/test_streams.py | 13 ++- docs/integrations/sources/retently.md | 7 +- 12 files changed, 287 insertions(+), 35 deletions(-) create mode 100644 airbyte-integrations/connectors/source-retently/source_retently/schemas/outbox.json create mode 100644 airbyte-integrations/connectors/source-retently/source_retently/schemas/templates.json diff --git a/airbyte-integrations/connectors/source-retently/acceptance-test-config.yml b/airbyte-integrations/connectors/source-retently/acceptance-test-config.yml index 35e05d2aa4e0e..1b5c42decce11 100644 --- a/airbyte-integrations/connectors/source-retently/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-retently/acceptance-test-config.yml @@ -17,3 +17,6 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: ["reports"] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-retently/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-retently/integration_tests/configured_catalog.json index 23552e0aa88c9..dcaeaf54b78ec 100644 --- a/airbyte-integrations/connectors/source-retently/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-retently/integration_tests/configured_catalog.json @@ -2,7 +2,7 @@ "streams": [ { "stream": { - "name": "customers", + "name": "campaigns", "json_schema": { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object" @@ -24,6 +24,18 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "customers", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object" + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "nps", @@ -50,7 +62,7 @@ }, { "stream": { - "name": "campaigns", + "name": "outbox", "json_schema": { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object" @@ -71,6 +83,18 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "templates", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object" + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-retently/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-retently/sample_files/configured_catalog.json index 8d19736aadcca..d6c36509573ef 100644 --- a/airbyte-integrations/connectors/source-retently/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-retently/sample_files/configured_catalog.json @@ -2,7 +2,7 @@ "streams": [ { "stream": { - "name": "customers", + "name": "campaigns", "json_schema": { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object" @@ -24,6 +24,42 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "customers", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object" + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "feedback", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object" + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "outbox", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object" + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "reports", @@ -35,6 +71,18 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "templates", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object" + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-retently/setup.py b/airbyte-integrations/connectors/source-retently/setup.py index 83501a9efc3fa..b978c63b1b732 100644 --- a/airbyte-integrations/connectors/source-retently/setup.py +++ b/airbyte-integrations/connectors/source-retently/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.2", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-retently/source_retently/schemas/campaigns.json b/airbyte-integrations/connectors/source-retently/source_retently/schemas/campaigns.json index 26d6e832dfeae..8079b5a96e543 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-retently/source_retently/schemas/campaigns.json @@ -1,13 +1,26 @@ { "type": "object", - "properties": - { - "id": { "type": "string" }, - "name": { "type": "string" }, - "isActive": { "type": "boolean" }, - "templateId": { "type": "string" }, - "metric": { "type": "string" }, - "type": { "type": "string" }, - "channel": { "type": "string" } + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "isActive": { + "type": "boolean" + }, + "templateId": { + "type": "string" + }, + "metric": { + "type": "string" + }, + "type": { + "type": "string" + }, + "channel": { + "type": "string" + } } } diff --git a/airbyte-integrations/connectors/source-retently/source_retently/schemas/outbox.json b/airbyte-integrations/connectors/source-retently/source_retently/schemas/outbox.json new file mode 100644 index 0000000000000..2bfe40896ccb2 --- /dev/null +++ b/airbyte-integrations/connectors/source-retently/source_retently/schemas/outbox.json @@ -0,0 +1,77 @@ +{ + "type": "object", + "properties": { + "email": { + "type": "string" + }, + "customerId": { + "type": "string" + }, + "firstName": { + "type": "string" + }, + "lastName": { + "type": "string" + }, + "companyName": { + "type": "string" + }, + "companyId": { + "type": "string" + }, + "sentDate": { + "type": "string" + }, + "channel": { + "type": "string" + }, + "campaign": { + "type": "string" + }, + "campaignId": { + "type": "string" + }, + "surveyTemplateId": { + "type": "string" + }, + "subject": { + "type": "string" + }, + "sentBy": { + "type": "string" + }, + "status": { + "type": "string" + }, + "detailedStatus": { + "type": "object", + "properties": { + "isOpened": { + "type": "boolean" + }, + "openedDate": { + "type": "string" + }, + "isResponded": { + "type": "boolean" + }, + "respondedDate": { + "type": "string" + }, + "hasFeedback": { + "type": "boolean" + }, + "isOptedOut": { + "type": "boolean" + }, + "isBounced": { + "type": "boolean" + } + } + }, + "additionalRecipients": { + "type": "array", + "items": { "type": "object"} + } + } +} diff --git a/airbyte-integrations/connectors/source-retently/source_retently/schemas/templates.json b/airbyte-integrations/connectors/source-retently/source_retently/schemas/templates.json new file mode 100644 index 0000000000000..e69851218c026 --- /dev/null +++ b/airbyte-integrations/connectors/source-retently/source_retently/schemas/templates.json @@ -0,0 +1,18 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "channel": { + "type": "string" + }, + "metric": { + "type": "string" + } + }, + "required": ["id", "name", "channel", "metric"] +} diff --git a/airbyte-integrations/connectors/source-retently/source_retently/source.py b/airbyte-integrations/connectors/source-retently/source_retently/source.py index e42beff2779ed..c93ab590d7b54 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/source.py +++ b/airbyte-integrations/connectors/source-retently/source_retently/source.py @@ -13,6 +13,8 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator +BASE_URL = "https://app.retently.com/api/v2/" + class SourceRetently(AbstractSource): @staticmethod @@ -35,6 +37,8 @@ def get_authenticator(config): def check_connection(self, logger, config) -> Tuple[bool, any]: try: auth = self.get_authenticator(config) + + # NOTE: not all retently instances have companies stream = Customers(auth) records = stream.read_records(sync_mode=SyncMode.full_refresh) next(records) @@ -44,15 +48,23 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = self.get_authenticator(config) - return [Customers(auth), Companies(auth), Reports(auth), Nps(auth), Campaigns(auth), Feedback(auth)] + + return [ + Campaigns(auth), + Companies(auth), + Customers(auth), + Feedback(auth), + Outbox(auth), + Reports(auth), + Nps(auth), + Templates(auth), + ] class RetentlyStream(HttpStream): primary_key = None - @property - def url_base(self) -> str: - return "https://app.retently.com/api/v2/" + url_base = BASE_URL @property @abstractmethod @@ -62,10 +74,9 @@ def json_path(self): def parse_response( self, response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, + **kwargs, ) -> Iterable[Mapping]: + data = response.json().get("data") stream_data = data.get(self.json_path) if self.json_path else data for d in stream_data: @@ -88,32 +99,71 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - return next_page_token + next_page = next_page_token or {} + return { + # The companies endpoint only supports limit 100 + "limit": 1000 if self.json_path != "companies" else 100, + **next_page, + } + + +class Campaigns(RetentlyStream): + json_path = "campaigns" + + def path(self, **kwargs) -> str: + return "campaigns" + + # does not support pagination + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + +class Companies(RetentlyStream): + json_path = "companies" + + def path( + self, + **kwargs, + ) -> str: + return "companies" class Customers(RetentlyStream): json_path = "subscribers" def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + self, + **kwargs, ) -> str: return "nps/customers" -class Companies(RetentlyStream): - json_path = "companies" +class Feedback(RetentlyStream): + json_path = "responses" def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + self, + **kwargs, ) -> str: - return "companies" + return "feedback" + + +class Outbox(RetentlyStream): + json_path = "surveys" + + def path( + self, + **kwargs, + ) -> str: + return "nps/outbox" class Reports(RetentlyStream): json_path = None def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + self, + **kwargs, ) -> str: return "reports" # does not support pagination @@ -132,6 +182,16 @@ def path( def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None + +class Templates(RetentlyStream): + json_path = "templates" + + def path( + self, + **kwargs, + ) -> str: + return "templates" + def parse_response( self, response: requests.Response, @@ -169,3 +229,4 @@ def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return "feedback" + diff --git a/airbyte-integrations/connectors/source-retently/source_retently/spec.json b/airbyte-integrations/connectors/source-retently/source_retently/spec.json index 59b7f3408017c..bec3525fa1851 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/spec.json +++ b/airbyte-integrations/connectors/source-retently/source_retently/spec.json @@ -1,5 +1,5 @@ { - "documentationUrl": "https://docsurl.com", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/retently", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Retently Api Spec", diff --git a/airbyte-integrations/connectors/source-retently/unit_tests/test_source.py b/airbyte-integrations/connectors/source-retently/unit_tests/test_source.py index 18bf1bd5c4c96..2a41a774598f7 100644 --- a/airbyte-integrations/connectors/source-retently/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-retently/unit_tests/test_source.py @@ -28,5 +28,5 @@ def test_streams(mocker): source = SourceRetently() config_mock = MagicMock() streams = source.streams(config_mock) - expected_streams_number = 6 + expected_streams_number = 8 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-retently/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-retently/unit_tests/test_streams.py index 92c941d2bfb2b..ff429279b1a8f 100644 --- a/airbyte-integrations/connectors/source-retently/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-retently/unit_tests/test_streams.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock import pytest -from source_retently.source import Companies +from source_retently.source import Campaigns, Companies @pytest.fixture @@ -18,10 +18,17 @@ def patch_base_class(mocker): mocker.patch.object(Companies, "__abstractmethods__", set()) -def test_request_params(patch_base_class): +def test_request_params_companies(patch_base_class): stream = Companies() inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = None + expected_params = {'limit': 100} + assert stream.request_params(**inputs) == expected_params + + +def test_request_params_other(patch_base_class): + stream = Campaigns() + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {'limit': 1000} assert stream.request_params(**inputs) == expected_params diff --git a/docs/integrations/sources/retently.md b/docs/integrations/sources/retently.md index d69ec17ef93f3..d0c5a393d6ec1 100644 --- a/docs/integrations/sources/retently.md +++ b/docs/integrations/sources/retently.md @@ -41,6 +41,7 @@ OAuth application is [here](https://app.retently.com/settings/oauth). | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.2 | 2021-12-28 | [9045](https://github.com/airbytehq/airbyte/pull/9045) | Update titles and descriptions | -| 0.1.1 | 2021-12-06 | [8043](https://github.com/airbytehq/airbyte/pull/8043) | 🎉 Source Retently: add OAuth 2.0 | -| 0.1.0 | 2021-11-02 | [6966](https://github.com/airbytehq/airbyte/pull/6966) | 🎉 New Source: Retently | +| 0.1.3 | 2022-11-15 | [19456](https://github.com/airbytehq/airbyte/pull/19456) | Add campaign, feedback, outbox and templates streams | +| 0.1.2 | 2021-12-28 | [9045](https://github.com/airbytehq/airbyte/pull/9045) | Update titles and descriptions | +| 0.1.1 | 2021-12-06 | [8043](https://github.com/airbytehq/airbyte/pull/8043) | 🎉 Source Retently: add OAuth 2.0 | +| 0.1.0 | 2021-11-02 | [6966](https://github.com/airbytehq/airbyte/pull/6966) | 🎉 New Source: Retently |