From a6313ec0c85e40025e5acf448bb79b782fc2c801 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Fri, 4 Nov 2022 20:27:29 +0000 Subject: [PATCH 1/7] get_typed_env added Signed-off-by: Sergey Chvalyuk --- .../integration_tests/spec.json | 22 +++------------ .../source_amazon_ads/spec.yaml | 27 +++---------------- .../streams/report_streams/report_streams.py | 10 ++++--- .../source_amazon_ads/utils.py | 17 ++++++++++++ 4 files changed, 29 insertions(+), 47 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json index d1ba1851a4b5f..9400deac956e6 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json @@ -38,33 +38,17 @@ "default": "NA", "order": 4 }, - "report_wait_timeout": { - "title": "Report Wait Timeout", - "description": "Timeout duration in minutes for Reports. Default is 60 minutes.", - "default": 60, - "examples": [60, 120], - "order": 5, - "type": "integer" - }, - "report_generation_max_retries": { - "title": "Report Generation Maximum Retries", - "description": "Maximum retries Airbyte will attempt for fetching report data. Default is 5.", - "default": 5, - "examples": [5, 10, 15], - "order": 6, - "type": "integer" - }, "start_date": { "title": "Start Date", "description": "The Start date for collecting reports, should not be more than 60 days in the past. In YYYY-MM-DD format", "examples": ["2022-10-10", "2022-10-22"], - "order": 7, + "order": 5, "type": "string" }, "profiles": { "title": "Profile IDs", "description": "Profile IDs you want to fetch data for. See docs for more details.", - "order": 8, + "order": 6, "type": "array", "items": { "type": "integer" @@ -79,7 +63,7 @@ }, "type": "array", "uniqueItems": true, - "order": 9 + "order": 7 } }, "required": ["client_id", "client_secret", "refresh_token"], diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml index 5523b774e86fc..d77e1120f72fb 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml @@ -46,27 +46,6 @@ connectionSpecification: type: string default: NA order: 4 - report_wait_timeout: - title: Report Wait Timeout - description: Timeout duration in minutes for Reports. Default is 60 minutes. - default: 60 - examples: - - 60 - - 120 - order: 5 - type: integer - report_generation_max_retries: - title: Report Generation Maximum Retries - description: - Maximum retries Airbyte will attempt for fetching report data. - Default is 5. - default: 5 - examples: - - 5 - - 10 - - 15 - order: 6 - type: integer start_date: title: Start Date description: @@ -75,14 +54,14 @@ connectionSpecification: examples: - "2022-10-10" - "2022-10-22" - order: 7 + order: 5 type: string profiles: title: Profile IDs description: Profile IDs you want to fetch data for. See docs for more details. - order: 8 + order: 6 type: array items: type: integer @@ -97,7 +76,7 @@ connectionSpecification: - archived type: array uniqueItems: true - order: 9 + order: 7 required: - client_id - client_secret diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 9945f65edbe59..516ac561f90e0 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -22,7 +22,7 @@ from pydantic import BaseModel from source_amazon_ads.schemas import CatalogModel, MetricsReport, Profile from source_amazon_ads.streams.common import BasicAmazonAdsStream -from source_amazon_ads.utils import iterate_one_by_one +from source_amazon_ads.utils import get_typed_env, iterate_one_by_one class RecordType(str, Enum): @@ -112,14 +112,16 @@ class ReportStream(BasicAmazonAdsStream, ABC): ] def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authenticator: Oauth2Authenticator): + super().__init__(config, profiles) self._state = {} self._authenticator = authenticator self._session = requests.Session() self._model = self._generate_model() - self.report_wait_timeout = config.get("report_wait_timeout", 60) - self.report_generation_maximum_retries = config.get("report_generation_max_retries", 5) self._start_date: Optional[Date] = config.get("start_date") - super().__init__(config, profiles) + # Timeout duration in minutes for Reports + self.report_wait_timeout: int = get_typed_env("REPORT_WAIT_TIMEOUT", 60) + # Maximum retries Airbyte will attempt for fetching report data + self.report_generation_maximum_retries: int = get_typed_env("REPORT_GENERATION_MAX_RETRIES", 5) @property def model(self) -> CatalogModel: diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py index caa66c1d13bbb..881567c04c9a1 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py @@ -2,6 +2,12 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import logging +import os +from typing import Union + +logger = logging.getLogger("airbyte") + def iterate_one_by_one(*iterables): iterables = list(iterables) @@ -13,3 +19,14 @@ def iterate_one_by_one(*iterables): pass else: iterables.append(iterable) + + +def get_typed_env(name: str, default: Union[str, int]) -> Union[str, int]: + convert = type(default) + assert convert in [str, int] + value = os.environ.get(name, default) + try: + return convert(value) + except ValueError: + logger.warning(f"Cannot convert environment variable {name}={value!r} to type {convert}") + return default From 5d27ce8d9afaa1bc1ff62df4e4b3219cfcccc90b Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 8 Nov 2022 08:33:23 +0200 Subject: [PATCH 2/7] REPORT_WAIT_TIMEOUT 60 -> 180 Signed-off-by: Sergey Chvalyuk --- .../streams/report_streams/report_streams.py | 2 +- .../unit_tests/test_report_streams.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 516ac561f90e0..fa7fe075c9f2e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -119,7 +119,7 @@ def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authentic self._model = self._generate_model() self._start_date: Optional[Date] = config.get("start_date") # Timeout duration in minutes for Reports - self.report_wait_timeout: int = get_typed_env("REPORT_WAIT_TIMEOUT", 60) + self.report_wait_timeout: int = get_typed_env("REPORT_WAIT_TIMEOUT", 180) # Maximum retries Airbyte will attempt for fetching report data self.report_generation_maximum_retries: int = get_typed_env("REPORT_GENERATION_MAX_RETRIES", 5) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index f53a10777ae06..a72a5f709e33f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -244,7 +244,7 @@ def test_display_report_stream_init_too_many_requests(mocker, config): ), ( [ - (lambda x: x > 5, None, "2021-01-02 04:04:05"), + (lambda x: x > 5, None, "2021-01-02 06:04:05"), ], ReportGenerationInProgress, ), @@ -259,11 +259,11 @@ def test_display_report_stream_init_too_many_requests(mocker, config): ( [ (lambda x: True, "FAILURE", None), - (lambda x: x >= 10, None, "2021-01-02 04:04:05"), - (lambda x: x >= 15, None, "2021-01-02 05:04:05"), - (lambda x: x >= 20, None, "2021-01-02 06:04:05"), - (lambda x: x >= 25, None, "2021-01-02 07:04:05"), - (lambda x: x >= 30, None, "2021-01-02 08:04:05"), + (lambda x: x >= 10, None, "2021-01-02 06:04:05"), + (lambda x: x >= 15, None, "2021-01-02 09:04:05"), + (lambda x: x >= 20, None, "2021-01-02 12:04:05"), + (lambda x: x >= 25, None, "2021-01-02 15:04:05"), + (lambda x: x >= 30, None, "2021-01-02 18:04:05"), ], ReportGenerationFailure, ), From ec55a437bf02bbe0aab1eb5435b5fd7d2c4a57f6 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 8 Nov 2022 08:54:46 +0200 Subject: [PATCH 3/7] test_get_typed_env added Signed-off-by: Sergey Chvalyuk --- .../streams/report_streams/report_streams.py | 4 ++-- .../source-amazon-ads/unit_tests/test_utils.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_utils.py diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index fa7fe075c9f2e..9e3feb68e0d14 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -118,9 +118,9 @@ def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authentic self._session = requests.Session() self._model = self._generate_model() self._start_date: Optional[Date] = config.get("start_date") - # Timeout duration in minutes for Reports + # Timeout duration in minutes for Reports. Default is 180 minutes. self.report_wait_timeout: int = get_typed_env("REPORT_WAIT_TIMEOUT", 180) - # Maximum retries Airbyte will attempt for fetching report data + # Maximum retries Airbyte will attempt for fetching report data. Default is 5. self.report_generation_maximum_retries: int = get_typed_env("REPORT_GENERATION_MAX_RETRIES", 5) @property diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_utils.py new file mode 100644 index 0000000000000..6c488e674425a --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_utils.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from source_amazon_ads.utils import get_typed_env + + +def test_get_typed_env(monkeypatch): + assert get_typed_env("REPORT_WAIT_TIMEOUT", 180) == 180 + assert get_typed_env("BOOLEAN_PARAM", "1") == "1" + assert get_typed_env("STRING_PARAM", "string") == "string" + monkeypatch.setenv("REPORT_WAIT_TIMEOUT", "60") + assert get_typed_env("REPORT_WAIT_TIMEOUT", 180) == 60 + monkeypatch.setenv("REPORT_WAIT_TIMEOUT", "60") + assert get_typed_env("REPORT_WAIT_TIMEOUT", "180") == "60" + monkeypatch.setenv("REPORT_WAIT_TIMEOUT", "string") + assert get_typed_env("REPORT_WAIT_TIMEOUT", 180) == 180 From a52a4edf0572eb600fd1d6aa712055a856f4ee7f Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 8 Nov 2022 09:02:11 +0200 Subject: [PATCH 4/7] get_error_display_message updated Signed-off-by: Sergey Chvalyuk --- .../source_amazon_ads/streams/report_streams/report_streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 9e3feb68e0d14..b315fd6ae4674 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -409,7 +409,7 @@ def _download_report(self, report_info: ReportInfo, url: str) -> List[dict]: def get_error_display_message(self, exception: BaseException) -> Optional[str]: if isinstance(exception, ReportGenerationInProgress): - return f'Report(s) generation time took more than {self.report_wait_timeout} minutes, please increase the "report_wait_timeout" parameter in configuration.' + return f'Report(s) generation time took more than {self.report_wait_timeout} minutes and failed because of Amazon API issues. Please wait some time and run synchronization again.' return super().get_error_display_message(exception) def _get_response_error_details(self, response) -> Optional[str]: From 72d87c15111945386a51d74fde782eeb32a77722 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 8 Nov 2022 09:03:20 +0200 Subject: [PATCH 5/7] bump 0.1.25 Signed-off-by: Sergey Chvalyuk --- airbyte-integrations/connectors/source-amazon-ads/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index 68fcde896d8f7..a47bfb1d7151e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.24 +LABEL io.airbyte.version=0.1.25 LABEL io.airbyte.name=airbyte/source-amazon-ads From 9d4b51340691b6f07099b9f1485907e34d8e4cff Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 8 Nov 2022 09:05:52 +0200 Subject: [PATCH 6/7] amazon-ads.md updated Signed-off-by: Sergey Chvalyuk --- docs/integrations/sources/amazon-ads.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index acdb987934c4c..5c804ce5501b7 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -91,7 +91,8 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| -| 0.1.24 | 2022-10-19 | [17475](https://github.com/airbytehq/airbyte/pull/17475) | Add filters for state on brand, product and display campaigns | +| 0.1.25 | 2022-11-08 | [18985](https://github.com/airbytehq/airbyte/pull/18985) | Remove "report_wait_timeout", "report_generation_max_retries" from config | +| 0.1.24 | 2022-10-19 | [17475](https://github.com/airbytehq/airbyte/pull/17475) | Add filters for state on brand, product and display campaigns | | 0.1.23 | 2022-09-06 | [16342](https://github.com/airbytehq/airbyte/pull/16342) | Add attribution reports | | 0.1.22 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | | 0.1.21 | 2022-09-27 | [17202](https://github.com/airbytehq/airbyte/pull/17202) | Improved handling if known reporting errors | From a4f795446267f722da3e050cc495bd8c3628e9e4 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 8 Nov 2022 21:41:08 +0000 Subject: [PATCH 7/7] auto-bump connector version --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 28 +++---------------- .../streams/report_streams/report_streams.py | 2 +- 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d1d24f7123326..7e0119a9a2ac9 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -47,7 +47,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 0.1.24 + dockerImageTag: 0.1.25 documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 91a3ff9033118..636fd3c1bf005 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -725,7 +725,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:0.1.24" +- dockerImage: "airbyte/source-amazon-ads:0.1.25" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads" connectionSpecification: @@ -770,26 +770,6 @@ type: "string" default: "NA" order: 4 - report_wait_timeout: - title: "Report Wait Timeout" - description: "Timeout duration in minutes for Reports. Default is 60 minutes." - default: 60 - examples: - - 60 - - 120 - order: 5 - type: "integer" - report_generation_max_retries: - title: "Report Generation Maximum Retries" - description: "Maximum retries Airbyte will attempt for fetching report data.\ - \ Default is 5." - default: 5 - examples: - - 5 - - 10 - - 15 - order: 6 - type: "integer" start_date: title: "Start Date" description: "The Start date for collecting reports, should not be more\ @@ -797,13 +777,13 @@ examples: - "2022-10-10" - "2022-10-22" - order: 7 + order: 5 type: "string" profiles: title: "Profile IDs" description: "Profile IDs you want to fetch data for. See docs for more details." - order: 8 + order: 6 type: "array" items: type: "integer" @@ -820,7 +800,7 @@ - "archived" type: "array" uniqueItems: true - order: 9 + order: 7 required: - "client_id" - "client_secret" diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index b315fd6ae4674..c6228d52d334e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -409,7 +409,7 @@ def _download_report(self, report_info: ReportInfo, url: str) -> List[dict]: def get_error_display_message(self, exception: BaseException) -> Optional[str]: if isinstance(exception, ReportGenerationInProgress): - return f'Report(s) generation time took more than {self.report_wait_timeout} minutes and failed because of Amazon API issues. Please wait some time and run synchronization again.' + return f"Report(s) generation time took more than {self.report_wait_timeout} minutes and failed because of Amazon API issues. Please wait some time and run synchronization again." return super().get_error_display_message(exception) def _get_response_error_details(self, response) -> Optional[str]: