diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index bd780d432974a..4d37727352262 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.14 +LABEL io.airbyte.version=0.2.15 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/abnormal_state.json index 437161ace3dce..f1dfc2e605e05 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/abnormal_state.json @@ -1,42 +1,46 @@ { "campaigns": { - "updated_time": "2021-07-25T13:34:26Z", + "updated_time": "2121-07-25T13:34:26Z", "include_deleted": true }, "ad_creatives": { - "updated_time": "2021-07-25T13:34:26Z", + "updated_time": "2121-07-25T13:34:26Z", "include_deleted": true }, "ad_sets": { - "updated_time": "2021-07-25T13:34:26Z", + "updated_time": "2121-07-25T13:34:26Z", "include_deleted": true }, "ads": { - "updated_time": "2021-07-25T13:34:26Z", + "updated_time": "2121-07-25T13:34:26Z", "include_deleted": true }, "ads_insights": { - "date_start": "2021-07-25T13:34:26Z", + "date_start": "2121-07-25T13:34:26Z", "include_deleted": true }, "ads_insights_age_and_gender": { - "date_start": "2021-07-25T13:34:26Z", + "date_start": "2121-07-25T13:34:26Z", "include_deleted": true }, "ads_insights_country": { - "date_start": "2021-07-25T13:34:26Z", + "date_start": "2121-07-25T13:34:26Z", "include_deleted": true }, "ads_insights_dma": { - "date_start": "2021-07-25T13:34:26Z", + "date_start": "2121-07-25T13:34:26Z", "include_deleted": true }, "ads_insights_platfrom_and_device": { - "date_start": "2021-07-25T13:34:26Z", + "date_start": "2121-07-25T13:34:26Z", "include_deleted": true }, "ads_insights_region": { - "date_start": "2021-07-25T13:34:26Z", + "date_start": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "ads_insights_action_types": { + "date_start": "2121-07-25T13:34:26Z", "include_deleted": true } } diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index f04806dda92aa..7a9ed9829f238 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -35,6 +35,7 @@ Ads, AdSets, AdsInsights, + AdsInsightsActionType, AdsInsightsAgeAndGender, AdsInsightsCountry, AdsInsightsDma, @@ -124,6 +125,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: AdsInsightsRegion(**insights_args), AdsInsightsDma(**insights_args), AdsInsightsPlatformAndDevice(**insights_args), + AdsInsightsActionType(**insights_args), ] def spec(self, *args, **kwargs) -> ConnectorSpecification: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py index 1c375bb4f7bb5..5ea87e9564d50 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py @@ -46,16 +46,19 @@ backoff_policy = retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) -def remove_params_from_url(url, params): - parsed_url = urlparse.urlparse(url) - res_query = [] - for q in parsed_url.query.split("&"): - key, value = q.split("=") - if key not in params: - res_query.append(f"{key}={value}") - - parse_result = parsed_url._replace(query="&".join(res_query)) - return urlparse.urlunparse(parse_result) +def remove_params_from_url(url: str, params: [str]) -> str: + """ + Parses a URL and removes the query parameters specified in params + :param url: URL + :param params: list of query parameters + :return: URL with params removed + """ + parsed = urlparse.urlparse(url) + query = urlparse.parse_qs(parsed.query, keep_blank_values=True) + filtered = dict((k, v) for k, v in query.items() if k not in params) + return urlparse.urlunparse( + [parsed.scheme, parsed.netloc, parsed.path, parsed.params, urlparse.urlencode(filtered, doseq=True), parsed.fragment] + ) class FBMarketingStream(Stream, ABC): @@ -479,3 +482,8 @@ class AdsInsightsDma(AdsInsights): class AdsInsightsPlatformAndDevice(AdsInsights): breakdowns = ["publisher_platform", "platform_position", "impression_device"] action_breakdowns = ["action_type"] # FB Async Job fails for unknown reason if we set other breakdowns + + +class AdsInsightsActionType(AdsInsights): + breakdowns = [] + action_breakdowns = ["action_type"] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py new file mode 100644 index 0000000000000..b41e09faeeb78 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py @@ -0,0 +1,66 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +from source_facebook_marketing.streams import remove_params_from_url + + +class TestUrlParsing: + def test_empty_url(self): + url = "" + parsed_url = remove_params_from_url(url=url, params=[]) + assert parsed_url == url + + def test_does_not_raise_exception_for_invalid_url(self): + url = "abcd" + parsed_url = remove_params_from_url(url=url, params=["test"]) + assert parsed_url == url + + def test_escaped_characters(self): + url = "https://google.com?test=123%23%24%25%2A&test2=456" + parsed_url = remove_params_from_url(url=url, params=["test3"]) + assert parsed_url == url + + def test_no_params_url(self): + url = "https://google.com" + parsed_url = remove_params_from_url(url=url, params=["test"]) + assert parsed_url == url + + def test_no_params_arg(self): + url = "https://google.com?" + parsed_url = remove_params_from_url(url=url, params=["test"]) + assert parsed_url == "https://google.com" + + def test_partially_empty_params(self): + url = "https://google.com?test=122&&" + parsed_url = remove_params_from_url(url=url, params=[]) + assert parsed_url == "https://google.com?test=122" + + def test_no_matching_params(self): + url = "https://google.com?test=123" + parsed_url = remove_params_from_url(url=url, params=["test2"]) + assert parsed_url == url + + def test_removes_params(self): + url = "https://google.com?test=123&test2=456" + parsed_url = remove_params_from_url(url=url, params=["test2"]) + assert parsed_url == "https://google.com?test=123"