From 2a37e25c7036a8ce78dd2112564c0950ece7a464 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 24 Nov 2021 14:14:34 +0200 Subject: [PATCH 01/13] first changes --- .../source_paypal_transaction/source.py | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 766be4255c8eb..0e78cae6f55d3 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -6,13 +6,13 @@ import time from abc import ABC from datetime import datetime, timedelta -from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union +from typing import Any, Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, Oauth2Authenticator +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator, Oauth2Authenticator from dateutil.parser import isoparse @@ -52,12 +52,15 @@ class PaypalTransactionStream(HttpStream, ABC): def __init__( self, - authenticator: HttpAuthenticator, + authenticator: Union[Oauth2Authenticator, TokenAuthenticator], start_date: Union[datetime, str], end_date: Union[datetime, str] = None, is_sandbox: bool = False, **kwargs, ): + super().__init__(authenticator=authenticator) + self._authenticator = authenticator + now = datetime.now().replace(microsecond=0).astimezone() if end_date and isinstance(end_date, str): @@ -89,7 +92,6 @@ def __init__( self.is_sandbox = is_sandbox - super().__init__(authenticator=authenticator) def validate_input_dates(self): # Validate input dates @@ -334,6 +336,14 @@ def refresh_access_token(self) -> Tuple[str, int]: except Exception as e: raise Exception(f"Error while refreshing access token: {e}") from e +class PayPalAuthenticator: + + def __init__(self, config: Dict): + self.config = config + + def get_auth(self): + access_token = self.config.get("access_token") + return TokenAuthenticator(token = access_token) class SourcePaypalTransaction(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: @@ -342,12 +352,13 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - authenticator = PayPalOauth2Authenticator(config) + #authenticator = PayPalOauth2Authenticator(config) + authenticator = PayPalAuthenticator(config).get_auth() # Try to get API TOKEN - token = authenticator.get_access_token() - if not token: - return False, "Unable to fetch Paypal API token due to incorrect client_id or secret" + #token = authenticator.get_access_token() + #if not token: + # return False, "Unable to fetch Paypal API token due to incorrect client_id or secret" # Try to initiate a stream and validate input date params try: @@ -361,8 +372,9 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ :param config: A Mapping of the user input configuration as defined in the connector spec. """ - authenticator = PayPalOauth2Authenticator(config) - + # authenticator = PayPalOauth2Authenticator(config) + authenticator = PayPalAuthenticator(config).get_auth() + return [ Transactions(authenticator=authenticator, **config), Balances(authenticator=authenticator, **config), From a21d0f41d4ead963078cb277a9c995eeea4f8905 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 24 Nov 2021 17:07:38 +0200 Subject: [PATCH 02/13] removed INSIGHTS_DAYS_PER_JOB from spec --- .../connectors/source-facebook-marketing/README.md | 6 +++--- .../source_facebook_marketing/source.py | 11 ++++------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/README.md b/airbyte-integrations/connectors/source-facebook-marketing/README.md index 263520e27078a..324803b2375f8 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/README.md +++ b/airbyte-integrations/connectors/source-facebook-marketing/README.md @@ -39,7 +39,7 @@ From the Airbyte repository root, run: **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-marketing) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_marketing/spec.json` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source facebook-marketing test creds` and place them into `secrets/config.json`. @@ -49,7 +49,7 @@ and place them into `secrets/config.json`. python main.py spec python main.py check --config secrets/config.json python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` ### Locally running the connector docker image @@ -73,7 +73,7 @@ Then run any of the connector commands as follows: docker run --rm airbyte/source-facebook-marketing:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-marketing:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-marketing:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-facebook-marketing:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-facebook-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 0150bd0ec475f..76f0bffd5f786 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -40,6 +40,9 @@ logger = logging.getLogger("airbyte") +# deprecated from ConnectorSpec, using default value of 1 = 1 day. +INSIGHTS_DAYS_PER_JOB: int = 1 + class InsightConfig(BaseModel): @@ -85,12 +88,6 @@ class Config: maximum=28, ) - insights_days_per_job: int = Field( - default=7, - description="Number of days to sync in one job. The more data you have - the smaller you want this parameter to be.", - minimum=1, - maximum=30, - ) custom_insights: Optional[List[InsightConfig]] = Field( description="A list wich contains insights entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns)" ) @@ -130,7 +127,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: start_date=config.start_date, end_date=config.end_date, buffer_days=config.insights_lookback_window, - days_per_job=config.insights_days_per_job, + days_per_job=INSIGHTS_DAYS_PER_JOB, ) streams = [ From b3359181496394feaf88bbcdaa202653ab05fb68 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 24 Nov 2021 17:57:06 +0200 Subject: [PATCH 03/13] fixed spec.json for invalid_spec --- .../source-facebook-marketing/integration_tests/spec.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 2c6b7bbcd2058..1c946dd632c14 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -46,14 +46,6 @@ "maximum": 28, "type": "integer" }, - "insights_days_per_job": { - "title": "Insights Days Per Job", - "description": "Number of days to sync in one job. The more data you have - the smaller you want this parameter to be.", - "default": 7, - "minimum": 1, - "maximum": 30, - "type": "integer" - }, "custom_insights": { "title": "Custom Insights", "description": "A list wich contains insights entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns)", From bfd6b2bdebcdb72b886f02e7decf9e469a53283c Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 24 Nov 2021 17:57:53 +0200 Subject: [PATCH 04/13] fixed SAT tests --- .../source-facebook-marketing/acceptance-test-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml index d426101a5ef5f..611f51e9aaeae 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml @@ -14,7 +14,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 600 + timeout_seconds: 3600 incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_without_insights.json" From 688633ec951f2fbd8911617603fb57cc47224690 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 24 Nov 2021 17:59:42 +0200 Subject: [PATCH 05/13] increased timeout_seconds in SAT --- .../source-facebook-marketing/acceptance-test-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml index 611f51e9aaeae..c544061ab9a7f 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml @@ -22,6 +22,7 @@ tests: full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 3600 # Ad Insights API has estimated metrics in response, which is calculated based on another metrics. # Sometimes API doesn't return estimated metrics. E.g, cost_per_estimated_ad_recallers is calculated # as total amount spent divided by estimated ad recall lift rate. When second metric is equal to zero From 39eac3060efac3638b1a84dfe986f2300ea8ba32 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 25 Nov 2021 12:18:07 +0200 Subject: [PATCH 06/13] reverted paypal files --- .../source_paypal_transaction/source.py | 32 ++++++------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 0e78cae6f55d3..766be4255c8eb 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -6,13 +6,13 @@ import time from abc import ABC from datetime import datetime, timedelta -from typing import Any, Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union +from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator, Oauth2Authenticator +from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, Oauth2Authenticator from dateutil.parser import isoparse @@ -52,15 +52,12 @@ class PaypalTransactionStream(HttpStream, ABC): def __init__( self, - authenticator: Union[Oauth2Authenticator, TokenAuthenticator], + authenticator: HttpAuthenticator, start_date: Union[datetime, str], end_date: Union[datetime, str] = None, is_sandbox: bool = False, **kwargs, ): - super().__init__(authenticator=authenticator) - self._authenticator = authenticator - now = datetime.now().replace(microsecond=0).astimezone() if end_date and isinstance(end_date, str): @@ -92,6 +89,7 @@ def __init__( self.is_sandbox = is_sandbox + super().__init__(authenticator=authenticator) def validate_input_dates(self): # Validate input dates @@ -336,14 +334,6 @@ def refresh_access_token(self) -> Tuple[str, int]: except Exception as e: raise Exception(f"Error while refreshing access token: {e}") from e -class PayPalAuthenticator: - - def __init__(self, config: Dict): - self.config = config - - def get_auth(self): - access_token = self.config.get("access_token") - return TokenAuthenticator(token = access_token) class SourcePaypalTransaction(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: @@ -352,13 +342,12 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - #authenticator = PayPalOauth2Authenticator(config) - authenticator = PayPalAuthenticator(config).get_auth() + authenticator = PayPalOauth2Authenticator(config) # Try to get API TOKEN - #token = authenticator.get_access_token() - #if not token: - # return False, "Unable to fetch Paypal API token due to incorrect client_id or secret" + token = authenticator.get_access_token() + if not token: + return False, "Unable to fetch Paypal API token due to incorrect client_id or secret" # Try to initiate a stream and validate input date params try: @@ -372,9 +361,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ :param config: A Mapping of the user input configuration as defined in the connector spec. """ - # authenticator = PayPalOauth2Authenticator(config) - authenticator = PayPalAuthenticator(config).get_auth() - + authenticator = PayPalOauth2Authenticator(config) + return [ Transactions(authenticator=authenticator, **config), Balances(authenticator=authenticator, **config), From e9b0862d70bebdf2971dfa1502774355891e3afb Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 25 Nov 2021 13:08:22 +0200 Subject: [PATCH 07/13] reordered spec fields --- .../integration_tests/spec.json | 43 +++++++++++-------- .../source_facebook_marketing/source.py | 39 +++++++++++------ 2 files changed, 51 insertions(+), 31 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 1c946dd632c14..4bea87d18327d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -5,24 +5,14 @@ "title": "Source Facebook Marketing", "type": "object", "properties": { - "account_id": { - "title": "Account Id", - "description": "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API.", - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "The value of the access token generated. See the docs for more information", - "airbyte_secret": true, - "type": "string" - }, "start_date": { "title": "Start Date", "description": "The date from which you'd like to replicate data for AdCreatives and AdInsights APIs, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], "type": "string", - "format": "date-time" + "format": "date-time", + "order": 0 }, "end_date": { "title": "End Date", @@ -30,13 +20,28 @@ "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-26T00:00:00Z"], "type": "string", - "format": "date-time" + "format": "date-time", + "order": 1 + }, + "account_id": { + "title": "Account Id", + "description": "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API.", + "type": "string", + "order": 2 + }, + "access_token": { + "title": "Access Token", + "description": "The value of the access token generated. See the docs for more information", + "airbyte_secret": true, + "type": "string", + "order": 3 }, "include_deleted": { "title": "Include Deleted", "description": "Include data from deleted campaigns, ads, and adsets.", "default": false, - "type": "boolean" + "type": "boolean", + "order": 4 }, "insights_lookback_window": { "title": "Insights Lookback Window", @@ -44,14 +49,15 @@ "default": 28, "minimum": 0, "maximum": 28, - "type": "integer" + "type": "integer", + "order": 5 }, "custom_insights": { "title": "Custom Insights", "description": "A list wich contains insights entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns)", "type": "array", "items": { - "title": "InsightConfig", + "title": "Insight Config", "type": "object", "properties": { "name": { @@ -88,13 +94,14 @@ } }, "required": ["name"] - } + }, + "order": 6 } }, "required": ["account_id", "access_token", "start_date"], "definitions": { "InsightConfig": { - "title": "InsightConfig", + "title": "Insight Config", "type": "object", "properties": { "name": { diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 76f0bffd5f786..86388d7ff0a4c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -41,6 +41,7 @@ logger = logging.getLogger("airbyte") # deprecated from ConnectorSpec, using default value of 1 = 1 day. +# Related Issue here: https://github.com/airbytehq/airbyte/issues/8027 INSIGHTS_DAYS_PER_JOB: int = 1 @@ -59,37 +60,49 @@ class ConnectorConfig(BaseModel): class Config: title = "Source Facebook Marketing" - account_id: str = Field(description="The Facebook Ad account ID to use when pulling data from the Facebook Marketing API.") - - access_token: str = Field( - description='The value of the access token generated. See the docs for more information', - airbyte_secret=True, - ) - start_date: datetime = Field( + title="Start Date", description="The date from which you'd like to replicate data for AdCreatives and AdInsights APIs, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", examples=["2017-01-25T00:00:00Z"], + order=0, ) end_date: Optional[datetime] = Field( + title="End Date (Optional)", description="The date until which you'd like to replicate data for AdCreatives and AdInsights APIs, in the format YYYY-MM-DDT00:00:00Z. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the latest data.", pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", examples=["2017-01-26T00:00:00Z"], default_factory=pendulum.now, + order=1, + ) + + account_id: str = Field( + title="Account ID", + description="The Facebook Ad account ID to use when pulling data from the Facebook Marketing API.", + examples=["111111111111111"], + order=2, ) - include_deleted: bool = Field(default=False, description="Include data from deleted campaigns, ads, and adsets.") + access_token: str = Field( + title="Access Token", + description='The value of the access token generated. See the docs for more information', + airbyte_secret=True, + order=3, + ) + + include_deleted: bool = Field( + title="Include Deleted", default=False, description="Include data from deleted campaigns, ads, and adsets.", order=4 + ) insights_lookback_window: int = Field( - default=28, - description="The attribution window for the actions", - minimum=0, - maximum=28, + title="Insights Lookback Window", default=28, description="The attribution window for the actions", minimum=0, maximum=28, order=5 ) custom_insights: Optional[List[InsightConfig]] = Field( - description="A list wich contains insights entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns)" + title="Custom Insights (Optional)", + description="A list wich contains insights entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns)", + order=6, ) From 97b9fd6588a9a288ebeea1ebdd8e818a240ca336 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 25 Nov 2021 13:30:46 +0200 Subject: [PATCH 08/13] bumbed version, added changelog --- .../e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../connectors/source-facebook-marketing/Dockerfile | 2 +- docs/integrations/sources/facebook-marketing.md | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json index bc4aedf95d141..92867ce4f1505 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "e7778cfc-e97c-4458-9ecb-b4f2bba8946c", "name": "Facebook Marketing", "dockerRepository": "airbyte/source-facebook-marketing", - "dockerImageTag": "0.2.26", + "dockerImageTag": "0.2.27", "documentationUrl": "https://docs.airbyte.io/integrations/sources/facebook-marketing", "icon": "facebook.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 55103c37ee524..ffb91b6dfab5d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -143,7 +143,7 @@ - name: Facebook Marketing sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c dockerRepository: airbyte/source-facebook-marketing - dockerImageTag: 0.2.26 + dockerImageTag: 0.2.27 documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing icon: facebook.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index 87c2dd28e8d99..fdbebf36d1421 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.26 +LABEL io.airbyte.version=0.2.27 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 285114e1868d4..c3428d5c7dc8f 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -96,6 +96,7 @@ As a summary, custom insights allows to replicate only some fields, resulting in | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.27 | 2021-11-25 | [8234](https://github.com/airbytehq/airbyte/pull/8234) | Reordered setup fields, deprecated `insigths days per job` from setup configuration | | 0.2.26 | 2021-11-19 | [7855](https://github.com/airbytehq/airbyte/pull/7855) | Add Video stream | | 0.2.25 | 2021-11-12 | [7904](https://github.com/airbytehq/airbyte/pull/7904) | Implement retry logic for async jobs | | 0.2.24 | 2021-11-09 | [7744](https://github.com/airbytehq/airbyte/pull/7744) | Fix fail when async job takes too long | From 293073509aee48ffa5c468ab86bacfdad65906ec Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 25 Nov 2021 14:14:17 +0200 Subject: [PATCH 09/13] edited spec validation --- .../integration_tests/spec.json | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 4bea87d18327d..6b9088f36a3c4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -10,38 +10,39 @@ "description": "The date from which you'd like to replicate data for AdCreatives and AdInsights APIs, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], + "order": 0, "type": "string", - "format": "date-time", - "order": 0 + "format": "date-time" }, "end_date": { - "title": "End Date", + "title": "End Date (Optional)", "description": "The date until which you'd like to replicate data for AdCreatives and AdInsights APIs, in the format YYYY-MM-DDT00:00:00Z. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the latest data.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-26T00:00:00Z"], + "order": 1, "type": "string", - "format": "date-time", - "order": 1 + "format": "date-time" }, "account_id": { - "title": "Account Id", + "title": "Account ID", "description": "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API.", - "type": "string", - "order": 2 + "examples": ["111111111111111"], + "order": 2, + "type": "string" }, "access_token": { "title": "Access Token", "description": "The value of the access token generated. See the docs for more information", "airbyte_secret": true, - "type": "string", - "order": 3 + "order": 3, + "type": "string" }, "include_deleted": { "title": "Include Deleted", "description": "Include data from deleted campaigns, ads, and adsets.", "default": false, - "type": "boolean", - "order": 4 + "order": 4, + "type": "boolean" }, "insights_lookback_window": { "title": "Insights Lookback Window", @@ -49,15 +50,16 @@ "default": 28, "minimum": 0, "maximum": 28, - "type": "integer", - "order": 5 + "order": 5, + "type": "integer" }, "custom_insights": { - "title": "Custom Insights", + "title": "Custom Insights (Optional)", "description": "A list wich contains insights entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns)", + "order": 6, "type": "array", "items": { - "title": "Insight Config", + "title": "InsightConfig", "type": "object", "properties": { "name": { @@ -94,14 +96,13 @@ } }, "required": ["name"] - }, - "order": 6 + } } }, - "required": ["account_id", "access_token", "start_date"], + "required": ["start_date", "account_id", "access_token"], "definitions": { "InsightConfig": { - "title": "Insight Config", + "title": "InsightConfig", "type": "object", "properties": { "name": { From 89589e061792bdb048de3d4c233c4fc94bf09a61 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Fri, 26 Nov 2021 11:09:16 +0200 Subject: [PATCH 10/13] corrected formating --- .../source_facebook_marketing/source.py | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 86388d7ff0a4c..8bcf2a2e2b11e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -47,13 +47,24 @@ class InsightConfig(BaseModel): - name: str = Field(description="The name value of insight") + name: str = Field( + description="The name value of insight", + ) - fields: Optional[List[str]] = Field(description="A list of chosen fields for fields parameter", default=[]) + fields: Optional[List[str]] = Field( + description="A list of chosen fields for fields parameter", + default=[], + ) - breakdowns: Optional[List[str]] = Field(description="A list of chosen breakdowns for breakdowns", default=[]) + breakdowns: Optional[List[str]] = Field( + description="A list of chosen breakdowns for breakdowns", + default=[], + ) - action_breakdowns: Optional[List[str]] = Field(description="A list of chosen action_breakdowns for action_breakdowns", default=[]) + action_breakdowns: Optional[List[str]] = Field( + description="A list of chosen action_breakdowns for action_breakdowns", + default=[], + ) class ConnectorConfig(BaseModel): @@ -92,11 +103,19 @@ class Config: ) include_deleted: bool = Field( - title="Include Deleted", default=False, description="Include data from deleted campaigns, ads, and adsets.", order=4 + title="Include Deleted", + default=False, + description="Include data from deleted campaigns, ads, and adsets.", + order=4, ) insights_lookback_window: int = Field( - title="Insights Lookback Window", default=28, description="The attribution window for the actions", minimum=0, maximum=28, order=5 + title="Insights Lookback Window", + default=28, + description="The attribution window for the actions", + minimum=0, + maximum=28, + order=5, ) custom_insights: Optional[List[InsightConfig]] = Field( From c87be7d7b3ecd23ae9854e3be20775be7a2514fb Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Fri, 26 Nov 2021 11:11:25 +0200 Subject: [PATCH 11/13] changed insights_days_per_job to 7 --- .../source-facebook-marketing/acceptance-test-config.yml | 4 ++-- .../source_facebook_marketing/source.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml index c544061ab9a7f..ee9f271797350 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml @@ -14,7 +14,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 + timeout_seconds: 1200 incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_without_insights.json" @@ -22,7 +22,7 @@ tests: full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 + timeout_seconds: 1200 # Ad Insights API has estimated metrics in response, which is calculated based on another metrics. # Sometimes API doesn't return estimated metrics. E.g, cost_per_estimated_ad_recallers is calculated # as total amount spent divided by estimated ad recall lift rate. When second metric is equal to zero diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 8bcf2a2e2b11e..8049cdec81447 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -42,7 +42,7 @@ # deprecated from ConnectorSpec, using default value of 1 = 1 day. # Related Issue here: https://github.com/airbytehq/airbyte/issues/8027 -INSIGHTS_DAYS_PER_JOB: int = 1 +INSIGHTS_DAYS_PER_JOB: int = 7 class InsightConfig(BaseModel): From 31415bf40550125fe7353c18cc95397be5bc6b7a Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 30 Nov 2021 15:19:39 +0200 Subject: [PATCH 12/13] updated default value of insigts_days_per_job --- .../source_facebook_marketing/source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 8049cdec81447..4299710a877b3 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -40,9 +40,9 @@ logger = logging.getLogger("airbyte") -# deprecated from ConnectorSpec, using default value of 1 = 1 day. +# deprecated from ConnectorSpec, using default value of 5 = 5 days. # Related Issue here: https://github.com/airbytehq/airbyte/issues/8027 -INSIGHTS_DAYS_PER_JOB: int = 7 +INSIGHTS_DAYS_PER_JOB: int = 5 class InsightConfig(BaseModel): From d363222bc8da72437069e39000a0c7ec9b6da6f0 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 30 Nov 2021 15:20:13 +0200 Subject: [PATCH 13/13] updated default value of insigts_days_per_job --- .../source_facebook_marketing/streams.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py index c0d236a80e702..36758cef66997 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py @@ -361,6 +361,7 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite yield {"job": running_jobs.popleft()} while running_jobs: + #print(f"\nTEST: 'job': {running_jobs.popleft()}\n") yield {"job": running_jobs.popleft()} @retry_pattern(backoff.expo, JobException, max_tries=10, factor=5)