🎉 Source TikTok Marketing: support oauth flow

airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile

@@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.3
+LABEL io.airbyte.version=0.1.4
 LABEL io.airbyte.name=airbyte/source-tiktok-marketing

airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml

@@ -10,17 +10,33 @@ tests:
       status: "succeed"
     - config_path: "secrets/config.json"
       status: "succeed"
+    - config_path: "secrets/new_config_prod.json"
+      status: "succeed"
+    - config_path: "secrets/new_config_sandbox.json"
+      status: "succeed"
+    - config_path: "secrets/config_oauth.json"
+      status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
+    - config_path: "integration_tests/invalid_config_access_token.json"
+      status: "failed"
+    - config_path: "integration_tests/invalid_config_oauth.json"
+      status: "failed"
   discovery:
     - config_path: "secrets/config.json"
     - config_path: "secrets/prod_config.json"
+    - config_path: "secrets/new_config_prod.json"
+    - config_path: "secrets/config_oauth.json"
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
       empty_streams: [ "ads" , "ads_reports", "ad_groups_reports", "campaigns_reports" ]
     - config_path: "secrets/prod_config.json"
       configured_catalog_path: "integration_tests/configured_prod_catalog.json"
+    - config_path: "secrets/new_config_prod.json"
+      configured_catalog_path: "integration_tests/configured_prod_catalog.json"
+    - config_path: "secrets/config_oauth.json"
+      configured_catalog_path: "integration_tests/configured_prod_catalog.json"
   incremental:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"

airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-docker.sh

@@ -13,4 +13,3 @@ docker run --rm -it \
     -v $(pwd):/test_input \
     airbyte/source-acceptance-test:latest \
     --acceptance-test-config /test_input
-

airbyte-integrations/connectors/source-tiktok-marketing/bootstrap.md

@@ -11,5 +11,3 @@ The basic entity is 'advertiser'. All other streams use this required parameter
 
 Dependent streams have required parameter advertiser_id.
 As cursor field this connector uses "modify_time" values. But endpoints don't provide any mechanism for correct data filtering and sorting thus for incremental sync this connector tries to load all data and to validate a cursor field value on own side.
-
-

airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/invalid_config_access_token.json

@@ -0,0 +1,8 @@
+{
+  "credentials": {
+    "auth_type": "sandbox_access_token",
+    "access_token": "<broken_token>",
+    "start_date": "2021-01-01",
+    "advertiser_id": "11111111"
+  }
+}

airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/invalid_config_oauth.json

@@ -0,0 +1,8 @@
+{
+  "credentials": {
+    "auth_type": "oauth2.0",
+    "app_id": "1",
+    "secret": "2",
+    "access_token": "<broken_token>"
+  }
+}

airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json

@@ -5,22 +5,73 @@
     "title": "TikTok Marketing Source Spec",
     "type": "object",
     "properties": {
-      "environment": {
-        "title": "Environment",
-        "default": "Production",
-        "order": 2,
+      "start_date": {
+        "title": "Start Date",
+        "description": "The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated. If this parameter is not set, all data will be replicated.",
+        "default": "2016-09-01",
+        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$",
+        "order": 0,
+        "type": "string"
+      },
+      "report_granularity": {
+        "title": "Report Granularity",
+        "description": "Which time granularity should be grouped by; for LIFETIME there will be no grouping. This option is used for reports' streams only.",
+        "default": "DAY",
+        "enum": ["LIFETIME", "DAY", "HOUR"],
+        "order": 1,
+        "type": "string"
+      },
+      "credentials": {
+        "title": "Authorization Method",
+        "default": {},
+        "order": 3,
+        "type": "object",
         "oneOf": [
           {
-            "title": "Production",
+            "title": "OAuth2.0",
             "type": "object",
             "properties": {
-              "environment": {
-                "title": "Environment",
-                "const": "prod",
+              "auth_type": {
+                "title": "Auth Type",
+                "const": "oauth2.0",
+                "order": 0,
+                "enum": ["oauth2.0"],
                 "type": "string"
               },
               "app_id": {
-                "title": "App Id",
+                "title": "App ID",
+                "description": "The App ID applied by the developer.",
+                "airbyte_secret": true,
+                "type": "string"
+              },
+              "secret": {
+                "title": "Secret",
+                "description": "The private key of the developer's application.",
+                "airbyte_secret": true,
+                "type": "string"
+              },
+              "access_token": {
+                "title": "Access Token",
+                "description": "Long-term Authorized Access Token.",
+                "airbyte_secret": true,
+                "type": "string"
+              }
+            },
+            "required": ["app_id", "secret", "access_token"]
+          },
+          {
+            "title": "Production Access Token",
+            "type": "object",
+            "properties": {
+              "auth_type": {
+                "title": "Auth Type",
+                "const": "prod_access_token",
+                "order": 0,
+                "enum": ["prod_access_token"],
+                "type": "string"
+              },
+              "app_id": {
+                "title": "App ID",
                 "description": "The App ID applied by the developer.",
                 "type": "string"
               },
@@ -29,56 +80,97 @@
                 "description": "The private key of the developer application.",
                 "airbyte_secret": true,
                 "type": "string"
+              },
+              "access_token": {
+                "title": "Access Token",
+                "description": "The Long-term Authorized Access Token.",
+                "airbyte_secret": true,
+                "type": "string"
               }
             },
-            "required": ["app_id", "secret"]
+            "required": ["app_id", "secret", "access_token"]
           },
           {
-            "title": "Sandbox",
+            "title": "Sandbox Access Token",
             "type": "object",
             "properties": {
-              "environment": {
-                "title": "Environment",
-                "const": "sandbox",
+              "auth_type": {
+                "title": "Auth Type",
+                "const": "sandbox_access_token",
+                "order": 0,
+                "enum": ["sandbox_access_token"],
                 "type": "string"
               },
               "advertiser_id": {
-                "title": "Advertiser Id",
+                "title": "Advertiser ID",
                 "description": "The Advertiser ID  which generated for the developer's Sandbox application.",
                 "type": "string"
+              },
+              "access_token": {
+                "title": "Access Token",
+                "description": "The Long-term Authorized Access Token.",
+                "airbyte_secret": true,
+                "type": "string"
               }
             },
-            "required": ["advertiser_id"]
+            "required": ["advertiser_id", "access_token"]
           }
-        ],
-        "type": "object"
-      },
-      "access_token": {
-        "title": "Access Token",
-        "description": "The Long-term Authorized Access Token.",
-        "order": 1,
-        "airbyte_secret": true,
-        "type": "string"
+        ]
+      }
+    }
+  },
+  "supportsIncremental": true,
+  "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"],
+  "advanced_auth": {
+    "auth_flow_type": "oauth2.0",
+    "predicate_key": ["credentials", "auth_type"],
+    "predicate_value": "oauth2.0",
+    "oauth_config_specification": {
+      "complete_oauth_output_specification": {
+        "title": "CompleteOauthOutputSpecification",
+        "type": "object",
+        "properties": {
+          "access_token": {
+            "title": "Access Token",
+            "path_in_connector_config": ["credentials", "access_token"],
+            "type": "string"
+          }
+        },
+        "required": ["access_token"]
       },
-      "start_date": {
-        "title": "Start Date",
-        "description": "The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated. If this parameter is not set, all data will be replicated.",
-        "default": "2016-09-01",
-        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$",
-        "order": 3,
-        "type": "string"
+      "complete_oauth_server_input_specification": {
+        "title": "CompleteOauthServerInputSpecification",
+        "type": "object",
+        "properties": {
+          "app_id": {
+            "title": "App Id",
+            "type": "string"
+          },
+          "secret": {
+            "title": "Secret",
+            "type": "string"
+          }
+        },
+        "required": ["app_id", "secret"]
       },
-      "report_granularity": {
-        "title": "Report Granularity",
-        "description": "Which time granularity should be grouped by; for LIFETIME there will be no grouping. This option is used for reports' streams only.",
-        "default": "DAY",
-        "enum": ["LIFETIME", "DAY", "HOUR"],
-        "order": 4,
-        "type": "string"
+      "complete_oauth_server_output_specification": {
+        "title": "CompleteOauthServerOutputSpecification",
+        "type": "object",
+        "properties": {
+          "app_id": {
+            "title": "App Id",
+            "path_in_connector_config": ["credentials", "app_id"],
+            "type": "string"
+          },
+          "secret": {
+            "title": "Secret",
+            "path_in_connector_config": ["credentials", "secret"],
+            "type": "string"
+          }
+        },
+        "required": ["app_id", "secret"]
       }
-    },
-    "required": ["access_token"]
+    }
   },
-  "supportsIncremental": true,
-  "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"]
+  "additionalProperties": true
 }

airbyte-integrations/connectors/source-tiktok-marketing/setup.py

@@ -5,9 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = [
-    "airbyte-cdk~=0.1.42",
-]
+MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.42"]
 
 TEST_REQUIREMENTS = ["pytest~=6.1", "source-acceptance-test", "requests-mock==1.9.3", "timeout-decorator==0.5.0"]
 
@@ -19,7 +17,5 @@
     packages=find_packages(),
     install_requires=MAIN_REQUIREMENTS,
     package_data={"": ["*.json"]},
-    extras_require={
-        "tests": TEST_REQUIREMENTS,
-    },
+    extras_require={"tests": TEST_REQUIREMENTS},
 )

airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py

@@ -5,25 +5,21 @@
 from typing import Any, List, Mapping, Tuple
 
 from airbyte_cdk.logger import AirbyteLogger
-from airbyte_cdk.models import ConnectorSpecification, SyncMode
+from airbyte_cdk.models import (AdvancedAuth, AuthFlowType,
+                                ConnectorSpecification,
+                                OAuthConfigSpecification, SyncMode)
 from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
 
-from .spec import SourceTiktokMarketingSpec
-from .streams import (
-    DEFAULT_START_DATE,
-    AdGroups,
-    AdGroupsReports,
-    Ads,
-    AdsReports,
-    Advertisers,
-    AdvertisersReports,
-    Campaigns,
-    CampaignsReports,
-    ReportGranularity,
-)
+from .spec import (CompleteOauthOutputSpecification,
+                   CompleteOauthServerInputSpecification,
+                   CompleteOauthServerOutputSpecification,
+                   SourceTiktokMarketingSpec)
+from .streams import (DEFAULT_START_DATE, AdGroups, AdGroupsReports, Ads,
+                      AdsReports, Advertisers, AdvertisersReports, Campaigns,
+                      CampaignsReports, ReportGranularity)
 
 DOCUMENTATION_URL = "https://docs.airbyte.io/integrations/sources/tiktok-marketing"
 
@@ -50,17 +46,41 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification:
             supportsIncremental=True,
             supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append, DestinationSyncMode.append_dedup],
             connectionSpecification=SourceTiktokMarketingSpec.schema(),
+            additionalProperties=True,
+            advanced_auth=AdvancedAuth(
+                auth_flow_type=AuthFlowType.oauth2_0,
+                predicate_key=["credentials", "auth_type"],
+                predicate_value="oauth2.0",
+                oauth_config_specification=OAuthConfigSpecification(
+                    complete_oauth_output_specification=CompleteOauthOutputSpecification.schema(),
+                    complete_oauth_server_input_specification=CompleteOauthServerInputSpecification.schema(),
+                    complete_oauth_server_output_specification=CompleteOauthServerOutputSpecification.schema(),
+                ),
+            ),
         )
 
     @staticmethod
     def _prepare_stream_args(config: Mapping[str, Any]) -> Mapping[str, Any]:
         """Converts an input configure to stream arguments"""
+        credentials = config.get("credentials")
+        if credentials:
+            # used for new config format
+            access_token = credentials["access_token"]
+            secret = credentials.get("secret")
+            app_id = int(credentials.get("app_id", 0))
+            advertiser_id = int(credentials.get("advertiser_id", 0))
+        else:
+            access_token = config["access_token"]
+            secret = config.get("environment", {}).get("secret")
+            app_id = int(config.get("environment", {}).get("app_id", 0))
+            advertiser_id = int(config.get("environment", {}).get("advertiser_id", 0))
+
         return {
-            "authenticator": TiktokTokenAuthenticator(config["access_token"]),
+            "authenticator": TiktokTokenAuthenticator(access_token),
             "start_date": config.get("start_date") or DEFAULT_START_DATE,
-            "advertiser_id": int(config["environment"].get("advertiser_id", 0)),
-            "app_id": int(config["environment"].get("app_id", 0)),
-            "secret": config["environment"].get("secret"),
+            "advertiser_id": advertiser_id,
+            "app_id": app_id,
+            "secret": secret,
         }
 
     def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: