🐛Source Titktok Marketing: Fix schema types (#20415)

* Updated schema types * Updated version * Updated acceptance test config to high strictness * Updated SAT to high strictness * skip comprehensive incremental tests * Fixed SAT * disable high strictness * updated unittest config * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Jan 26, 2023 · ec934ad · ec934ad
1 parent ca8151e
commit ec934ad
Show file tree

Hide file tree

Showing 22 changed files with 557 additions and 151 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -1742,7 +1742,7 @@
 - name: TikTok Marketing
   sourceDefinitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35
   dockerRepository: airbyte/source-tiktok-marketing
-  dockerImageTag: 1.0.1
+  dockerImageTag: 2.0.0
   documentationUrl: https://docs.airbyte.com/integrations/sources/tiktok-marketing
   icon: tiktok.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -14909,7 +14909,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-tiktok-marketing:1.0.1"
+- dockerImage: "airbyte/source-tiktok-marketing:2.0.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/tiktok-marketing"
     changelogUrl: "https://docs.airbyte.com/integrations/sources/tiktok-marketing"

diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile
@@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=1.0.1
+LABEL io.airbyte.version=2.0.0
 LABEL io.airbyte.name=airbyte/source-tiktok-marketing
diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/README.md b/airbyte-integrations/connectors/source-tiktok-marketing/README.md
@@ -79,7 +79,7 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat
    Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
 First install test dependencies into your virtual environment:
 ```
-pip install .[tests]
+pip install .'[tests]'
 ```
 ### Unit Tests
 To run unit tests locally, from the connector directory run:

diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml
@@ -3,97 +3,130 @@
 
 connector_image: airbyte/source-tiktok-marketing:dev
 cache_discovered_catalog: False
-tests:
+acceptance_tests:
   spec:
-    - spec_path: "integration_tests/spec.json"
-
+    tests:
+      - spec_path: "integration_tests/spec.json"
   connection:
-    - config_path: "secrets/prod_config.json"
-      status: "succeed"
-    - config_path: "secrets/prod_config_with_day_granularity.json"
-      status: "succeed"
-    - config_path: "secrets/prod_config_with_lifetime_granularity.json"
-      status: "succeed"
-    - config_path: "secrets/new_config_prod.json"
-      status: "succeed"
-    - config_path: "secrets/config_oauth.json"
-      status: "succeed"
-    - config_path: "integration_tests/invalid_config.json"
-      status: "failed"
-    - config_path: "integration_tests/invalid_config_access_token.json"
-      status: "failed"
-    - config_path: "integration_tests/invalid_config_oauth.json"
-      status: "failed"
-
+    tests:
+      - config_path: "secrets/prod_config.json"
+        status: "succeed"
+      - config_path: "secrets/prod_config_with_day_granularity.json"
+        status: "succeed"
+      - config_path: "secrets/prod_config_with_lifetime_granularity.json"
+        status: "succeed"
+      - config_path: "secrets/new_config_prod.json"
+        status: "succeed"
+      - config_path: "secrets/config_oauth.json"
+        status: "succeed"
+      - config_path: "integration_tests/invalid_config.json"
+        status: "failed"
+      - config_path: "integration_tests/invalid_config_access_token.json"
+        status: "failed"
+      - config_path: "integration_tests/invalid_config_oauth.json"
+        status: "failed"
   discovery:
-    - config_path: "secrets/prod_config.json"
-      backward_compatibility_tests_config:
-        disable_for_version: "1.0.0"
-    - config_path: "secrets/prod_config_with_day_granularity.json"
-      backward_compatibility_tests_config:
-        disable_for_version: "1.0.0"
-    - config_path: "secrets/prod_config_with_lifetime_granularity.json"
-      backward_compatibility_tests_config:
-        disable_for_version: "1.0.0"
-    - config_path: "secrets/config.json"
-      backward_compatibility_tests_config:
-        disable_for_version: "1.0.0"
-    - config_path: "secrets/new_config_prod.json"
-      backward_compatibility_tests_config:
-        disable_for_version: "1.0.0"
-    - config_path: "secrets/config_oauth.json"
-      backward_compatibility_tests_config:
-        disable_for_version: "1.0.0"
-
+    tests:
+      - config_path: "secrets/prod_config.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "1.0.1"
+      - config_path: "secrets/prod_config_with_day_granularity.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "1.0.1"
+      - config_path: "secrets/prod_config_with_lifetime_granularity.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "1.0.1"
+      - config_path: "secrets/config.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "1.0.1"
+      - config_path: "secrets/new_config_prod.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "1.0.1"
+      - config_path: "secrets/config_oauth.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "1.0.1"
   basic_read:
-    # New style streams (for >= 0.1.13):
-    - config_path: "secrets/prod_config.json"
-      configured_catalog_path: "integration_tests/streams_basic.json"
-      timeout_seconds: 1200
-    - config_path: "secrets/prod_config.json"
-      configured_catalog_path: "integration_tests/streams_reports_daily.json"
-      timeout_seconds: 1200
-    - config_path: "secrets/prod_config.json"
-      configured_catalog_path: "integration_tests/streams_reports_lifetime.json"
-      timeout_seconds: 1200
-    # Old style streams with granularity config (for < 0.1.13)
-    # Note: not needed to be tested separately in full and incremental tests, because code of
-    # these streams is called directly in new style streams
-    - config_path: "secrets/prod_config_with_day_granularity.json"
-      configured_catalog_path: "integration_tests/streams_with_day_granularity.json"
-      timeout_seconds: 1200
-    - config_path: "secrets/prod_config_with_lifetime_granularity.json"
-      configured_catalog_path: "integration_tests/streams_with_lifetime_granularity.json"
-      timeout_seconds: 1200
-
-  incremental:
-    - config_path: "secrets/prod_config.json"
-      configured_catalog_path: "integration_tests/streams_basic.json"
-      timeout_seconds: 7200
-      future_state_path: "integration_tests/abnormal_state.json"
-    - config_path: "secrets/prod_config.json"
-      configured_catalog_path: "integration_tests/streams_reports_daily.json"
-      timeout_seconds: 7200
-      future_state_path: "integration_tests/abnormal_state.json"
-    # LIFETIME granularity: does not support incremental sync
-
+    tests:
+      - config_path: "secrets/config.json"
+        timeout_seconds: 1200
+        empty_streams:
+          - name: ads_reports_hourly
+            bypass_reason: "Tested with daily granularity."
+          - name: ad_groups_reports_hourly
+            bypass_reason: "Tested with daily granularity."
+          - name: advertisers_reports_hourly
+            bypass_reason: "Tested with daily granularity."
+          - name: campaigns_reports_hourly
+            bypass_reason: "Tested with daily granularity."
+          - name: advertisers_reports
+            bypass_reason: "Stream was added during discover with granularity config. Will be tested below."
+          - name: advertisers_audience_reports
+            bypass_reason: "Stream was added during discover with granularity config. Will be tested below."
+          - name: campaigns_audience_reports_by_country
+            bypass_reason: "Stream was added during discover with granularity config. Will be tested below."
+          - name: ad_group_audience_reports
+            bypass_reason: "Stream was added during discover with granularity config. Will be tested below."
+          - name: ads_audience_reports
+            bypass_reason: "Stream was added during discover with granularity config. Will be tested below."
+          - name: ad_groups_reports
+            bypass_reason: "Stream was added during discover with granularity config. Will be tested below."
+          - name: ads_reports
+            bypass_reason: "Stream was added during discover with granularity config. Will be tested below."
+          - name: campaigns_reports
+            bypass_reason: "Stream was added during discover with granularity config. Will be tested below."
+      # Old style streams with granularity config (for < 0.1.13)
+      # Note: not needed to be tested separately in full and incremental tests, because code of
+      # these streams is called directly in new style streams
+      - config_path: "secrets/prod_config_with_day_granularity.json"
+        timeout_seconds: 1200
+        empty_streams:
+          - name: ads_reports_lifetime
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: ad_groups_reports_daily
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: advertisers_audience_reports_lifetime
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: campaigns_reports_daily
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: ads_reports_daily
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: advertisers_reports_daily
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: ad_groups_reports_lifetime
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: campaigns_reports_hourly
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: ads_audience_reports_daily
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: ads_audience_reports_daily
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: ad_group_audience_reports_daily
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: campaigns_reports_lifetime
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: ad_groups_reports_hourly
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: advertisers_reports_lifetime
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: campaigns_audience_reports_by_country_daily
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: advertisers_audience_reports_daily
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: advertisers_reports_hourly
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
+          - name: ads_reports_hourly
+            bypass_reason: "Stream was added during discover without granularity config. Was tested abowe."
   full_refresh:
-    - config_path: "secrets/prod_config.json"
-      configured_catalog_path: "integration_tests/streams_basic.json"
-      timeout_seconds: 7200
-      ignored_fields: # Important: sometimes some streams does not return the same records in subsequent syncs
-        "ad_groups":
-          [
-            "dayparting",
-            "enable_search_result",
-            "display_mode",
-            "schedule_infos",
-            "feed_type",
-            "status",
-          ]
-    - config_path: "secrets/prod_config.json"
-      configured_catalog_path: "integration_tests/streams_reports_daily.json"
-      timeout_seconds: 7200
-    - config_path: "secrets/prod_config.json"
-      configured_catalog_path: "integration_tests/streams_reports_lifetime.json"
-      timeout_seconds: 7200
+    tests:
+      - config_path: "secrets/prod_config_with_day_granularity.json"
+        configured_catalog_path: "integration_tests/configured_catalogs/configured_catalog.json"
+        timeout_seconds: 7200
+  incremental:
+    tests:
+      - config_path: "secrets/prod_config_with_day_granularity.json"
+        configured_catalog_path: "integration_tests/configured_catalogs/configured_catalog_incremental.json"
+        timeout_seconds: 7200
+        skip_comprehensive_incremental_tests: true
+        future_state:
+           future_state_path: "integration_tests/abnormal_state.json"
+      # LIFETIME granularity: does not support incremental sync
diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/bootstrap.md b/airbyte-integrations/connectors/source-tiktok-marketing/bootstrap.md
@@ -11,3 +11,50 @@ The basic entity is 'advertiser'. All other streams use this required parameter
 
 Dependent streams have required parameter advertiser_id.
 As cursor field this connector uses "modify_time" values. But endpoints don't provide any mechanism for correct data filtering and sorting thus for incremental sync this connector tries to load all data and to validate a cursor field value on own side.
+
+
+
+`stream` method has granularity condition depend on that report streams supports by different connector version:
+- For all version:
+  basic streams list: 
+     * ad_groups
+     * ads
+     * campaigns
+     * advertisers
+- for < 0.1.13 - expose report streams initialized with 'report_granularity' argument, like:
+  Example: 
+  - AdsReports(report_granularity='DAILY')
+  - AdsReports(report_granularity='LIFETIME')
+  streams list:
+    * advertisers_reports
+    * advertisers_audience_reports
+    * campaigns_audience_reports_by_country
+    * ad_group_audience_reports
+    * ads_audience_reports
+    * ad_groups_reports
+    * ads_reports
+    * campaigns_reports
+
+- for >= 0.1.13 - expose report streams in format: <report_type>_<granularity>, like:
+  Example: 
+  - AdsReportsDaily(Daily, AdsReports)
+  - AdsReportsLifetime(Lifetime, AdsReports)
+  streams:
+    * campaigns_audience_reports_by_country_daily
+    * campaigns_reports_daily 
+    * advertisers_audience_reports_daily
+    * advertisers_reports_daily
+    * ad_group_audience_reports_daily
+    * ads_reports_lifetime
+    * advertiser_ids
+    * campaigns_reports_lifetime
+    * advertisers_audience_reports_lifetime
+    * ad_groups_reports_lifetime
+    * ad_groups_reports_daily
+    * advertisers_reports_lifetime
+    * ads_reports_daily
+    * ads_audience_reports_daily
+    * ads_reports_hourly
+    * ad_groups_reports_hourly
+    * advertisers_reports_hourly
+    * campaigns_reports_hourly
diff --git a/...yte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json b/...yte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json
@@ -27,7 +27,7 @@
         "stat_time_day": "2030-01-01",
         "dimensions": { "stat_time_day": "2030-01-01" }
       },
-      "stream_descriptor": { "name": "ads_reports_daily" }
+      "stream_descriptor": { "name": "ads_reports" }
     }
   },
   {
@@ -37,7 +37,7 @@
         "stat_time_day": "2030-01-01",
         "dimensions": { "stat_time_day": "2030-01-01" }
       },
-      "stream_descriptor": { "name": "advertisers_reports_daily" }
+      "stream_descriptor": { "name": "advertisers_reports" }
     }
   },
   {
@@ -47,7 +47,7 @@
         "stat_time_day": "2030-01-01",
         "dimensions": { "stat_time_day": "2030-01-01" }
       },
-      "stream_descriptor": { "name": "ad_groups_reports_daily" }
+      "stream_descriptor": { "name": "ad_groups_reports" }
     }
   },
   {
@@ -57,7 +57,7 @@
         "stat_time_day": "2030-01-01",
         "dimensions": { "stat_time_day": "2030-01-01" }
       },
-      "stream_descriptor": { "name": "campaigns_reports_daily" }
+      "stream_descriptor": { "name": "campaigns_reports" }
     }
   },
   {
@@ -67,7 +67,7 @@
         "stat_time_day": "2030-01-01",
         "dimensions": { "stat_time_day": "2030-01-01" }
       },
-      "stream_descriptor": { "name": "ads_audience_reports_daily" }
+      "stream_descriptor": { "name": "ads_audience_reports" }
     }
   },
   {
@@ -77,7 +77,7 @@
         "stat_time_day": "2030-01-01",
         "dimensions": { "stat_time_day": "2030-01-01" }
       },
-      "stream_descriptor": { "name": "advertisers_audience_reports_daily" }
+      "stream_descriptor": { "name": "advertisers_audience_reports" }
     }
   },
   {
@@ -87,7 +87,7 @@
         "stat_time_day": "2030-01-01",
         "dimensions": { "stat_time_day": "2030-01-01" }
       },
-      "stream_descriptor": { "name": "ad_group_audience_reports_daily" }
+      "stream_descriptor": { "name": "ad_group_audience_reports" }
     }
   },
   {
@@ -98,7 +98,7 @@
         "dimensions": { "stat_time_day": "2030-01-01" }
       },
       "stream_descriptor": {
-        "name": "campaigns_audience_reports_by_country_daily"
+        "name": "campaigns_audience_reports_by_country"
       }
     }
   }