🎉 🎉 Source FB Marketing: performance and reliability fixes (#9805)

* Facebook Marketing performance improvement

* add comments and little refactoring

* fix integration tests with the new config

* improve job status handling, limit concurrency to 10

* fix campaign jobs, refactor manager

* big refactoring of async jobs, support random order of slices

* update source _read_incremental to hook new state logic

* fix issues with timeout

* remove debugging and clean up, improve retry logic

* merge changes from #8234

* fix call super _read_increment

* generalize batch execution, add use_batch flag

* improve coverage, do some refactoring of spec

* update test, remove overrides of source

* add split by AdSet

* add smaller insights

* fix end_date < start_date case

* add account_id to PK

* add notes

* fix new streams

* fix reversed incremental stream

* update spec.json for SAT

* upgrade CDK and bump version

Co-authored-by: Dmytro Rezchykov <dmitry.rezchykov@zazmic.com>
Co-authored-by: Eugene Kulak <kulak.eugene@gmail.com>

Author: 3 people

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json

@@ -196,7 +196,7 @@
 - name: Facebook Marketing
   sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
   dockerRepository: airbyte/source-facebook-marketing
-  dockerImageTag: 0.2.33
+  dockerImageTag: 0.2.34
   documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing
   icon: facebook.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -1630,7 +1630,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-facebook-marketing:0.2.33"
+- dockerImage: "airbyte/source-facebook-marketing:0.2.34"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.33
+LABEL io.airbyte.version=0.2.34
 LABEL io.airbyte.name=airbyte/source-facebook-marketing

airbyte-integrations/connectors/source-facebook-marketing/Dockerfile

@@ -39,7 +39,7 @@ From the Airbyte repository root, run:
 **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-marketing)
 to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_marketing/spec.json` file.
 Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
-See `sample_files/sample_config.json` for a sample config file.
+See `integration_tests/sample_config.json` for a sample config file.
 
 **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source facebook-marketing test creds`
 and place them into `secrets/config.json`.
@@ -73,7 +73,7 @@ Then run any of the connector commands as follows:
 docker run --rm airbyte/source-facebook-marketing:dev spec
 docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-marketing:dev check --config /secrets/config.json
 docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-marketing:dev discover --config /secrets/config.json
-docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-facebook-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
+docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-facebook-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
 ```
 ## Testing
    Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.

airbyte-integrations/connectors/source-facebook-marketing/README.md

@@ -13,23 +13,10 @@ tests:
     - config_path: "secrets/config.json"
   basic_read:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      timeout_seconds: 600
+      empty_streams: ["videos"]
   incremental:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog_without_insights.json"
       future_state_path: "integration_tests/future_state.json"
   full_refresh:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      # Ad Insights API has estimated metrics in response, which is calculated based on another metrics.
-      # Sometimes API doesn't return estimated metrics. E.g, cost_per_estimated_ad_recallers is calculated
-      # as total amount spent divided by estimated ad recall lift rate. When second metric is equal to zero
-      # API may or may not return value. Such behavior causes sequential reads test failing.
-      # Because one read response contains this metric, and other doesn't.
-      # Therefore, it's needed to ignore fields like this in API responses.
-      ignored_fields:
-        "ads_insights_age_and_gender": ["cost_per_estimated_ad_recallers"]
-        "ad_creatives": ["thumbnail_url"]
-        "ad_account": ["age"]
-        "images": ["permalink_url"]

airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml

@@ -34,61 +34,29 @@
       "stream": {
         "name": "ads_insights",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
         "default_cursor_field": ["date_start"],
-        "source_defined_primary_key": null,
-        "namespace": null
-      },
-      "sync_mode": "incremental",
-      "cursor_field": ["date_start"],
-      "destination_sync_mode": "append",
-      "primary_key": null
-    },
-    {
-      "stream": {
-        "name": "ads_insights_age_and_gender",
-        "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["date_start"],
-        "source_defined_primary_key": null,
-        "namespace": null
+        "source_defined_primary_key": []
       },
       "sync_mode": "incremental",
+      "primary_key": [],
       "cursor_field": ["date_start"],
-      "destination_sync_mode": "append",
-      "primary_key": null
+      "destination_sync_mode": "append"
     },
     {
       "stream": {
-        "name": "images",
+        "name": "ads_insights_platform_and_device",
         "json_schema": {},
+        "default_cursor_field": ["date_start"],
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["updated_time"],
-        "source_defined_primary_key": [["id"]],
-        "namespace": null
+        "source_defined_primary_key": []
       },
       "sync_mode": "incremental",
-      "cursor_field": null,
-      "destination_sync_mode": "append",
-      "primary_key": null
-    },
-    {
-      "stream": {
-        "name": "ad_account",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
-        "source_defined_cursor": null,
-        "default_cursor_field": null,
-        "source_defined_primary_key": [["id"]],
-        "namespace": null
-      },
-      "sync_mode": "full_refresh",
-      "cursor_field": null,
-      "destination_sync_mode": "append",
-      "primary_key": null
+      "primary_key": [],
+      "cursor_field": ["date_start"],
+      "destination_sync_mode": "append"
     }
   ]
 }

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/configured_catalog.json

@@ -7,8 +7,7 @@
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
         "default_cursor_field": ["updated_time"],
-        "source_defined_primary_key": [["id"]],
-        "namespace": null
+        "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "cursor_field": null,
@@ -22,13 +21,11 @@
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
         "default_cursor_field": ["updated_time"],
-        "source_defined_primary_key": [["id"]],
-        "namespace": null
+        "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "cursor_field": null,
-      "destination_sync_mode": "append",
-      "primary_key": null
+      "destination_sync_mode": "append"
     },
     {
       "stream": {
@@ -37,8 +34,7 @@
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
         "default_cursor_field": ["updated_time"],
-        "source_defined_primary_key": [["id"]],
-        "namespace": null
+        "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "cursor_field": null,

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/configured_catalog_without_insights.json

@@ -26,4 +26,7 @@ def config_with_wrong_account_fixture(config):
 
 @pytest.fixture(scope="session", name="config_with_include_deleted")
 def config_with_include_deleted_fixture(config):
-    return {**config, "include_deleted": True}
+    new_config = {**config, "include_deleted": True}
+    new_config.pop("_limit", None)
+    new_config.pop("end_date", None)
+    return new_config