Resolve discover catalog ref fields (#7734)

Author: Dmytro

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.1.33
+ Resolve $ref fields for discover json schema.
+
 ## 0.1.32
 - Added Sphinx docs `airbyte-cdk/python/reference_docs` module. 
 - Added module documents at `airbyte-cdk/python/sphinx-docs.md`.

airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py

@@ -75,36 +75,29 @@ def __call__(self, uri: str) -> Dict[str, Any]:
         package = importlib.import_module(self.package_name)
         base = os.path.dirname(package.__file__) + "/"
 
-        def create_definitions(obj: dict, definitions: dict) -> Dict[str, Any]:
+        def resolve_ref_links(obj: Any) -> Dict[str, Any]:
             """
-            Scan resolved schema and compose definitions section, also convert
-            jsonref.JsonRef object to JSON serializable dict.
+            Scan resolved schema and convert jsonref.JsonRef object to JSON
+            serializable dict.
 
             :param obj - jsonschema object with ref field resovled.
-            :definitions - object for storing generated definitions.
             :return JSON serializable object with references without external dependencies.
             """
             if isinstance(obj, jsonref.JsonRef):
-                def_key = obj.__reference__["$ref"]
-                def_key = def_key.replace("#/definitions/", "").replace(".json", "_")
-                definition = create_definitions(obj.__subject__, definitions)
+                obj = resolve_ref_links(obj.__subject__)
                 # Omit existance definitions for extenal resource since
                 # we dont need it anymore.
-                definition.pop("definitions", None)
-                definitions[def_key] = definition
-                return {"$ref": "#/definitions/" + def_key}
+                obj.pop("definitions", None)
+                return obj
             elif isinstance(obj, dict):
-                return {k: create_definitions(v, definitions) for k, v in obj.items()}
+                return {k: resolve_ref_links(v) for k, v in obj.items()}
             elif isinstance(obj, list):
-                return [create_definitions(item, definitions) for item in obj]
+                return [resolve_ref_links(item) for item in obj]
             else:
                 return obj
 
         resolved = jsonref.JsonRef.replace_refs(raw_schema, loader=JsonFileLoader(base, "schemas/shared"), base_uri=base)
-        definitions = {}
-        resolved = create_definitions(resolved, definitions)
-        if definitions:
-            resolved["definitions"] = definitions
+        resolved = resolve_ref_links(resolved)
         return resolved
 
 

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.1.32",
+    version="0.1.33",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py

@@ -96,9 +96,8 @@ def test_shared_schemas_resolves():
             "properties": {
                 "str": {"type": "string"},
                 "int": {"type": "integer"},
-                "obj": {"$ref": "#/definitions/shared_schema_"},
+                "obj": {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}},
             },
-            "definitions": {"shared_schema_": {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}}},
         }
 
         partial_schema = {
@@ -123,10 +122,9 @@ def test_shared_schemas_resolves_nested():
             "properties": {
                 "str": {"type": "string"},
                 "int": {"type": "integer"},
-                "one_of": {"oneOf": [{"type": "string"}, {"$ref": "#/definitions/shared_schema_type_one"}]},
-                "obj": {"$ref": "#/definitions/shared_schema_type_one"},
+                "one_of": {"oneOf": [{"type": "string"}, {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}}]},
+                "obj": {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}},
             },
-            "definitions": {"shared_schema_type_one": {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}}},
         }
         partial_schema = {
             "type": ["null", "object"],

airbyte-integrations/connectors/source-facebook-marketing/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.22
+LABEL io.airbyte.version=0.2.23
 LABEL io.airbyte.name=airbyte/source-facebook-marketing

airbyte-integrations/connectors/source-facebook-marketing/setup.py

@@ -6,7 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-cdk~=0.1.24",
+    "airbyte-cdk~=0.1.33",
     "cached_property~=1.5",
     "facebook_business~=12.0",
     "pendulum>=2,<3",

docs/integrations/sources/facebook-marketing.md

@@ -96,7 +96,8 @@ As a summary, custom insights allows to replicate only some fields, resulting in
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
-| 0.2.22  | 2021-11-05 | [4864](https://github.com/airbytehq/airbyte/pull/7605) | Add job retry logics to AdsInsights stream |
+| 0.2.23  | 2021-11-08 | [7734](https://github.com/airbytehq/airbyte/pull/7734) | Resolve $ref field for discover schema |
+| 0.2.22  | 2021-11-05 | [7605](https://github.com/airbytehq/airbyte/pull/7605) | Add job retry logics to AdsInsights stream |
 | 0.2.21  | 2021-10-05 | [4864](https://github.com/airbytehq/airbyte/pull/4864) | Update insights streams with custom entries for fields, breakdowns and action_breakdowns |
 | 0.2.20 | 2021-10-04 | [6719](https://github.com/airbytehq/airbyte/pull/6719) | Update version of facebook\_bussiness package to 12.0 |
 | 0.2.19 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification |