[low-code connectors] Get parent stream's full slice (#15631)

* always access parent stream using full_refresh mode * Update test * fix substream slicer * bump
airbytehq · Aug 18, 2022 · 313ac11 · 313ac11
1 parent 81eac50
commit 313ac11
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 15 deletions.
diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.1.76
+- Bugfix: Correctly set parent slice stream for sub-resource streams
+
 ## 0.1.75
 - Improve `filter_secrets` skip empty secret
 

diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/substream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/substream_slicer.py
@@ -127,7 +127,7 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Itera
                 stream_state_field = parent_stream_config.stream_slice_field
                 for parent_stream_slice in parent_stream.stream_slices(sync_mode=sync_mode, cursor_field=None, stream_state=stream_state):
                     empty_parent_slice = True
-                    parent_slice = parent_stream_slice.get("slice")
+                    parent_slice = parent_stream_slice
 
                     for parent_record in parent_stream.read_records(
                         sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice=parent_stream_slice, stream_state=None

diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py
@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.1.75",
+    version="0.1.76",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_substream_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_substream_slicer.py
@@ -47,6 +47,8 @@ def read_records(
         stream_slice: Mapping[str, Any] = None,
         stream_state: Mapping[str, Any] = None,
     ) -> Iterable[Mapping[str, Any]]:
+        # The parent stream's records should always be read as full refresh
+        assert sync_mode == SyncMode.full_refresh
         if not stream_slice:
             yield from self._records
         else:
@@ -64,7 +66,7 @@ def read_records(
                     stream=MockStream([{}], [], "first_stream"), parent_key="id", stream_slice_field="first_stream_id", options={}
                 )
             ],
-            [{"first_stream_id": None, "parent_slice": None}],
+            [{"first_stream_id": None, "parent_slice": {}}],
         ),
         (
             "test_single_parent_slices_with_records",
@@ -76,7 +78,7 @@ def read_records(
                     options={},
                 )
             ],
-            [{"first_stream_id": 1, "parent_slice": None}, {"first_stream_id": 2, "parent_slice": None}],
+            [{"first_stream_id": 1, "parent_slice": {}}, {"first_stream_id": 2, "parent_slice": {}}],
         ),
         (
             "test_with_parent_slices_and_records",
@@ -89,10 +91,10 @@ def read_records(
                 )
             ],
             [
-                {"parent_slice": "first", "first_stream_id": 0},
-                {"parent_slice": "first", "first_stream_id": 1},
-                {"parent_slice": "second", "first_stream_id": 2},
-                {"parent_slice": "third", "first_stream_id": None},
+                {"parent_slice": {"slice": "first"}, "first_stream_id": 0},
+                {"parent_slice": {"slice": "first"}, "first_stream_id": 1},
+                {"parent_slice": {"slice": "second"}, "first_stream_id": 2},
+                {"parent_slice": {"slice": "third"}, "first_stream_id": None},
             ],
         ),
         (
@@ -112,12 +114,12 @@ def read_records(
                 ),
             ],
             [
-                {"parent_slice": "first", "first_stream_id": 0},
-                {"parent_slice": "first", "first_stream_id": 1},
-                {"parent_slice": "second", "first_stream_id": 2},
-                {"parent_slice": "third", "first_stream_id": None},
-                {"parent_slice": "second_parent", "second_stream_id": 10},
-                {"parent_slice": "second_parent", "second_stream_id": 20},
+                {"parent_slice": {"slice": "first"}, "first_stream_id": 0},
+                {"parent_slice": {"slice": "first"}, "first_stream_id": 1},
+                {"parent_slice": {"slice": "second"}, "first_stream_id": 2},
+                {"parent_slice": {"slice": "third"}, "first_stream_id": None},
+                {"parent_slice": {"slice": "second_parent"}, "second_stream_id": 10},
+                {"parent_slice": {"slice": "second_parent"}, "second_stream_id": 20},
             ],
         ),
     ],

diff --git a/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs b/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs
@@ -6,7 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-cdk~=0.1.73",
+    "airbyte-cdk~=0.1.76",
 ]
 
 TEST_REQUIREMENTS = [