From c8f9a7af1678457abc911537236c836c4451cf44 Mon Sep 17 00:00:00 2001 From: maxi297 Date: Thu, 27 Feb 2025 16:57:50 -0800 Subject: [PATCH 1/3] Allow mock server tests for manifest only sources --- .../parsers/model_to_component_factory.py | 11 ++++++++++- .../declarative/yaml_declarative_source.py | 17 +++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index f6e1cc0d6..a74e1fd84 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -1405,7 +1405,16 @@ def _get_class_from_fully_qualified_class_name( try: module_ref = importlib.import_module(module_name_full) except ModuleNotFoundError as e: - raise ValueError(f"Could not load module `{module_name_full}`.") from e + if split[0] == "source_declarative_manifest": + # During testing, the modules containing the custom components are not moved to source_declarative_manifest. In order to run the test, add the source folder to your PYTHONPATH or add it runtime using sys.path.append + try: + import os + module_name_with_source_declarative_manifest = ".".join(split[1:-1]) + module_ref = importlib.import_module(module_name_with_source_declarative_manifest) + except ModuleNotFoundError: + raise ValueError(f"Could not load module `{module_name_full}`.") from e + else: + raise ValueError(f"Could not load module `{module_name_full}`.") from e try: return getattr(module_ref, class_name) diff --git a/airbyte_cdk/sources/declarative/yaml_declarative_source.py b/airbyte_cdk/sources/declarative/yaml_declarative_source.py index 04ccda4cf..282e6c0c1 100644 --- a/airbyte_cdk/sources/declarative/yaml_declarative_source.py +++ b/airbyte_cdk/sources/declarative/yaml_declarative_source.py @@ -39,13 +39,18 @@ def __init__( ) def _read_and_parse_yaml_file(self, path_to_yaml_file: str) -> ConnectionDefinition: - package = self.__class__.__module__.split(".")[0] + try: + # For testing purposes, we want to allow to just pass a file. However, this + with open(path_to_yaml_file, "r") as f: + return yaml.safe_load(f) # type: ignore # we assume the yaml represents a ConnectionDefinition + except FileNotFoundError: + # Running inside the container, the working directory during an operation is not structured the same as the static files + package = self.__class__.__module__.split(".")[0] - yaml_config = pkgutil.get_data(package, path_to_yaml_file) - if yaml_config: - decoded_yaml = yaml_config.decode() - return self._parse(decoded_yaml) - else: + yaml_config = pkgutil.get_data(package, path_to_yaml_file) + if yaml_config: + decoded_yaml = yaml_config.decode() + return self._parse(decoded_yaml) return {} def _emit_manifest_debug_message(self, extra_args: dict[str, Any]) -> None: From 3719e7c6af9ad7632dd2ede2fea08b98831b0b78 Mon Sep 17 00:00:00 2001 From: octavia-squidington-iii Date: Fri, 28 Feb 2025 01:07:16 +0000 Subject: [PATCH 2/3] Auto-fix lint and format issues --- .../declarative/parsers/model_to_component_factory.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index a74e1fd84..fbdf3e0de 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -1409,8 +1409,11 @@ def _get_class_from_fully_qualified_class_name( # During testing, the modules containing the custom components are not moved to source_declarative_manifest. In order to run the test, add the source folder to your PYTHONPATH or add it runtime using sys.path.append try: import os + module_name_with_source_declarative_manifest = ".".join(split[1:-1]) - module_ref = importlib.import_module(module_name_with_source_declarative_manifest) + module_ref = importlib.import_module( + module_name_with_source_declarative_manifest + ) except ModuleNotFoundError: raise ValueError(f"Could not load module `{module_name_full}`.") from e else: From 151edc3562a9e4c2e91af80a29bfd18b205f7bfe Mon Sep 17 00:00:00 2001 From: maxi297 Date: Thu, 27 Feb 2025 23:49:49 -0800 Subject: [PATCH 3/3] code review --- airbyte_cdk/sources/declarative/yaml_declarative_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/yaml_declarative_source.py b/airbyte_cdk/sources/declarative/yaml_declarative_source.py index 282e6c0c1..a2a08f03b 100644 --- a/airbyte_cdk/sources/declarative/yaml_declarative_source.py +++ b/airbyte_cdk/sources/declarative/yaml_declarative_source.py @@ -40,7 +40,7 @@ def __init__( def _read_and_parse_yaml_file(self, path_to_yaml_file: str) -> ConnectionDefinition: try: - # For testing purposes, we want to allow to just pass a file. However, this + # For testing purposes, we want to allow to just pass a file with open(path_to_yaml_file, "r") as f: return yaml.safe_load(f) # type: ignore # we assume the yaml represents a ConnectionDefinition except FileNotFoundError: