From 2f8d3dc41792dc0f14c5d31579f9807b2035b7e3 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 12 Mar 2025 17:53:20 +0200 Subject: [PATCH 1/2] fix --- .../sources/declarative/requesters/http_requester.py | 11 ++++++++--- .../declarative/requesters/test_http_requester.py | 6 +++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte_cdk/sources/declarative/requesters/http_requester.py index 45671fc59..82a30925e 100644 --- a/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -132,7 +132,7 @@ def get_url_base( stream_slice=stream_slice, next_page_token=next_page_token, ) - return os.path.join(self._url_base.eval(self.config, **interpolation_context), EmptyString) + return str(self._url_base.eval(self.config, **interpolation_context)) def get_path( self, @@ -370,13 +370,18 @@ def _join_url(cls, url_base: str, path: str) -> str: Example: 1) _join_url("https://example.com/api/", "endpoint") >> 'https://example.com/api/endpoint' 2) _join_url("https://example.com/api", "/endpoint") >> 'https://example.com/api/endpoint' - 3) _join_url("https://example.com/api/", "") >> 'https://example.com/api' + 3) _join_url("https://example.com/api/", "") >> 'https://example.com/api/' 4) _join_url("https://example.com/api", None) >> 'https://example.com/api' """ # return a full-url if provided directly from interpolation context if path == EmptyString or path is None: - return url_base.rstrip("/") + return url_base + else: + # since we didn't provide a full-url, the url_base might not have a trailing slash + # so we join the url_base and path correctly + if not url_base.endswith("/"): + url_base = os.path.join(url_base, EmptyString) return urljoin(url_base, path) diff --git a/unit_tests/sources/declarative/requesters/test_http_requester.py b/unit_tests/sources/declarative/requesters/test_http_requester.py index dfe78011a..8fce688d7 100644 --- a/unit_tests/sources/declarative/requesters/test_http_requester.py +++ b/unit_tests/sources/declarative/requesters/test_http_requester.py @@ -121,7 +121,7 @@ def test_http_requester(): parameters={}, ) - assert requester.get_url_base() == "https://airbyte.io/" + assert requester.get_url_base() == "https://airbyte.io" assert ( requester.get_path(stream_state={}, stream_slice=stream_slice, next_page_token={}) == "v1/1234" @@ -145,9 +145,9 @@ def test_http_requester(): @pytest.mark.parametrize( "test_name, base_url, expected_base_url", [ - ("test_no_trailing_slash", "https://example.com", "https://example.com/"), + ("test_no_trailing_slash", "https://example.com", "https://example.com"), ("test_with_trailing_slash", "https://example.com/", "https://example.com/"), - ("test_with_v1_no_trailing_slash", "https://example.com/v1", "https://example.com/v1/"), + ("test_with_v1_no_trailing_slash", "https://example.com/v1", "https://example.com/v1"), ("test_with_v1_with_trailing_slash", "https://example.com/v1/", "https://example.com/v1/"), ], ) From 649c6d03481ecab06e437c23566ea2ea9fea3e4f Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 12 Mar 2025 20:14:18 +0200 Subject: [PATCH 2/2] updated else condition --- airbyte_cdk/sources/declarative/requesters/http_requester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte_cdk/sources/declarative/requesters/http_requester.py index 82a30925e..78c07b725 100644 --- a/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -381,7 +381,7 @@ def _join_url(cls, url_base: str, path: str) -> str: # since we didn't provide a full-url, the url_base might not have a trailing slash # so we join the url_base and path correctly if not url_base.endswith("/"): - url_base = os.path.join(url_base, EmptyString) + url_base += "/" return urljoin(url_base, path)