From 2f8d3dc41792dc0f14c5d31579f9807b2035b7e3 Mon Sep 17 00:00:00 2001
From: Oleksandr Bazarnov <oleksandr.bazarnov@globallogic.com>
Date: Wed, 12 Mar 2025 17:53:20 +0200
Subject: [PATCH 1/2] fix
---
.../sources/declarative/requesters/http_requester.py | 11 ++++++++---
.../declarative/requesters/test_http_requester.py | 6 +++---
2 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte_cdk/sources/declarative/requesters/http_requester.py
index 45671fc59..82a30925e 100644
--- a/airbyte_cdk/sources/declarative/requesters/http_requester.py
+++ b/airbyte_cdk/sources/declarative/requesters/http_requester.py
@@ -132,7 +132,7 @@ def get_url_base(
stream_slice=stream_slice,
next_page_token=next_page_token,
)
- return os.path.join(self._url_base.eval(self.config, **interpolation_context), EmptyString)
+ return str(self._url_base.eval(self.config, **interpolation_context))
def get_path(
self,
@@ -370,13 +370,18 @@ def _join_url(cls, url_base: str, path: str) -> str:
Example:
1) _join_url("https://example.com/api/", "endpoint") >> 'https://example.com/api/endpoint'
2) _join_url("https://example.com/api", "/endpoint") >> 'https://example.com/api/endpoint'
- 3) _join_url("https://example.com/api/", "") >> 'https://example.com/api'
+ 3) _join_url("https://example.com/api/", "") >> 'https://example.com/api/'
4) _join_url("https://example.com/api", None) >> 'https://example.com/api'
"""
# return a full-url if provided directly from interpolation context
if path == EmptyString or path is None:
- return url_base.rstrip("/")
+ return url_base
+ else:
+ # since we didn't provide a full-url, the url_base might not have a trailing slash
+ # so we join the url_base and path correctly
+ if not url_base.endswith("/"):
+ url_base = os.path.join(url_base, EmptyString)
return urljoin(url_base, path)
diff --git a/unit_tests/sources/declarative/requesters/test_http_requester.py b/unit_tests/sources/declarative/requesters/test_http_requester.py
index dfe78011a..8fce688d7 100644
--- a/unit_tests/sources/declarative/requesters/test_http_requester.py
+++ b/unit_tests/sources/declarative/requesters/test_http_requester.py
@@ -121,7 +121,7 @@ def test_http_requester():
parameters={},
)
- assert requester.get_url_base() == "https://airbyte.io/"
+ assert requester.get_url_base() == "https://airbyte.io"
assert (
requester.get_path(stream_state={}, stream_slice=stream_slice, next_page_token={})
== "v1/1234"
@@ -145,9 +145,9 @@ def test_http_requester():
@pytest.mark.parametrize(
"test_name, base_url, expected_base_url",
[
- ("test_no_trailing_slash", "https://example.com", "https://example.com/"),
+ ("test_no_trailing_slash", "https://example.com", "https://example.com"),
("test_with_trailing_slash", "https://example.com/", "https://example.com/"),
- ("test_with_v1_no_trailing_slash", "https://example.com/v1", "https://example.com/v1/"),
+ ("test_with_v1_no_trailing_slash", "https://example.com/v1", "https://example.com/v1"),
("test_with_v1_with_trailing_slash", "https://example.com/v1/", "https://example.com/v1/"),
],
)
From 649c6d03481ecab06e437c23566ea2ea9fea3e4f Mon Sep 17 00:00:00 2001
From: Oleksandr Bazarnov <oleksandr.bazarnov@globallogic.com>
Date: Wed, 12 Mar 2025 20:14:18 +0200
Subject: [PATCH 2/2] updated else condition
---
airbyte_cdk/sources/declarative/requesters/http_requester.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte_cdk/sources/declarative/requesters/http_requester.py
index 82a30925e..78c07b725 100644
--- a/airbyte_cdk/sources/declarative/requesters/http_requester.py
+++ b/airbyte_cdk/sources/declarative/requesters/http_requester.py
@@ -381,7 +381,7 @@ def _join_url(cls, url_base: str, path: str) -> str:
# since we didn't provide a full-url, the url_base might not have a trailing slash
# so we join the url_base and path correctly
if not url_base.endswith("/"):
- url_base = os.path.join(url_base, EmptyString)
+ url_base += "/"
return urljoin(url_base, path)