diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index daf785e5dd580..964339b722544 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.1.67 +- Add support declarative token authenticator. + ## 0.1.66 - Call init_uncaught_exception_handler from AirbyteEntrypoint.__init__ and Destination.run_cmd - Add the ability to remove & add records in YAML-based sources diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py new file mode 100644 index 0000000000000..2b28f7f859418 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py @@ -0,0 +1,98 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import base64 +from typing import Union + +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.types import Config +from airbyte_cdk.sources.streams.http.requests_native_auth.abtract_token import AbstractHeaderAuthenticator + + +class ApiKeyAuthenticator(AbstractHeaderAuthenticator): + """ + ApiKeyAuth sets a request header on the HTTP requests sent. + + The header is of the form: + `"
": ""` + + For example, + `ApiKeyAuthenticator("Authorization", "Bearer hello")` + will result in the following header set on the HTTP request + `"Authorization": "Bearer hello"` + + """ + + def __init__(self, header: Union[InterpolatedString, str], token: Union[InterpolatedString, str], config: Config): + """ + :param header: Header key to set on the HTTP requests + :param token: Header value to set on the HTTP requests + :param config: The user-provided configuration as specified by the source's spec + """ + self._header = InterpolatedString.create(header) + self._token = InterpolatedString.create(token) + self._config = config + + @property + def auth_header(self) -> str: + return self._header.eval(self._config) + + @property + def token(self) -> str: + return self._token.eval(self._config) + + +class BearerAuthenticator(AbstractHeaderAuthenticator): + """ + Authenticator that sets the Authorization header on the HTTP requests sent. + + The header is of the form: + `"Authorization": "Bearer "` + """ + + def __init__(self, token: Union[InterpolatedString, str], config: Config): + """ + :param token: The bearer token + :param config: The user-provided configuration as specified by the source's spec + """ + self._token = InterpolatedString.create(token) + self._config = config + + @property + def auth_header(self) -> str: + return "Authorization" + + @property + def token(self) -> str: + return f"Bearer {self._token.eval(self._config)}" + + +class BasicHttpAuthenticator(AbstractHeaderAuthenticator): + """ + Builds auth based off the basic authentication scheme as defined by RFC 7617, which transmits credentials as USER ID/password pairs, encoded using bas64 + https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme + + The header is of the form + `"Authorization": "Basic "` + """ + + def __init__(self, username: Union[InterpolatedString, str], config: Config, password: Union[InterpolatedString, str] = ""): + """ + :param username: The username + :param config: The user-provided configuration as specified by the source's spec + :param password: The password + """ + self._username = InterpolatedString.create(username) + self._password = InterpolatedString.create(password) + self._config = config + + @property + def auth_header(self) -> str: + return "Authorization" + + @property + def token(self) -> str: + auth_string = f"{self._username.eval(self._config)}:{self._password.eval(self._config)}".encode("utf8") + b64_encoded = base64.b64encode(auth_string).decode("utf8") + return f"Basic {b64_encoded}" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py index 5ec2e94e0996e..e9494effd8cff 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py @@ -45,8 +45,8 @@ def create( """ Helper function to obtain an InterpolatedString from either a raw string or an InterpolatedString. - :param string_or_interpolated: Either a raw string or an InterpolatedString. - :param default: The default value to return if the evaluation returns an empty string + :param string_or_interpolated: either a raw string or an InterpolatedString. + :param options: options parameters propagated from parent component :return: InterpolatedString representing the input string. """ if isinstance(string_or_interpolated, str): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index 7006bfaa79868..4f58d4e64487b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -4,10 +4,13 @@ from typing import Mapping, Type +from airbyte_cdk.sources.declarative.auth.token import ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector +from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.constant_backoff_strategy import ConstantBackoffStrategy from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.exponential_backoff_strategy import ( ExponentialBackoffStrategy, @@ -26,13 +29,15 @@ from airbyte_cdk.sources.declarative.stream_slicers.list_stream_slicer import ListStreamSlicer from airbyte_cdk.sources.declarative.transformations import RemoveFields from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields -from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator """ CLASS_TYPES_REGISTRY contains a mapping of developer-friendly string -> class to abstract the specific class referred to """ CLASS_TYPES_REGISTRY: Mapping[str, Type] = { "AddFields": AddFields, + "ApiKeyAuthenticator": ApiKeyAuthenticator, + "BasicHttpAuthenticator": BasicHttpAuthenticator, + "BearerAuthenticator": BearerAuthenticator, "CartesianProductStreamSlicer": CartesianProductStreamSlicer, "CompositeErrorHandler": CompositeErrorHandler, "ConstantBackoffStrategy": ConstantBackoffStrategy, @@ -42,6 +47,8 @@ "DefaultErrorHandler": DefaultErrorHandler, "ExponentialBackoffStrategy": ExponentialBackoffStrategy, "HttpRequester": HttpRequester, + "InterpolatedBoolean": InterpolatedBoolean, + "InterpolatedString": InterpolatedString, "JelloExtractor": JelloExtractor, "JsonSchema": JsonSchema, "LimitPaginator": LimitPaginator, @@ -52,5 +59,4 @@ "RecordSelector": RecordSelector, "RemoveFields": RemoveFields, "SimpleRetriever": SimpleRetriever, - "TokenAuthenticator": TokenAuthenticator, } diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py index 26e6f60099a1a..eeefe39bceec8 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py @@ -1,10 +1,12 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + from .oauth import Oauth2Authenticator -from .token import MultipleTokenAuthenticator, TokenAuthenticator +from .token import BasicHttpAuthenticator, MultipleTokenAuthenticator, TokenAuthenticator __all__ = [ + "BasicHttpAuthenticator", "Oauth2Authenticator", "TokenAuthenticator", "MultipleTokenAuthenticator", diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abtract_token.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abtract_token.py new file mode 100644 index 0000000000000..57a7d5f82d9e6 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abtract_token.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from abc import abstractmethod +from typing import Any, Mapping + +from requests.auth import AuthBase + + +class AbstractHeaderAuthenticator(AuthBase): + """ + Abstract class for header-based authenticators that set a key-value pair in outgoing HTTP headers + """ + + def __call__(self, request): + """Attach the HTTP headers required to authenticate on the HTTP request""" + request.headers.update(self.get_auth_header()) + return request + + def get_auth_header(self) -> Mapping[str, Any]: + """HTTP header to set on the requests""" + return {self.auth_header: self.token} + + @property + @abstractmethod + def auth_header(self) -> str: + """HTTP header to set on the requests""" + + @property + @abstractmethod + def token(self) -> str: + """Value of the HTTP header to set on the requests""" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/token.py index 77324751fdcd1..b5708f297919f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/token.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/token.py @@ -4,49 +4,70 @@ import base64 from itertools import cycle -from typing import Any, List, Mapping +from typing import List -from requests.auth import AuthBase +from airbyte_cdk.sources.streams.http.requests_native_auth.abtract_token import AbstractHeaderAuthenticator -class MultipleTokenAuthenticator(AuthBase): +class MultipleTokenAuthenticator(AbstractHeaderAuthenticator): """ Builds auth header, based on the list of tokens provided. Auth header is changed per each `get_auth_header` call, using each token in cycle. The token is attached to each request via the `auth_header` header. """ + @property + def auth_header(self) -> str: + return self._auth_header + + @property + def token(self) -> str: + return f"{self._auth_method} {next(self._tokens_iter)}" + def __init__(self, tokens: List[str], auth_method: str = "Bearer", auth_header: str = "Authorization"): - self.auth_method = auth_method - self.auth_header = auth_header + self._auth_method = auth_method + self._auth_header = auth_header self._tokens = tokens self._tokens_iter = cycle(self._tokens) - def __call__(self, request): - request.headers.update(self.get_auth_header()) - return request - - def get_auth_header(self) -> Mapping[str, Any]: - return {self.auth_header: f"{self.auth_method} {next(self._tokens_iter)}"} - -class TokenAuthenticator(MultipleTokenAuthenticator): +class TokenAuthenticator(AbstractHeaderAuthenticator): """ Builds auth header, based on the token provided. The token is attached to each request via the `auth_header` header. """ + @property + def auth_header(self) -> str: + return self._auth_header + + @property + def token(self) -> str: + return f"{self._auth_method} {self._token}" + def __init__(self, token: str, auth_method: str = "Bearer", auth_header: str = "Authorization"): - super().__init__([token], auth_method, auth_header) + self._auth_header = auth_header + self._auth_method = auth_method + self._token = token -class BasicHttpAuthenticator(TokenAuthenticator): +class BasicHttpAuthenticator(AbstractHeaderAuthenticator): """ Builds auth based off the basic authentication scheme as defined by RFC 7617, which transmits credentials as USER ID/password pairs, encoded using bas64 https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme """ + @property + def auth_header(self) -> str: + return self._auth_header + + @property + def token(self) -> str: + return f"{self._auth_method} {self._token}" + def __init__(self, username: str, password: str = "", auth_method: str = "Basic", auth_header: str = "Authorization"): auth_string = f"{username}:{password}".encode("utf8") b64_encoded = base64.b64encode(auth_string).decode("utf8") - super().__init__(b64_encoded, auth_method, auth_header) + self._auth_header = auth_header + self._auth_method = auth_method + self._token = b64_encoded diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 903006f23c746..6960e3131c75f 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.1.66", + version="0.1.67", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py new file mode 100644 index 0000000000000..b46f9eac643e0 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging + +import pytest +import requests +from airbyte_cdk.sources.declarative.auth.token import ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator +from requests import Response + +LOGGER = logging.getLogger(__name__) + +resp = Response() +config = {"username": "user", "password": "password", "header": "header"} + + +@pytest.mark.parametrize( + "test_name, token, expected_header_value", + [ + ("test_static_token", "test-token", "Bearer test-token"), + ("test_token_from_config", "{{ config.username }}", "Bearer user"), + ], +) +def test_bearer_token_authenticator(test_name, token, expected_header_value): + """ + Should match passed in token, no matter how many times token is retrieved. + """ + token_auth = BearerAuthenticator(token, config) + header1 = token_auth.get_auth_header() + header2 = token_auth.get_auth_header() + + prepared_request = requests.PreparedRequest() + prepared_request.headers = {} + token_auth(prepared_request) + + assert {"Authorization": expected_header_value} == prepared_request.headers + assert {"Authorization": expected_header_value} == header1 + assert {"Authorization": expected_header_value} == header2 + + +@pytest.mark.parametrize( + "test_name, username, password, expected_header_value", + [ + ("test_static_creds", "user", "password", "Basic dXNlcjpwYXNzd29yZA=="), + ("test_creds_from_config", "{{ config.username }}", "{{ config.password }}", "Basic dXNlcjpwYXNzd29yZA=="), + ], +) +def test_basic_authenticator(test_name, username, password, expected_header_value): + """ + Should match passed in token, no matter how many times token is retrieved. + """ + token_auth = BasicHttpAuthenticator(username=username, password=password, config=config) + header1 = token_auth.get_auth_header() + header2 = token_auth.get_auth_header() + + prepared_request = requests.PreparedRequest() + prepared_request.headers = {} + token_auth(prepared_request) + + assert {"Authorization": expected_header_value} == prepared_request.headers + assert {"Authorization": expected_header_value} == header1 + assert {"Authorization": expected_header_value} == header2 + + +@pytest.mark.parametrize( + "test_name, header, token, expected_header, expected_header_value", + [ + ("test_static_token", "Authorization", "test-token", "Authorization", "test-token"), + ("test_token_from_config", "{{ config.header }}", "{{ config.username }}", "header", "user"), + ], +) +def test_api_key_authenticator(test_name, header, token, expected_header, expected_header_value): + """ + Should match passed in token, no matter how many times token is retrieved. + """ + token_auth = ApiKeyAuthenticator(header, token, config) + header1 = token_auth.get_auth_header() + header2 = token_auth.get_auth_header() + + prepared_request = requests.PreparedRequest() + prepared_request.headers = {} + token_auth(prepared_request) + + assert {expected_header: expected_header_value} == prepared_request.headers + assert {expected_header: expected_header_value} == header1 + assert {expected_header: expected_header_value} == header2 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index 88b856f288d7e..26152d3de5c62 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -4,6 +4,7 @@ import datetime +from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder @@ -27,7 +28,6 @@ from airbyte_cdk.sources.declarative.stream_slicers.datetime_stream_slicer import DatetimeStreamSlicer from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition -from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator factory = DeclarativeComponentFactory() @@ -73,6 +73,7 @@ def test_interpolate_config(): authenticator = factory.create_component(config["authenticator"], input_config)() assert authenticator._client_id.eval(input_config) == "some_client_id" assert authenticator._client_secret._string == "some_client_secret" + assert authenticator._token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth" assert authenticator._refresh_token.eval(input_config) == "verysecrettoken" assert authenticator._refresh_request_body._mapping == {"body_field": "yoyoyo", "interpolated_body_field": "{{ config['apikey'] }}"} @@ -172,7 +173,7 @@ def test_full_config(): url_base: "https://api.sendgrid.com/v3/" http_method: "GET" authenticator: - class_name: airbyte_cdk.sources.streams.http.requests_native_auth.token.TokenAuthenticator + type: BearerAuthenticator token: "{{ config['apikey'] }}" request_parameters_provider: "*ref(request_options_provider)" error_handler: @@ -231,7 +232,7 @@ def test_full_config(): assert type(stream._schema_loader) == JsonSchema assert type(stream._retriever) == SimpleRetriever assert stream._retriever._requester._method == HttpMethod.GET - assert stream._retriever._requester._authenticator._tokens == ["verysecrettoken"] + assert stream._retriever._requester._authenticator._token.eval(input_config) == "verysecrettoken" assert type(stream._retriever._record_selector) == RecordSelector assert type(stream._retriever._record_selector._extractor._decoder) == JsonDecoder @@ -275,11 +276,13 @@ def test_create_requester(): requester: type: HttpRequester path: "/v3/marketing/lists" - name: lists + options: + name: lists url_base: "https://api.sendgrid.com" authenticator: - type: "TokenAuthenticator" - token: "{{ config.apikey }}" + type: "BasicHttpAuthenticator" + username: "{{ config.apikey }}" + password: "{{ config.apikey }}" request_options_provider: request_parameters: page_size: 10 @@ -292,7 +295,9 @@ def test_create_requester(): assert isinstance(component._error_handler, DefaultErrorHandler) assert component._path._string == "/v3/marketing/lists" assert component._url_base._string == "https://api.sendgrid.com" - assert isinstance(component._authenticator, TokenAuthenticator) + assert isinstance(component._authenticator, BasicHttpAuthenticator) + assert component._authenticator._username.eval(input_config) == "verysecrettoken" + assert component._authenticator._password.eval(input_config) == "verysecrettoken" assert component._method == HttpMethod.GET assert component._request_options_provider._parameter_interpolator._interpolator._mapping["page_size"] == 10 assert component._request_options_provider._headers_interpolator._interpolator._mapping["header"] == "header_value" @@ -346,7 +351,7 @@ def test_config_with_defaults(): requester: path: "/v3/marketing/lists" authenticator: - type: "TokenAuthenticator" + type: "BearerAuthenticator" token: "{{ config.apikey }}" request_parameters: page_size: 10 @@ -366,7 +371,8 @@ def test_config_with_defaults(): assert type(stream._schema_loader) == JsonSchema assert type(stream._retriever) == SimpleRetriever assert stream._retriever._requester._method == HttpMethod.GET - assert stream._retriever._requester._authenticator._tokens == ["verysecrettoken"] + + assert stream._retriever._requester._authenticator._token.eval(input_config) == "verysecrettoken" assert stream._retriever._record_selector._extractor._transform.eval(input_config) == "_.result" assert stream._schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.yaml" assert isinstance(stream._retriever._paginator, LimitPaginator) @@ -407,7 +413,7 @@ class TestCreateTransformations: primary_key: id url_base: "https://api.sendgrid.com" schema_loader: - file_path: "./source_sendgrid/schemas/{{options.name}}.yaml" + file_path: "./source_sendgrid/schemas/{{name}}.yaml" retriever: requester: path: "/v3/marketing/lists" diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index 5185950f4a8f4..c70c88ecdbc4c 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -6,7 +6,12 @@ import pendulum import requests -from airbyte_cdk.sources.streams.http.requests_native_auth import MultipleTokenAuthenticator, Oauth2Authenticator, TokenAuthenticator +from airbyte_cdk.sources.streams.http.requests_native_auth import ( + BasicHttpAuthenticator, + MultipleTokenAuthenticator, + Oauth2Authenticator, + TokenAuthenticator, +) from requests import Response LOGGER = logging.getLogger(__name__) @@ -31,6 +36,23 @@ def test_token_authenticator(): assert {"Authorization": "Bearer test-token"} == header2 +def test_basic_http_authenticator(): + """ + Should match passed in token, no matter how many times token is retrieved. + """ + token_auth = BasicHttpAuthenticator(username="user", password="password") + header1 = token_auth.get_auth_header() + header2 = token_auth.get_auth_header() + + prepared_request = requests.PreparedRequest() + prepared_request.headers = {} + token_auth(prepared_request) + + assert {"Authorization": "Basic dXNlcjpwYXNzd29yZA=="} == prepared_request.headers + assert {"Authorization": "Basic dXNlcjpwYXNzd29yZA=="} == header1 + assert {"Authorization": "Basic dXNlcjpwYXNzd29yZA=="} == header2 + + def test_multiple_token_authenticator(): multiple_token_auth = MultipleTokenAuthenticator(tokens=["token1", "token2"]) header1 = multiple_token_auth.get_auth_header()