From 581ee80673d2b4d7ae0dc051b45b195fd374d7bc Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 21 Jul 2022 07:27:06 -0700 Subject: [PATCH 01/10] interpolatedauth --- .../sources/declarative/auth/oauth.py | 17 ++-- .../sources/declarative/auth/token.py | 84 ++++++++++++++++++ .../interpolation/interpolated_string.py | 26 +++++- .../parsers/class_types_registry.py | 6 +- .../requests_native_auth/abtract_token.py | 27 ++++++ .../http/requests_native_auth/token.py | 53 +++++++---- .../declarative/auth/test_token_auth.py | 87 +++++++++++++++++++ .../sources/declarative/test_factory.py | 36 ++++---- .../test_requests_native_auth.py | 24 ++++- 9 files changed, 314 insertions(+), 46 deletions(-) create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abtract_token.py create mode 100644 airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py index 95cc5f162b3c9..d764b57d9e784 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py @@ -29,19 +29,20 @@ def __init__( access_token_name: str = "access_token", expires_in_name: str = "expires_in", refresh_request_body: Mapping[str, Any] = None, + **kwargs, ): self.config = config - self.token_refresh_endpoint = InterpolatedString(token_refresh_endpoint) - self.client_secret = InterpolatedString(client_secret) - self.client_id = InterpolatedString(client_id) - self.refresh_token = InterpolatedString(refresh_token) + self.token_refresh_endpoint = InterpolatedString.create(token_refresh_endpoint, options=kwargs) + self.client_secret = InterpolatedString.create(client_secret, options=kwargs) + self.client_id = InterpolatedString.create(client_id, options=kwargs) + self.refresh_token = InterpolatedString.create(refresh_token, options=kwargs) self.scopes = scopes - self.access_token_name = InterpolatedString(access_token_name) - self.expires_in_name = InterpolatedString(expires_in_name) - self.refresh_request_body = InterpolatedMapping(refresh_request_body) + self.access_token_name = InterpolatedString.create(access_token_name, options=kwargs) + self.expires_in_name = InterpolatedString.create(expires_in_name, options=kwargs) + self.refresh_request_body = InterpolatedMapping(refresh_request_body, options=kwargs) self.token_expiry_date = ( - pendulum.parse(InterpolatedString(token_expiry_date).eval(self.config)) + pendulum.parse(InterpolatedString.create(token_expiry_date, options=kwargs).eval(self.config)) if token_expiry_date else pendulum.now().subtract(days=1) ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py new file mode 100644 index 0000000000000..7e547b9332c2d --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py @@ -0,0 +1,84 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import base64 +from typing import Union + +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.types import Config +from airbyte_cdk.sources.streams.http.requests_native_auth.abtract_token import AbstractHeaderAuthenticator + + +class ApiKeyAuth(AbstractHeaderAuthenticator): + """ + ApiKeyAuth sets a request header on the HTTP requests sent + """ + + def __init__(self, header: Union[InterpolatedString, str], token: Union[InterpolatedString, str], config: Config, **kwargs): + """ + :param header: Header key to set on the HTTP requests + :param token: Header value to set on the HTTP requests + :param config: connection config + """ + self._header = InterpolatedString.create(header, options=kwargs) + self._token = InterpolatedString.create(token, options=kwargs) + self._config = config + + @property + def auth_header(self) -> str: + return self._header.eval(self._config) + + @property + def token(self) -> str: + return self._token.eval(self._config) + + +class BearerAuth(AbstractHeaderAuthenticator): + """ + Authenticator that sets the Authorization header on the HTTP requests sent. + `Authorization: Bearer ` + """ + + def __init__(self, token: Union[InterpolatedString, str], config: Config, **kwargs): + """ + :param token: + :param config: + """ + self._token = InterpolatedString.create(token, options=kwargs) + self._config = config + + @property + def auth_header(self) -> str: + return "Authorization" + + @property + def token(self) -> str: + return f"Bearer {self._token.eval(self._config)}" + + +class BasicHttpAuth(AbstractHeaderAuthenticator): + """ + Builds auth based off the basic authentication scheme as defined by RFC 7617, which transmits credentials as USER ID/password pairs, encoded using bas64 + https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme + """ + + def __init__(self, username: Union[InterpolatedString, str], config: Config, password: Union[InterpolatedString, str] = "", **kwargs): + """ + :param username: + :param config: + :param password: + """ + self._username = InterpolatedString.create(username, options=kwargs) + self._password = InterpolatedString.create(password, options=kwargs) + self._config = config + + @property + def auth_header(self) -> str: + return "Authorization" + + @property + def token(self) -> str: + auth_string = f"{self._username.eval(self._config)}:{self._password.eval(self._config)}".encode("utf8") + b64_encoded = base64.b64encode(auth_string).decode("utf8") + return f"Basic {b64_encoded}" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py index fffc66eb6751e..7b6aaa9eaf097 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py @@ -2,21 +2,41 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import Optional +from typing import Any, Mapping, Optional, Union from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation class InterpolatedString: - def __init__(self, string: str, default: Optional[str] = None): + def __init__(self, string: str, default: Optional[str] = None, options=None): self._string = string self._default = default or string self._interpolation = JinjaInterpolation() + self._options = options or {} def eval(self, config, **kwargs): - return self._interpolation.eval(self._string, config, self._default, **kwargs) + return self._interpolation.eval(self._string, config, self._default, options=self._options, **kwargs) def __eq__(self, other): if not isinstance(other, InterpolatedString): return False return self._string == other._string and self._default == other._default + + @classmethod + def create( + cls, + string_or_interpolated: Union["InterpolatedString", str], + /, + options: Mapping[str, Any], + default=None, + ): + """ + Helper function to obtain an InterpolatedString from either a raw string or an InterpolatedString. + :param string_or_interpolated: either a raw string or an InterpolatedString. + :param options: options parameters propagated from parent component + :return: InterpolatedString representing the input string. + """ + if isinstance(string_or_interpolated, str): + return InterpolatedString(string_or_interpolated, default=default, options=options) + else: + return string_or_interpolated diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index 8d85ff7196ebd..603127d26dc95 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -4,6 +4,7 @@ from typing import Mapping, Type +from airbyte_cdk.sources.declarative.auth.token import ApiKeyAuth, BasicHttpAuth, BearerAuth from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor @@ -22,7 +23,6 @@ from airbyte_cdk.sources.declarative.stream_slicers.list_stream_slicer import ListStreamSlicer from airbyte_cdk.sources.declarative.transformations import RemoveFields from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields -from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator CLASS_TYPES_REGISTRY: Mapping[str, Type] = { "AddFields": AddFields, @@ -41,5 +41,7 @@ "MinMaxDatetime": MinMaxDatetime, "OffsetIncrement": OffsetIncrement, "RemoveFields": RemoveFields, - "TokenAuthenticator": TokenAuthenticator, + "ApiKeyAuth": ApiKeyAuth, + "BearerAuth": BearerAuth, + "BasicHttpAuth": BasicHttpAuth, } diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abtract_token.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abtract_token.py new file mode 100644 index 0000000000000..f52fb7bea27cb --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abtract_token.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from abc import abstractmethod +from typing import Any, Mapping + +from requests.auth import AuthBase + + +class AbstractHeaderAuthenticator(AuthBase): + def __call__(self, request): + request.headers.update(self.get_auth_header()) + return request + + def get_auth_header(self) -> Mapping[str, Any]: + return {self.auth_header: self.token} + + @property + @abstractmethod + def auth_header(self) -> str: + pass + + @property + @abstractmethod + def token(self) -> str: + pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/token.py index d117c24a44bb8..0c70c3f43487b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/token.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/token.py @@ -4,49 +4,70 @@ import base64 from itertools import cycle -from typing import Any, List, Mapping +from typing import List -from requests.auth import AuthBase +from airbyte_cdk.sources.streams.http.requests_native_auth.abtract_token import AbstractHeaderAuthenticator -class MultipleTokenAuthenticator(AuthBase): +class MultipleTokenAuthenticator(AbstractHeaderAuthenticator): """ Builds auth header, based on the list of tokens provided. Auth header is changed per each `get_auth_header` call, using each token in cycle. The token is attached to each request via the `auth_header` header. """ + @property + def auth_header(self) -> str: + return self._auth_header + + @property + def token(self) -> str: + return f"{self._auth_method} {next(self._tokens_iter)}" + def __init__(self, tokens: List[str], auth_method: str = "Bearer", auth_header: str = "Authorization"): - self.auth_method = auth_method - self.auth_header = auth_header + self._auth_method = auth_method + self._auth_header = auth_header self._tokens = tokens self._tokens_iter = cycle(self._tokens) - def __call__(self, request): - request.headers.update(self.get_auth_header()) - return request - - def get_auth_header(self) -> Mapping[str, Any]: - return {self.auth_header: f"{self.auth_method} {next(self._tokens_iter)}"} - -class TokenAuthenticator(MultipleTokenAuthenticator): +class TokenAuthenticator(AbstractHeaderAuthenticator): """ Builds auth header, based on the token provided. The token is attached to each request via the `auth_header` header. """ + @property + def auth_header(self) -> str: + return self._auth_header + + @property + def token(self) -> str: + return f"{self._auth_method} {self._token}" + def __init__(self, token: str, auth_method: str = "Bearer", auth_header: str = "Authorization"): - super().__init__([token], auth_method, auth_header) + self._auth_header = auth_header + self._auth_method = auth_method + self._token = token -class BasicHttpAuthenticator(TokenAuthenticator): +class BasicHttpAuthenticator(AbstractHeaderAuthenticator): """ Builds auth based off the basic authentication scheme as defined by RFC 7617, which transmits credentials as USER ID/password pairs, encoded using bas64 https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme """ + @property + def auth_header(self) -> str: + return self._auth_header + + @property + def token(self) -> str: + return f"{self._auth_method} {self._token}" + def __init__(self, username: str, password: str, auth_method: str = "Basic", auth_header: str = "Authorization"): auth_string = f"{username}:{password}".encode("utf8") b64_encoded = base64.b64encode(auth_string).decode("utf8") - super().__init__(b64_encoded, auth_method, auth_header) + self._auth_header = auth_header + self._auth_method = auth_method + self._token = b64_encoded diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py new file mode 100644 index 0000000000000..54d97f252fe0f --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging + +import pytest +import requests +from airbyte_cdk.sources.declarative.auth.token import ApiKeyAuth, BasicHttpAuth, BearerAuth +from requests import Response + +LOGGER = logging.getLogger(__name__) + +resp = Response() +config = {"username": "user", "password": "password", "header": "header"} + + +@pytest.mark.parametrize( + "test_name, token, expected_header_value", + [ + ("test_static_token", "test-token", "Bearer test-token"), + ("test_token_from_config", "{{ config.username }}", "Bearer user"), + ], +) +def test_bearer_token_authenticator(test_name, token, expected_header_value): + """ + Should match passed in token, no matter how many times token is retrieved. + """ + token_auth = BearerAuth(token, config) + header1 = token_auth.get_auth_header() + header2 = token_auth.get_auth_header() + + prepared_request = requests.PreparedRequest() + prepared_request.headers = {} + token_auth(prepared_request) + + assert {"Authorization": expected_header_value} == prepared_request.headers + assert {"Authorization": expected_header_value} == header1 + assert {"Authorization": expected_header_value} == header2 + + +@pytest.mark.parametrize( + "test_name, username, password, expected_header_value", + [ + ("test_static_creds", "user", "password", "Basic dXNlcjpwYXNzd29yZA=="), + ("test_creds_from_config", "{{ config.username }}", "{{ config.password }}", "Basic dXNlcjpwYXNzd29yZA=="), + ], +) +def test_basic_authenticator(test_name, username, password, expected_header_value): + """ + Should match passed in token, no matter how many times token is retrieved. + """ + token_auth = BasicHttpAuth(username=username, password=password, config=config) + header1 = token_auth.get_auth_header() + header2 = token_auth.get_auth_header() + + prepared_request = requests.PreparedRequest() + prepared_request.headers = {} + token_auth(prepared_request) + + assert {"Authorization": expected_header_value} == prepared_request.headers + assert {"Authorization": expected_header_value} == header1 + assert {"Authorization": expected_header_value} == header2 + + +@pytest.mark.parametrize( + "test_name, header, token, expected_header, expected_header_value", + [ + ("test_static_token", "Authorization", "test-token", "Authorization", "test-token"), + ("test_token_from_config", "{{ config.header }}", "{{ config.username }}", "header", "user"), + ], +) +def test_api_key_authenticator(test_name, header, token, expected_header, expected_header_value): + """ + Should match passed in token, no matter how many times token is retrieved. + """ + token_auth = ApiKeyAuth(header, token, config) + header1 = token_auth.get_auth_header() + header2 = token_auth.get_auth_header() + + prepared_request = requests.PreparedRequest() + prepared_request.headers = {} + token_auth(prepared_request) + + assert {expected_header: expected_header_value} == prepared_request.headers + assert {expected_header: expected_header_value} == header1 + assert {expected_header: expected_header_value} == header2 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index a0dab74052945..b9929e46a465e 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -4,6 +4,7 @@ import datetime +from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuth from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder @@ -27,7 +28,6 @@ from airbyte_cdk.sources.declarative.stream_slicers.datetime_stream_slicer import DatetimeStreamSlicer from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition -from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator factory = DeclarativeComponentFactory() @@ -74,7 +74,7 @@ def test_interpolate_config(): assert authenticator._client_id._string == "some_client_id" assert authenticator._client_secret._string == "some_client_secret" assert authenticator._token_refresh_endpoint._string == "https://api.sendgrid.com/v3/auth" - assert authenticator._refresh_token._string == "verysecrettoken" + assert authenticator._refresh_token.eval(input_config) == "verysecrettoken" assert authenticator._refresh_request_body._mapping == {"body_field": "yoyoyo", "interpolated_body_field": "{{ config['apikey'] }}"} @@ -109,7 +109,7 @@ def test_datetime_stream_slicer(): content = """ stream_slicer: type: DatetimeStreamSlicer - options: + $options: datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -172,7 +172,7 @@ def test_full_config(): url_base: "https://api.sendgrid.com/v3/" http_method: "GET" authenticator: - class_name: airbyte_cdk.sources.streams.http.requests_native_auth.token.TokenAuthenticator + type: BearerAuth token: "{{ config['apikey'] }}" request_parameters_provider: "*ref(request_options_provider)" error_handler: @@ -195,7 +195,7 @@ def test_full_config(): cursor_field: [ ] list_stream: ref: "*ref(partial_stream)" - options: + $options: name: "lists" primary_key: "id" extractor: @@ -231,7 +231,7 @@ def test_full_config(): assert type(stream._schema_loader) == JsonSchema assert type(stream._retriever) == SimpleRetriever assert stream._retriever._requester._method == HttpMethod.GET - assert stream._retriever._requester._authenticator._tokens == ["verysecrettoken"] + assert stream._retriever._requester._authenticator._token.eval(input_config) == "verysecrettoken" assert type(stream._retriever._record_selector) == RecordSelector assert type(stream._retriever._record_selector._extractor._decoder) == JsonDecoder @@ -275,11 +275,13 @@ def test_create_requester(): requester: type: HttpRequester path: "/v3/marketing/lists" - name: lists + $options: + name: lists url_base: "https://api.sendgrid.com" authenticator: - type: "TokenAuthenticator" - token: "{{ config.apikey }}" + type: "BasicHttpAuth" + username: "{{ options.name }}" + password: "{{ config.apikey }}" request_options_provider: request_parameters: page_size: 10 @@ -292,7 +294,9 @@ def test_create_requester(): assert isinstance(component._error_handler, DefaultErrorHandler) assert component._path._string == "/v3/marketing/lists" assert component._url_base._string == "https://api.sendgrid.com" - assert isinstance(component._authenticator, TokenAuthenticator) + assert isinstance(component._authenticator, BasicHttpAuth) + assert component._authenticator._username.eval(input_config) == "lists" + assert component._authenticator._password.eval(input_config) == "verysecrettoken" assert component._method == HttpMethod.GET assert component._request_options_provider._parameter_interpolator._interpolator._mapping["page_size"] == 10 assert component._request_options_provider._headers_interpolator._interpolator._mapping["header"] == "header_value" @@ -325,7 +329,7 @@ def test_config_with_defaults(): content = """ lists_stream: type: "DeclarativeStream" - options: + $options: name: "lists" primary_key: id url_base: "https://api.sendgrid.com" @@ -346,7 +350,7 @@ def test_config_with_defaults(): requester: path: "/v3/marketing/lists" authenticator: - type: "TokenAuthenticator" + type: "BearerAuth" token: "{{ config.apikey }}" request_parameters: page_size: 10 @@ -366,7 +370,7 @@ def test_config_with_defaults(): assert type(stream._schema_loader) == JsonSchema assert type(stream._retriever) == SimpleRetriever assert stream._retriever._requester._method == HttpMethod.GET - assert stream._retriever._requester._authenticator._tokens == ["verysecrettoken"] + assert stream._retriever._requester._authenticator._token.eval(input_config) == "verysecrettoken" assert stream._retriever._record_selector._extractor._transform == ".result[]" assert stream._schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.yaml" assert isinstance(stream._retriever._paginator, LimitPaginator) @@ -422,7 +426,7 @@ def test_no_transformations(self): content = f""" the_stream: type: DeclarativeStream - options: + $options: {self.base_options} """ config = parser.parse(content) @@ -434,7 +438,7 @@ def test_remove_fields(self): content = f""" the_stream: type: DeclarativeStream - options: + $options: {self.base_options} transformations: - type: RemoveFields @@ -452,7 +456,7 @@ def test_add_fields(self): content = f""" the_stream: class_name: airbyte_cdk.sources.declarative.declarative_stream.DeclarativeStream - options: + $options: {self.base_options} transformations: - type: AddFields diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index 5185950f4a8f4..c70c88ecdbc4c 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -6,7 +6,12 @@ import pendulum import requests -from airbyte_cdk.sources.streams.http.requests_native_auth import MultipleTokenAuthenticator, Oauth2Authenticator, TokenAuthenticator +from airbyte_cdk.sources.streams.http.requests_native_auth import ( + BasicHttpAuthenticator, + MultipleTokenAuthenticator, + Oauth2Authenticator, + TokenAuthenticator, +) from requests import Response LOGGER = logging.getLogger(__name__) @@ -31,6 +36,23 @@ def test_token_authenticator(): assert {"Authorization": "Bearer test-token"} == header2 +def test_basic_http_authenticator(): + """ + Should match passed in token, no matter how many times token is retrieved. + """ + token_auth = BasicHttpAuthenticator(username="user", password="password") + header1 = token_auth.get_auth_header() + header2 = token_auth.get_auth_header() + + prepared_request = requests.PreparedRequest() + prepared_request.headers = {} + token_auth(prepared_request) + + assert {"Authorization": "Basic dXNlcjpwYXNzd29yZA=="} == prepared_request.headers + assert {"Authorization": "Basic dXNlcjpwYXNzd29yZA=="} == header1 + assert {"Authorization": "Basic dXNlcjpwYXNzd29yZA=="} == header2 + + def test_multiple_token_authenticator(): multiple_token_auth = MultipleTokenAuthenticator(tokens=["token1", "token2"]) header1 = multiple_token_auth.get_auth_header() From fd863f42f11f29e144f54a10ba1d519d9930177b Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 21 Jul 2022 07:32:28 -0700 Subject: [PATCH 02/10] fix tests --- .../sources/declarative/auth/oauth.py | 17 ++++++++-------- .../sources/declarative/auth/token.py | 16 +++++++-------- .../interpolation/interpolated_string.py | 10 ++++------ .../sources/declarative/test_factory.py | 20 +++++++++---------- 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py index d764b57d9e784..82ba539afe5ef 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py @@ -29,20 +29,19 @@ def __init__( access_token_name: str = "access_token", expires_in_name: str = "expires_in", refresh_request_body: Mapping[str, Any] = None, - **kwargs, ): self.config = config - self.token_refresh_endpoint = InterpolatedString.create(token_refresh_endpoint, options=kwargs) - self.client_secret = InterpolatedString.create(client_secret, options=kwargs) - self.client_id = InterpolatedString.create(client_id, options=kwargs) - self.refresh_token = InterpolatedString.create(refresh_token, options=kwargs) + self.token_refresh_endpoint = InterpolatedString.create(token_refresh_endpoint) + self.client_secret = InterpolatedString.create(client_secret) + self.client_id = InterpolatedString.create(client_id) + self.refresh_token = InterpolatedString.create(refresh_token) self.scopes = scopes - self.access_token_name = InterpolatedString.create(access_token_name, options=kwargs) - self.expires_in_name = InterpolatedString.create(expires_in_name, options=kwargs) - self.refresh_request_body = InterpolatedMapping(refresh_request_body, options=kwargs) + self.access_token_name = InterpolatedString.create(access_token_name) + self.expires_in_name = InterpolatedString.create(expires_in_name) + self.refresh_request_body = InterpolatedMapping(refresh_request_body) self.token_expiry_date = ( - pendulum.parse(InterpolatedString.create(token_expiry_date, options=kwargs).eval(self.config)) + pendulum.parse(InterpolatedString.create(token_expiry_date).eval(self.config)) if token_expiry_date else pendulum.now().subtract(days=1) ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py index 7e547b9332c2d..51fcd7f36e66c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py @@ -15,14 +15,14 @@ class ApiKeyAuth(AbstractHeaderAuthenticator): ApiKeyAuth sets a request header on the HTTP requests sent """ - def __init__(self, header: Union[InterpolatedString, str], token: Union[InterpolatedString, str], config: Config, **kwargs): + def __init__(self, header: Union[InterpolatedString, str], token: Union[InterpolatedString, str], config: Config): """ :param header: Header key to set on the HTTP requests :param token: Header value to set on the HTTP requests :param config: connection config """ - self._header = InterpolatedString.create(header, options=kwargs) - self._token = InterpolatedString.create(token, options=kwargs) + self._header = InterpolatedString.create(header) + self._token = InterpolatedString.create(token) self._config = config @property @@ -40,12 +40,12 @@ class BearerAuth(AbstractHeaderAuthenticator): `Authorization: Bearer ` """ - def __init__(self, token: Union[InterpolatedString, str], config: Config, **kwargs): + def __init__(self, token: Union[InterpolatedString, str], config: Config): """ :param token: :param config: """ - self._token = InterpolatedString.create(token, options=kwargs) + self._token = InterpolatedString.create(token) self._config = config @property @@ -63,14 +63,14 @@ class BasicHttpAuth(AbstractHeaderAuthenticator): https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme """ - def __init__(self, username: Union[InterpolatedString, str], config: Config, password: Union[InterpolatedString, str] = "", **kwargs): + def __init__(self, username: Union[InterpolatedString, str], config: Config, password: Union[InterpolatedString, str] = ""): """ :param username: :param config: :param password: """ - self._username = InterpolatedString.create(username, options=kwargs) - self._password = InterpolatedString.create(password, options=kwargs) + self._username = InterpolatedString.create(username) + self._password = InterpolatedString.create(password) self._config = config @property diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py index 7b6aaa9eaf097..0a0897852c409 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py @@ -2,20 +2,19 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import Any, Mapping, Optional, Union +from typing import Optional, Union from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation class InterpolatedString: - def __init__(self, string: str, default: Optional[str] = None, options=None): + def __init__(self, string: str, default: Optional[str] = None): self._string = string self._default = default or string self._interpolation = JinjaInterpolation() - self._options = options or {} def eval(self, config, **kwargs): - return self._interpolation.eval(self._string, config, self._default, options=self._options, **kwargs) + return self._interpolation.eval(self._string, config, self._default, **kwargs) def __eq__(self, other): if not isinstance(other, InterpolatedString): @@ -27,7 +26,6 @@ def create( cls, string_or_interpolated: Union["InterpolatedString", str], /, - options: Mapping[str, Any], default=None, ): """ @@ -37,6 +35,6 @@ def create( :return: InterpolatedString representing the input string. """ if isinstance(string_or_interpolated, str): - return InterpolatedString(string_or_interpolated, default=default, options=options) + return InterpolatedString(string_or_interpolated, default=default) else: return string_or_interpolated diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index b9929e46a465e..b29ce705c00d5 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -109,7 +109,7 @@ def test_datetime_stream_slicer(): content = """ stream_slicer: type: DatetimeStreamSlicer - $options: + options: datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -195,7 +195,7 @@ def test_full_config(): cursor_field: [ ] list_stream: ref: "*ref(partial_stream)" - $options: + options: name: "lists" primary_key: "id" extractor: @@ -275,12 +275,12 @@ def test_create_requester(): requester: type: HttpRequester path: "/v3/marketing/lists" - $options: + options: name: lists url_base: "https://api.sendgrid.com" authenticator: type: "BasicHttpAuth" - username: "{{ options.name }}" + username: "{{ config.apikey }}" password: "{{ config.apikey }}" request_options_provider: request_parameters: @@ -295,7 +295,7 @@ def test_create_requester(): assert component._path._string == "/v3/marketing/lists" assert component._url_base._string == "https://api.sendgrid.com" assert isinstance(component._authenticator, BasicHttpAuth) - assert component._authenticator._username.eval(input_config) == "lists" + assert component._authenticator._username.eval(input_config) == "verysecrettoken" assert component._authenticator._password.eval(input_config) == "verysecrettoken" assert component._method == HttpMethod.GET assert component._request_options_provider._parameter_interpolator._interpolator._mapping["page_size"] == 10 @@ -329,7 +329,7 @@ def test_config_with_defaults(): content = """ lists_stream: type: "DeclarativeStream" - $options: + options: name: "lists" primary_key: id url_base: "https://api.sendgrid.com" @@ -411,7 +411,7 @@ class TestCreateTransformations: primary_key: id url_base: "https://api.sendgrid.com" schema_loader: - file_path: "./source_sendgrid/schemas/{{options.name}}.yaml" + file_path: "./source_sendgrid/schemas/{{name}}.yaml" retriever: requester: path: "/v3/marketing/lists" @@ -426,7 +426,7 @@ def test_no_transformations(self): content = f""" the_stream: type: DeclarativeStream - $options: + options: {self.base_options} """ config = parser.parse(content) @@ -438,7 +438,7 @@ def test_remove_fields(self): content = f""" the_stream: type: DeclarativeStream - $options: + options: {self.base_options} transformations: - type: RemoveFields @@ -456,7 +456,7 @@ def test_add_fields(self): content = f""" the_stream: class_name: airbyte_cdk.sources.declarative.declarative_stream.DeclarativeStream - $options: + options: {self.base_options} transformations: - type: AddFields From 5a6608be49281c414db7d0daf0c7594b6539c550 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 25 Jul 2022 09:47:47 -0700 Subject: [PATCH 03/10] fix import --- .../sources/streams/http/requests_native_auth/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py index 26e6f60099a1a..eeefe39bceec8 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py @@ -1,10 +1,12 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + from .oauth import Oauth2Authenticator -from .token import MultipleTokenAuthenticator, TokenAuthenticator +from .token import BasicHttpAuthenticator, MultipleTokenAuthenticator, TokenAuthenticator __all__ = [ + "BasicHttpAuthenticator", "Oauth2Authenticator", "TokenAuthenticator", "MultipleTokenAuthenticator", From c277a01f9ee031fbf80b540e5f4c77565c843504 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 25 Jul 2022 18:04:56 -0700 Subject: [PATCH 04/10] no need for default --- .../sources/declarative/interpolation/interpolated_string.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py index 0a0897852c409..e4f6dc7d68d77 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py @@ -25,8 +25,6 @@ def __eq__(self, other): def create( cls, string_or_interpolated: Union["InterpolatedString", str], - /, - default=None, ): """ Helper function to obtain an InterpolatedString from either a raw string or an InterpolatedString. @@ -35,6 +33,6 @@ def create( :return: InterpolatedString representing the input string. """ if isinstance(string_or_interpolated, str): - return InterpolatedString(string_or_interpolated, default=default) + return InterpolatedString(string_or_interpolated) else: return string_or_interpolated From 303002bc08443f3c93d9353e097cbff74846f541 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 25 Jul 2022 18:06:24 -0700 Subject: [PATCH 05/10] Bump version --- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index daf785e5dd580..964339b722544 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.1.67 +- Add support declarative token authenticator. + ## 0.1.66 - Call init_uncaught_exception_handler from AirbyteEntrypoint.__init__ and Destination.run_cmd - Add the ability to remove & add records in YAML-based sources diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 903006f23c746..6960e3131c75f 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.1.66", + version="0.1.67", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From 7129f3a1dc993ed62b53396ffcf6690ec14922b8 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 25 Jul 2022 18:15:02 -0700 Subject: [PATCH 06/10] Missing docstrings --- .../streams/http/requests_native_auth/abtract_token.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abtract_token.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abtract_token.py index f52fb7bea27cb..57a7d5f82d9e6 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abtract_token.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abtract_token.py @@ -9,19 +9,25 @@ class AbstractHeaderAuthenticator(AuthBase): + """ + Abstract class for header-based authenticators that set a key-value pair in outgoing HTTP headers + """ + def __call__(self, request): + """Attach the HTTP headers required to authenticate on the HTTP request""" request.headers.update(self.get_auth_header()) return request def get_auth_header(self) -> Mapping[str, Any]: + """HTTP header to set on the requests""" return {self.auth_header: self.token} @property @abstractmethod def auth_header(self) -> str: - pass + """HTTP header to set on the requests""" @property @abstractmethod def token(self) -> str: - pass + """Value of the HTTP header to set on the requests""" From 535050ee764b76bd42d70c607912e4dc07368ef7 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 25 Jul 2022 18:17:13 -0700 Subject: [PATCH 07/10] example --- .../airbyte_cdk/sources/declarative/auth/token.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py index 89b9ddad006cf..df2eac791a578 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py @@ -12,7 +12,16 @@ class ApiKeyAuthenticator(AbstractHeaderAuthenticator): """ - ApiKeyAuth sets a request header on the HTTP requests sent + ApiKeyAuth sets a request header on the HTTP requests sent. + + The header is of the form: + `"
": ""` + + For example, + `ApiKeyAuthenticator("Authorization", "Bearer hello")` + will result in the following header set on the HTTP request + `"Authorization": "Bearer hello"` + """ def __init__(self, header: Union[InterpolatedString, str], token: Union[InterpolatedString, str], config: Config): From b68caf1cf4a1bf7456c2a04581c5c35a9a4ad679 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 25 Jul 2022 18:18:51 -0700 Subject: [PATCH 08/10] missing example --- .../python/airbyte_cdk/sources/declarative/auth/token.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py index df2eac791a578..aa87f6b08023a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py @@ -46,7 +46,9 @@ def token(self) -> str: class BearerAuthenticator(AbstractHeaderAuthenticator): """ Authenticator that sets the Authorization header on the HTTP requests sent. - `Authorization: Bearer ` + + The header is of the form: + `"Authorization": "Bearer "` """ def __init__(self, token: Union[InterpolatedString, str], config: Config): @@ -70,6 +72,9 @@ class BasicHttpAuthenticator(AbstractHeaderAuthenticator): """ Builds auth based off the basic authentication scheme as defined by RFC 7617, which transmits credentials as USER ID/password pairs, encoded using bas64 https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme + + The header is of the form + `"Authorization": "Basic "` """ def __init__(self, username: Union[InterpolatedString, str], config: Config, password: Union[InterpolatedString, str] = ""): From f242af874b524fd4682a1ccb0c355269f9ac9a61 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 25 Jul 2022 18:19:47 -0700 Subject: [PATCH 09/10] more docstrings --- .../airbyte_cdk/sources/declarative/auth/token.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py index aa87f6b08023a..2b28f7f859418 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py @@ -28,7 +28,7 @@ def __init__(self, header: Union[InterpolatedString, str], token: Union[Interpol """ :param header: Header key to set on the HTTP requests :param token: Header value to set on the HTTP requests - :param config: connection config + :param config: The user-provided configuration as specified by the source's spec """ self._header = InterpolatedString.create(header) self._token = InterpolatedString.create(token) @@ -53,8 +53,8 @@ class BearerAuthenticator(AbstractHeaderAuthenticator): def __init__(self, token: Union[InterpolatedString, str], config: Config): """ - :param token: - :param config: + :param token: The bearer token + :param config: The user-provided configuration as specified by the source's spec """ self._token = InterpolatedString.create(token) self._config = config @@ -79,9 +79,9 @@ class BasicHttpAuthenticator(AbstractHeaderAuthenticator): def __init__(self, username: Union[InterpolatedString, str], config: Config, password: Union[InterpolatedString, str] = ""): """ - :param username: - :param config: - :param password: + :param username: The username + :param config: The user-provided configuration as specified by the source's spec + :param password: The password """ self._username = InterpolatedString.create(username) self._password = InterpolatedString.create(password) From acf12f620bc49e2b4b75dd59e8ca06c0c35e6db4 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 25 Jul 2022 18:26:47 -0700 Subject: [PATCH 10/10] interpolated types --- .../sources/declarative/parsers/class_types_registry.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index c6204f153b3b7..4f58d4e64487b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -9,6 +9,8 @@ from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector +from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.constant_backoff_strategy import ConstantBackoffStrategy from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.exponential_backoff_strategy import ( ExponentialBackoffStrategy, @@ -45,6 +47,8 @@ "DefaultErrorHandler": DefaultErrorHandler, "ExponentialBackoffStrategy": ExponentialBackoffStrategy, "HttpRequester": HttpRequester, + "InterpolatedBoolean": InterpolatedBoolean, + "InterpolatedString": InterpolatedString, "JelloExtractor": JelloExtractor, "JsonSchema": JsonSchema, "LimitPaginator": LimitPaginator,