initial first pass converting every component to dataclasses (#15189)

* initial first pass converting every component to dataclasses

* replace the hackier options pass through

* get rid of the hackier way

* fix issues w/ type hints by making options required and lots of fixes to the language to fix compatability for dataclasses

* add dataclasses-jsonschema to setup

* fix oauth authenticator to avoid dataclass name collisions

* fix spacing for CI tests

* remove property from oauth and fix a interpolation bug

* pr feedback and cleaning up the code a bit, attempt at avoiding renaming

* fix templates and bugs surfaced during greenhouse spec testing

* fix tests

* fix missing options in some declarative components

* fix tests related to pulling latest master

* fix issue w/ passing state, slice, and token to subcomponents

* switch name back to get_access_token() since no name collision anymore

airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py

@@ -2,146 +2,94 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+from dataclasses import InitVar, dataclass, field
 from typing import Any, List, Mapping, Optional, Union
 
 import pendulum
 from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import AbstractOauth2Authenticator
+from dataclasses_jsonschema import JsonSchemaMixin
 
 
-class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator):
+@dataclass
+class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, JsonSchemaMixin):
     """
     Generates OAuth2.0 access tokens from an OAuth2.0 refresh token and client credentials based on
     a declarative connector configuration file. Credentials can be defined explicitly or via interpolation
     at runtime. The generated access token is attached to each request via the Authorization header.
+
+    Attributes:
+        token_refresh_endpoint (Union[InterpolatedString, str]): The endpoint to refresh the access token
+        client_id (Union[InterpolatedString, str]): The client id
+        client_secret (Union[InterpolatedString, str]): Client secret
+        refresh_token (Union[InterpolatedString, str]): The token used to refresh the access token
+        access_token_name (Union[InterpolatedString, str]): THe field to extract access token from in the response
+        expires_in_name (Union[InterpolatedString, str]): The field to extract expires_in from in the response
+        config (Mapping[str, Any]): The user-provided configuration as specified by the source's spec
+        scopes (Optional[List[str]]): The scopes to request
+        token_expiry_date (Optional[Union[InterpolatedString, str]]): The access token expiration date
+        refresh_request_body (Optional[Mapping[str, Any]]): The request body to send in the refresh request
     """
 
-    def __init__(
-        self,
-        token_refresh_endpoint: Union[InterpolatedString, str],
-        client_id: Union[InterpolatedString, str],
-        client_secret: Union[InterpolatedString, str],
-        refresh_token: Union[InterpolatedString, str],
-        config: Mapping[str, Any],
-        scopes: Optional[List[str]] = None,
-        token_expiry_date: Optional[Union[InterpolatedString, str]] = None,
-        access_token_name: Union[InterpolatedString, str] = "access_token",
-        expires_in_name: Union[InterpolatedString, str] = "expires_in",
-        refresh_request_body: Optional[Mapping[str, Any]] = None,
-        **options: Optional[Mapping[str, Any]],
-    ):
-        """
-        :param token_refresh_endpoint: The endpoint to refresh the access token
-        :param client_id: The client id
-        :param client_secret: Client secret
-        :param refresh_token: The token used to refresh the access token
-        :param config: The user-provided configuration as specified by the source's spec
-        :param scopes: The scopes to request
-        :param token_expiry_date: The access token expiration date
-        :param access_token_name: THe field to extract access token from in the response
-        :param expires_in_name:The field to extract expires_in from in the response
-        :param refresh_request_body: The request body to send in the refresh request
-        :param options: Additional runtime parameters to be used for string interpolation
-        """
-        self.config = config
-        self.token_refresh_endpoint = InterpolatedString.create(token_refresh_endpoint, options=options)
-        self.client_secret = InterpolatedString.create(client_secret, options=options)
-        self.client_id = InterpolatedString.create(client_id, options=options)
-        self.refresh_token = InterpolatedString.create(refresh_token, options=options)
-        self.scopes = scopes
-        self.access_token_name = InterpolatedString.create(access_token_name, options=options)
-        self.expires_in_name = InterpolatedString.create(expires_in_name, options=options)
-        self.refresh_request_body = InterpolatedMapping(refresh_request_body or {}, options=options)
-
-        self.token_expiry_date = (
-            pendulum.parse(InterpolatedString.create(token_expiry_date, options=options).eval(self.config))
-            if token_expiry_date
+    token_refresh_endpoint: Union[InterpolatedString, str]
+    client_id: Union[InterpolatedString, str]
+    client_secret: Union[InterpolatedString, str]
+    refresh_token: Union[InterpolatedString, str]
+    config: Mapping[str, Any]
+    options: InitVar[Mapping[str, Any]]
+    scopes: Optional[List[str]] = None
+    token_expiry_date: Optional[Union[InterpolatedString, str]] = None
+    _token_expiry_date: pendulum.DateTime = field(init=False, repr=False)
+    access_token_name: Union[InterpolatedString, str] = "access_token"
+    expires_in_name: Union[InterpolatedString, str] = "expires_in"
+    refresh_request_body: Optional[Mapping[str, Any]] = None
+
+    def __post_init__(self, options: Mapping[str, Any]):
+        self.token_refresh_endpoint = InterpolatedString.create(self.token_refresh_endpoint, options=options)
+        self.client_id = InterpolatedString.create(self.client_id, options=options)
+        self.client_secret = InterpolatedString.create(self.client_secret, options=options)
+        self.refresh_token = InterpolatedString.create(self.refresh_token, options=options)
+        self.access_token_name = InterpolatedString.create(self.access_token_name, options=options)
+        self.expires_in_name = InterpolatedString.create(self.expires_in_name, options=options)
+        self._refresh_request_body = InterpolatedMapping(self.refresh_request_body or {}, options=options)
+        self._token_expiry_date = (
+            pendulum.parse(InterpolatedString.create(self.token_expiry_date, options=options).eval(self.config))
+            if self.token_expiry_date
             else pendulum.now().subtract(days=1)
         )
-        self.access_token = None
-
-    @property
-    def config(self) -> Mapping[str, Any]:
-        return self._config
-
-    @config.setter
-    def config(self, value: Mapping[str, Any]):
-        self._config = value
-
-    @property
-    def token_refresh_endpoint(self) -> InterpolatedString:
-        get_some = self._token_refresh_endpoint.eval(self.config)
-        return get_some
+        self._access_token = None
 
-    @token_refresh_endpoint.setter
-    def token_refresh_endpoint(self, value: InterpolatedString):
-        self._token_refresh_endpoint = value
+    def get_token_refresh_endpoint(self) -> str:
+        return self.token_refresh_endpoint.eval(self.config)
 
-    @property
-    def client_id(self) -> InterpolatedString:
-        return self._client_id.eval(self.config)
-
-    @client_id.setter
-    def client_id(self, value: InterpolatedString):
-        self._client_id = value
+    def get_client_id(self) -> str:
+        return self.client_id.eval(self.config)
 
-    @property
-    def client_secret(self) -> InterpolatedString:
-        return self._client_secret.eval(self.config)
+    def get_client_secret(self) -> str:
+        return self.client_secret.eval(self.config)
 
-    @client_secret.setter
-    def client_secret(self, value: InterpolatedString):
-        self._client_secret = value
+    def get_refresh_token(self) -> str:
+        return self.refresh_token.eval(self.config)
 
-    @property
-    def refresh_token(self) -> InterpolatedString:
-        return self._refresh_token.eval(self.config)
+    def get_scopes(self) -> [str]:
+        return self.scopes
 
-    @refresh_token.setter
-    def refresh_token(self, value: InterpolatedString):
-        self._refresh_token = value
+    def get_access_token_name(self) -> InterpolatedString:
+        return self.access_token_name.eval(self.config)
 
-    @property
-    def scopes(self) -> [str]:
-        return self._scopes
+    def get_expires_in_name(self) -> InterpolatedString:
+        return self.expires_in_name.eval(self.config)
 
-    @scopes.setter
-    def scopes(self, value: [str]):
-        self._scopes = value
+    def get_refresh_request_body(self) -> Mapping[str, Any]:
+        return self._refresh_request_body.eval(self.config)
 
-    @property
-    def token_expiry_date(self) -> pendulum.DateTime:
+    def get_token_expiry_date(self) -> pendulum.DateTime:
         return self._token_expiry_date
 
-    @token_expiry_date.setter
-    def token_expiry_date(self, value: pendulum.DateTime):
+    def set_token_expiry_date(self, value: pendulum.DateTime):
         self._token_expiry_date = value
 
-    @property
-    def access_token_name(self) -> InterpolatedString:
-        return self._access_token_name.eval(self.config)
-
-    @access_token_name.setter
-    def access_token_name(self, value: InterpolatedString):
-        self._access_token_name = value
-
-    @property
-    def expires_in_name(self) -> InterpolatedString:
-        return self._expires_in_name.eval(self.config)
-
-    @expires_in_name.setter
-    def expires_in_name(self, value: InterpolatedString):
-        self._expires_in_name = value
-
-    @property
-    def refresh_request_body(self) -> InterpolatedMapping:
-        return self._refresh_request_body.eval(self.config)
-
-    @refresh_request_body.setter
-    def refresh_request_body(self, value: InterpolatedMapping):
-        self._refresh_request_body = value
-
     @property
     def access_token(self) -> str:
         return self._access_token

airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py

@@ -3,14 +3,17 @@
 #
 
 import base64
-from typing import Any, Mapping, Optional, Union
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping, Union
 
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.types import Config
 from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
+from dataclasses_jsonschema import JsonSchemaMixin
 
 
-class ApiKeyAuthenticator(AbstractHeaderAuthenticator):
+@dataclass
+class ApiKeyAuthenticator(AbstractHeaderAuthenticator, JsonSchemaMixin):
     """
     ApiKeyAuth sets a request header on the HTTP requests sent.
 
@@ -22,92 +25,92 @@ class ApiKeyAuthenticator(AbstractHeaderAuthenticator):
     will result in the following header set on the HTTP request
     `"Authorization": "Bearer hello"`
 
+    Attributes:
+        header (Union[InterpolatedString, str]): Header key to set on the HTTP requests
+        api_token (Union[InterpolatedString, str]): Header value to set on the HTTP requests
+        config (Config): The user-provided configuration as specified by the source's spec
+        options (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation
     """
 
-    def __init__(
-        self,
-        header: Union[InterpolatedString, str],
-        token: Union[InterpolatedString, str],
-        config: Config,
-        **options: Optional[Mapping[str, Any]],
-    ):
-        """
-        :param header: Header key to set on the HTTP requests
-        :param token: Header value to set on the HTTP requests
-        :param config: The user-provided configuration as specified by the source's spec
-        :param options: Additional runtime parameters to be used for string interpolation
-        """
-        self._header = InterpolatedString.create(header, options=options)
-        self._token = InterpolatedString.create(token, options=options)
-        self._config = config
+    header: Union[InterpolatedString, str]
+    api_token: Union[InterpolatedString, str]
+    config: Config
+    options: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, options: Mapping[str, Any]):
+        self._header = InterpolatedString.create(self.header, options=options)
+        self._token = InterpolatedString.create(self.api_token, options=options)
 
     @property
     def auth_header(self) -> str:
-        return self._header.eval(self._config)
+        return self._header.eval(self.config)
 
     @property
     def token(self) -> str:
-        return self._token.eval(self._config)
+        return self._token.eval(self.config)
 
 
-class BearerAuthenticator(AbstractHeaderAuthenticator):
+@dataclass
+class BearerAuthenticator(AbstractHeaderAuthenticator, JsonSchemaMixin):
     """
     Authenticator that sets the Authorization header on the HTTP requests sent.
 
     The header is of the form:
     `"Authorization": "Bearer <token>"`
+
+    Attributes:
+        api_token (Union[InterpolatedString, str]): The bearer token
+        config (Config): The user-provided configuration as specified by the source's spec
+        options (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation
     """
 
-    def __init__(self, token: Union[InterpolatedString, str], config: Config, **options: Optional[Mapping[str, Any]]):
-        """
-        :param token: The bearer token
-        :param config: The user-provided configuration as specified by the source's spec
-        :param options: Additional runtime parameters to be used for string interpolation
-        """
-        self._token = InterpolatedString.create(token, options=options)
-        self._config = config
+    api_token: Union[InterpolatedString, str]
+    config: Config
+    options: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, options: Mapping[str, Any]):
+        self._token = InterpolatedString.create(self.api_token, options=options)
 
     @property
     def auth_header(self) -> str:
         return "Authorization"
 
     @property
     def token(self) -> str:
-        return f"Bearer {self._token.eval(self._config)}"
+        return f"Bearer {self._token.eval(self.config)}"
 
 
+@dataclass
 class BasicHttpAuthenticator(AbstractHeaderAuthenticator):
     """
     Builds auth based off the basic authentication scheme as defined by RFC 7617, which transmits credentials as USER ID/password pairs, encoded using bas64
     https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme
 
     The header is of the form
     `"Authorization": "Basic <encoded_credentials>"`
+
+    Attributes:
+        username (Union[InterpolatedString, str]): The username
+        config (Config): The user-provided configuration as specified by the source's spec
+        password (Union[InterpolatedString, str]): The password
+        options (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation
     """
 
-    def __init__(
-        self,
-        username: Union[InterpolatedString, str],
-        config: Config,
-        password: Union[InterpolatedString, str] = "",
-        **options: Optional[Mapping[str, Any]],
-    ):
-        """
-        :param username: The username
-        :param config: The user-provided configuration as specified by the source's spec
-        :param password: The password
-        :param options: Additional runtime parameters to be used for string interpolation
-        """
-        self._username = InterpolatedString.create(username, options=options)
-        self._password = InterpolatedString.create(password, options=options)
-        self._config = config
+    username: Union[InterpolatedString, str]
+    config: Config
+    options: InitVar[Mapping[str, Any]]
+    password: Union[InterpolatedString, str] = ""
+
+    def __post_init__(self, options):
+        self._username = InterpolatedString.create(self.username, options=options)
+        self._password = InterpolatedString.create(self.password, options=options)
 
     @property
     def auth_header(self) -> str:
         return "Authorization"
 
     @property
     def token(self) -> str:
-        auth_string = f"{self._username.eval(self._config)}:{self._password.eval(self._config)}".encode("utf8")
+        auth_string = f"{self._username.eval(self.config)}:{self._password.eval(self.config)}".encode("utf8")
         b64_encoded = base64.b64encode(auth_string).decode("utf8")
         return f"Basic {b64_encoded}"

airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/__init__.py

@@ -1,3 +1,8 @@
 #
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
+
+from airbyte_cdk.sources.declarative.checks.check_stream import CheckStream
+from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
+
+__all__ = ["CheckStream", "ConnectionChecker"]