🎉 Source Zendesk Sunshine: support oauth

airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.1.1
 LABEL io.airbyte.name=airbyte/source-zendesk-sunshine

airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml

@@ -7,13 +7,25 @@ tests:
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"
+    - config_path: "secrets/config_oauth.json"
+      status: "succeed"
+    - config_path: "secrets/config_api_token.json"
+      status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
+    - config_path: "integration_tests/invalid_config_api_token.json"
+      status: "failed"
+    - config_path: "integration_tests/invalid_config_oauth.json"
+      status: "failed"
   discovery:
     - config_path: "secrets/config.json"
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
+    - config_path: "secrets/config_api_token.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+    - config_path: "secrets/config_oauth.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
 #  incremental: #  complex state ( {parent_id: {cur_field: value}} still not supported )
 #    - config_path: "secrets/config.json"
 #      configured_catalog_path: "integration_tests/configured_catalog.json"

airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/invalid_config_api_token.json

@@ -0,0 +1,9 @@
+{
+  "credentials": {
+    "auth_method": "api_token",
+    "email": "test@ayhghghte.io",
+    "api_token": "fgfgvf ghnbvg hnghbvnhbvnvbn"
+  },
+  "subdomain": "d3v-airbyte",
+  "start_date": "2020-01-01T00:00:00Z"
+}

airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/invalid_config_oauth.json

@@ -0,0 +1,10 @@
+{
+  "credentials": {
+    "auth_method": "oauth2.0",
+    "client_id": "some_client_id",
+    "client_secret": "some_client_secret",
+    "access_token": "some_access_token"
+  },
+  "subdomain": "d3v-airbyte",
+  "start_date": "2020-01-01T00:00:00Z"
+}

airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/source.py

@@ -4,7 +4,7 @@
 
 
 import base64
-from typing import Any, List, Mapping, Tuple
+from typing import Any, List, Mapping, Tuple, Union
 
 import pendulum
 from airbyte_cdk.logger import AirbyteLogger
@@ -23,11 +23,24 @@ def __init__(self, auth: Tuple[str, str], auth_method: str = "Basic", **kwargs):
         super().__init__(token=b64_encoded, auth_method=auth_method, **kwargs)
 
 
+class ZendeskSunshineAuthenticator:
+    """Provides the authentication capabilities for both old and new methods."""
+
+    @staticmethod
+    def get_auth(config: Mapping[str, Any]) -> Union[Base64HttpAuthenticator, TokenAuthenticator]:
+        credentials = config.get("credentials", {})
+        token = config.get("api_token") or credentials.get("api_token")
+        email = config.get("email") or credentials.get("email")
+        if email and token:
+            return Base64HttpAuthenticator(auth=(f"{email}/token", token))
+        return TokenAuthenticator(token=credentials["access_token"])
+
+
 class SourceZendeskSunshine(AbstractSource):
     def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
         try:
             pendulum.parse(config["start_date"], strict=True)
-            authenticator = Base64HttpAuthenticator(auth=(f'{config["email"]}/token', config["api_token"]))
+            authenticator = ZendeskSunshineAuthenticator.get_auth(config)
             stream = Limits(authenticator=authenticator, subdomain=config["subdomain"], start_date=pendulum.parse(config["start_date"]))
             records = stream.read_records(sync_mode=SyncMode.full_refresh)
             next(records)
@@ -47,7 +60,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         After this time is passed we have no data. It will require permanent population, to pass
         the test criteria `stream should contain at least 1 record)
         """
-        authenticator = Base64HttpAuthenticator(auth=(f'{config["email"]}/token', config["api_token"]))
+        authenticator = ZendeskSunshineAuthenticator.get_auth(config)
         args = {"authenticator": authenticator, "subdomain": config["subdomain"], "start_date": config["start_date"]}
         return [
             ObjectTypes(**args),

airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/spec.json

@@ -4,28 +4,136 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Zendesk Sunshine Spec",
     "type": "object",
-    "required": ["api_token", "email", "start_date", "subdomain"],
-    "additionalProperties": false,
+    "required": ["start_date", "subdomain"],
+    "additionalProperties": true,
     "properties": {
-      "api_token": {
-        "type": "string",
-        "airbyte_secret": true,
-        "description": "API Token. See the <a href=\"https://docs.airbyte.io/integrations/sources/zendesk_sunshine\">docs</a> for information on how to generate this key."
-      },
-      "email": {
-        "type": "string",
-        "description": "The user email for your Zendesk account"
-      },
       "subdomain": {
+        "title": "Subdomain",
         "type": "string",
-        "description": "The subdomain for your Zendesk Account"
+        "description": "The subdomain for your Zendesk Account."
       },
       "start_date": {
         "title": "Start Date",
         "type": "string",
-        "description": "The date from which you'd like to replicate the data",
+        "description": "The date from which you'd like to replicate data for Zendesk Sunshine API, in the format YYYY-MM-DDT00:00:00Z.",
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
-        "examples": "2021-01-01T00:00:00.000000Z"
+        "examples": ["2021-01-01T00:00:00Z"]
+      },
+      "credentials": {
+        "title": "Authorization Method",
+        "type": "object",
+        "oneOf": [
+          {
+            "type": "object",
+            "title": "OAuth2.0",
+            "required": ["auth_method", "client_id", "client_secret", "access_token"],
+            "properties": {
+              "auth_method": {
+                "type": "string",
+                "const": "oauth2.0",
+                "enum": ["oauth2.0"],
+                "default": "oauth2.0",
+                "order": 0
+              },
+              "client_id": {
+                "type": "string",
+                "title": "Client ID",
+                "description": "The Client ID of your OAuth application.",
+                "airbyte_secret": true
+              },
+              "client_secret": {
+                "type": "string",
+                "title": "Client Secret",
+                "description": "The Client Secret of your OAuth application.",
+                "airbyte_secret": true
+              },
+              "access_token": {
+                "type": "string",
+                "title": "Access Token",
+                "description": "Long-term access Token for making authenticated requests.",
+                "airbyte_secret": true
+              }
+            }
+          },
+          {
+            "type": "object",
+            "title": "API Token",
+            "required": ["auth_method", "api_token", "email"],
+            "properties": {
+              "auth_method": {
+                "type": "string",
+                "const": "api_token",
+                "enum": ["api_token"],
+                "default": "api_token",
+                "order": 1
+              },
+              "api_token": {
+                "type": "string",
+                "title": "API Token",
+                "description": "API Token. See the <a href=\"https://docs.airbyte.io/integrations/sources/zendesk_sunshine\">docs</a> for information on how to generate this key.",
+                "airbyte_secret": true
+              },
+              "email": {
+                "type": "string",
+                "title": "Email",
+                "description": "The user email for your Zendesk account"
+              }
+            }
+          }
+        ]
+      }
+    }
+  },
+  "advanced_auth": {
+    "auth_flow_type": "oauth2.0",
+    "predicate_key": ["credentials", "auth_method"],
+    "predicate_value": "oauth2.0",
+    "oauth_config_specification": {
+      "complete_oauth_output_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "access_token": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "access_token"]
+          }
+        }
+      },
+      "complete_oauth_server_input_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "client_id": {
+            "type": "string"
+          },
+          "client_secret": {
+            "type": "string"
+          }
+        }
+      },
+      "complete_oauth_server_output_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "client_id": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_id"]
+          },
+          "client_secret": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_secret"]
+          }
+        }
+      },
+      "oauth_user_input_from_connector_config_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "subdomain": {
+            "type": "string",
+            "path_in_connector_config": ["subdomain"]
+          }
+        }
       }
     }
   }

airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/streams.py

@@ -77,6 +77,8 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late
 
 
 class ObjectTypes(SunshineStream):
+    primary_key = "key"
+
     def path(self, **kwargs) -> str:
         return "objects/types"
 
@@ -134,6 +136,8 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late
 
 
 class RelationshipTypes(SunshineStream):
+    primary_key = "key"
+
     def path(self, **kwargs) -> str:
         return "relationships/types"
 
@@ -170,6 +174,8 @@ def path(self, **kwargs) -> str:
 
 
 class ObjectTypePolicies(SunshineStream):
+    primary_key = None
+
     def stream_slices(self, **kwargs):
         parent_stream = ObjectTypes(authenticator=self.authenticator, subdomain=self.subdomain, start_date=self._start_date)
         for obj_type in parent_stream.read_records(sync_mode=SyncMode.full_refresh):
@@ -200,5 +206,7 @@ def path(self, **kwargs) -> str:
 
 
 class Limits(SunshineStream):
+    primary_key = "key"
+
     def path(self, **kwargs) -> str:
         return "limits"

airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java

@@ -54,6 +54,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final
         .put("airbyte/source-drift", new DriftOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-zendesk-chat", new ZendeskChatOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-monday", new MondayOAuthFlow(configRepository, httpClient))
+        .put("airbyte/source-zendesk-sunshine", new ZendeskSunshineOAuthFlow(configRepository, httpClient))
         .build();
   }
 

airbyte-oauth/src/main/java/io/airbyte/oauth/flows/ZendeskSunshineOAuthFlow.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.oauth.flows;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import io.airbyte.config.persistence.ConfigRepository;
+import io.airbyte.oauth.BaseOAuth2Flow;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.http.HttpClient;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Supplier;
+import org.apache.http.client.utils.URIBuilder;
+
+/**
+ * Following docs from
+ * https://developer.zendesk.com/api-reference/custom-data/introduction/#authentication
+ */
+public class ZendeskSunshineOAuthFlow extends BaseOAuth2Flow {
+
+  public ZendeskSunshineOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) {
+    super(configRepository, httpClient);
+  }
+
+  @VisibleForTesting
+  public ZendeskSunshineOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier<String> stateSupplier) {
+    super(configRepository, httpClient, stateSupplier);
+  }
+
+  @Override
+  protected String formatConsentUrl(final UUID definitionId,
+                                    final String clientId,
+                                    final String redirectUrl,
+                                    final JsonNode inputOAuthConfiguration)
+      throws IOException {
+
+    // getting subdomain value from user's config
+    final String subdomain = getConfigValueUnsafe(inputOAuthConfiguration, "subdomain");
+
+    final URIBuilder builder = new URIBuilder()
+        .setScheme("https")
+        .setHost(String.format("%s.zendesk.com", subdomain))
+        .setPath("oauth/authorizations/new")
+        // required
+        .addParameter("response_type", "code")
+        .addParameter("redirect_uri", redirectUrl)
+        .addParameter("client_id", clientId)
+        .addParameter("scope", "read")
+        .addParameter("state", getState());
+
+    try {
+      return builder.build().toString();
+    } catch (final URISyntaxException e) {
+      throw new IOException("Failed to format Consent URL for OAuth flow", e);
+    }
+  }
+
+  @Override
+  protected Map<String, String> getAccessTokenQueryParameters(String clientId,
+                                                              String clientSecret,
+                                                              String authCode,
+                                                              String redirectUrl) {
+    return ImmutableMap.<String, String>builder()
+        // required
+        .put("grant_type", "authorization_code")
+        .put("code", authCode)
+        .put("client_id", clientId)
+        .put("client_secret", clientSecret)
+        .put("redirect_uri", redirectUrl)
+        .put("scope", "read")
+        .build();
+  }
+
+  @Override
+  protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) {
+    // getting subdomain value from user's config
+    final String subdomain = getConfigValueUnsafe(inputOAuthConfiguration, "subdomain");
+
+    return String.format("https://%s.zendesk.com/oauth/tokens", subdomain);
+  }
+
+  @Override
+  protected Map<String, Object> extractOAuthOutput(final JsonNode data, final String accessTokenUrl) throws IOException {
+    final Map<String, Object> result = new HashMap<>();
+    // getting out access_token
+    if (data.has("access_token")) {
+      result.put("access_token", data.get("access_token").asText());
+    } else {
+      throw new IOException(String.format("Missing 'access_token' in query params from %s", accessTokenUrl));
+    }
+    return result;
+  }
+
+}