migrate sentry to config-based cdk (#15345)

* events and projects

* done minus pagination

* handle single records

* pagination

* bump min cdk version

* start on unit tests

* Update more unit tests

* Handle extracting no records from root

* additionalProperties=true

* handle empty streams

* skip backward compatibility tests

* check on project_detail

* remove unit tests

* handle missing keys

* delete stream classes

* record extractor interface

* dpath extractor

* docstring

* handle extract root array

* Use dpath extractor

* Revert "Merge branch 'alex/selectNoRecords' into alex/configbased-sentry"

This reverts commit bad4dd7, reversing
changes made to d9252ae.

* reset to master

* reset to master

* reset to master

* enable backward compatibility test

* bump cdk version

* reset

* Update airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml

Co-authored-by: Sherif A. Nada <snadalive@gmail.com>

* Use paginator

* fix pagination

* bump version

* auto-bump connector version [ci skip]

Co-authored-by: Sherif A. Nada <snadalive@gmail.com>
Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -1115,7 +1115,7 @@
 - sourceDefinitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781
   name: Sentry
   dockerRepository: airbyte/source-sentry
-  dockerImageTag: 0.1.1
+  dockerImageTag: 0.1.2
   documentationUrl: https://docs.airbyte.io/integrations/sources/sentry
   icon: sentry.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -10766,7 +10766,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-sentry:0.1.1"
+- dockerImage: "airbyte/source-sentry:0.1.2"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/sentry"
     connectionSpecification:
@@ -10777,7 +10777,7 @@
       - "auth_token"
       - "organization"
       - "project"
-      additionalProperties: false
+      additionalProperties: true
       properties:
         auth_token:
           type: "string"

airbyte-integrations/connectors/source-sentry/Dockerfile

@@ -34,5 +34,5 @@ COPY source_sentry ./source_sentry
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.version=0.1.2
 LABEL io.airbyte.name=airbyte/source-sentry

airbyte-integrations/connectors/source-sentry/setup.py

@@ -6,7 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-cdk",
+    "airbyte-cdk~=0.1.74",
 ]
 
 TEST_REQUIREMENTS = [

airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml

@@ -0,0 +1,109 @@
+definitions:
+  page_size: 50
+  schema_loader:
+    type: JsonSchema
+    file_path: "./source_sentry/schemas/{{ options.name }}.json"
+  selector:
+    type: RecordSelector
+    extractor:
+      type: DpathExtractor
+      field_pointer: []
+  requester:
+    type: HttpRequester
+    name: "{{ options['name'] }}"
+    url_base: "https://{{ config.hostname }}/api/0/"
+    http_method: "GET"
+    authenticator:
+      type: "BearerAuthenticator"
+      api_token: "{{ config.auth_token }}"
+  paginator:
+    type: LimitPaginator
+    url_base: "*ref(definitions.requester.url_base)"
+    page_size: "*ref(definitions.page_size)"
+    limit_option:
+      inject_into: "request_parameter"
+      field_name: ""
+    page_token_option:
+      inject_into: "request_parameter"
+      field_name: "cursor"
+    pagination_strategy:
+      type: "CursorPagination"
+      cursor_value: "{{ headers.link.next.cursor }}"
+      stop_condition: "{{ headers.link.next.results != 'true' }}"
+  retriever:
+    type: SimpleRetriever
+    name: "{{ options['name'] }}"
+    primary_key: "{{ options['primary_key'] }}"
+
+streams:
+  - type: DeclarativeStream
+    $options:
+      # https://docs.sentry.io/api/events/list-a-projects-events/
+      name: "events"
+    primary_key: "id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "projects/{{config.organization}}/{{config.project}}/events/"
+        request_options_provider:
+          request_parameters:
+            full: "true"
+      paginator:
+        $ref: "*ref(definitions.paginator)"
+  - type: DeclarativeStream
+    $options:
+      name: "issues"
+    primary_key: "id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "projects/{{config.organization}}/{{config.project}}/issues/"
+        request_options_provider:
+          request_parameters:
+            statsPeriod: ""
+            query: ""
+      paginator:
+        $ref: "*ref(definitions.paginator)"
+  - type: DeclarativeStream
+    $options:
+      name: "projects"
+    primary_key: "id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "projects/"
+      paginator:
+        $ref: "*ref(definitions.paginator)"
+  - type: DeclarativeStream
+    $options:
+      name: "project_detail"
+    primary_key: "id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "projects/{{config.organization}}/{{config.project}}/"
+      paginator:
+        type: NoPagination
+check:
+  type: CheckStream
+  stream_names: ["project_detail"]

airbyte-integrations/connectors/source-sentry/source_sentry/source.py

@@ -2,43 +2,16 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
 
-from typing import Any, List, Mapping, Tuple
+"""
+This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
+source connector.
+WARNING: Do not modify this file.
+"""
 
-from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources import AbstractSource
-from airbyte_cdk.sources.streams import Stream
-from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
 
-from .streams import Events, Issues, ProjectDetail, Projects
-
-
-# Source
-class SourceSentry(AbstractSource):
-    def check_connection(self, logger, config) -> Tuple[bool, Any]:
-        try:
-            projects_stream = Projects(
-                authenticator=TokenAuthenticator(token=config["auth_token"]),
-                hostname=config.get("hostname"),
-            )
-            next(projects_stream.read_records(sync_mode=SyncMode.full_refresh))
-            return True, None
-        except Exception as e:
-            return False, e
-
-    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        stream_args = {
-            "authenticator": TokenAuthenticator(token=config["auth_token"]),
-            "hostname": config.get("hostname"),
-        }
-        project_stream_args = {
-            **stream_args,
-            "organization": config["organization"],
-            "project": config["project"],
-        }
-        return [
-            Events(**project_stream_args),
-            Issues(**project_stream_args),
-            ProjectDetail(**project_stream_args),
-            Projects(**stream_args),
-        ]
+# Declarative Source
+class SourceSentry(YamlDeclarativeSource):
+    def __init__(self):
+        super().__init__(**{"path_to_yaml": "./source_sentry/sentry.yaml"})

airbyte-integrations/connectors/source-sentry/source_sentry/spec.json

@@ -5,7 +5,7 @@
     "title": "Sentry Spec",
     "type": "object",
     "required": ["auth_token", "organization", "project"],
-    "additionalProperties": false,
+    "additionalProperties": true,
     "properties": {
       "auth_token": {
         "type": "string",